From a79f31b006b012e2e3e9d96791259b662ec1dadc Mon Sep 17 00:00:00 2001 From: Yash Solanki Date: Tue, 7 Feb 2023 02:29:59 +0530 Subject: [PATCH] Feature/add accounts import export (#1635) * Add accounts to activities export * Add logic for importing accounts * Update changelog --- CHANGELOG.md | 5 + .../api/src/app/account/create-account.dto.ts | 4 + apps/api/src/app/export/export.service.ts | 17 ++++ apps/api/src/app/import/import-data.dto.ts | 9 +- apps/api/src/app/import/import.controller.ts | 9 +- apps/api/src/app/import/import.service.ts | 93 +++++++++++++++++-- .../import-activities-dialog.component.ts | 25 +++-- .../app/services/import-activities.service.ts | 42 ++++++--- .../src/lib/interfaces/export.interface.ts | 3 +- test/import/ok-without-accounts.json | 48 ++++++++++ test/import/ok.json | 21 ++++- 11 files changed, 242 insertions(+), 34 deletions(-) create mode 100644 test/import/ok-without-accounts.json diff --git a/CHANGELOG.md b/CHANGELOG.md index cadd77304..10f1db5fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added support to export accounts +- Added suport to import accounts + ### Changed - Improved the styling in the admin control panel diff --git a/apps/api/src/app/account/create-account.dto.ts b/apps/api/src/app/account/create-account.dto.ts index 3ea13e20a..6495fa518 100644 --- a/apps/api/src/app/account/create-account.dto.ts +++ b/apps/api/src/app/account/create-account.dto.ts @@ -17,6 +17,10 @@ export class CreateAccountDto { @IsString() currency: string; + @IsOptional() + @IsString() + id?: string; + @IsBoolean() @IsOptional() isExcluded?: boolean; diff --git a/apps/api/src/app/export/export.service.ts b/apps/api/src/app/export/export.service.ts index c4655e7d8..c57f1df4e 100644 --- a/apps/api/src/app/export/export.service.ts +++ b/apps/api/src/app/export/export.service.ts @@ -14,6 +14,22 @@ export class ExportService { activityIds?: string[]; userId: string; }): Promise { + const accounts = await this.prismaService.account.findMany({ + orderBy: { + name: 'asc' + }, + select: { + accountType: true, + balance: true, + currency: true, + id: true, + isExcluded: true, + name: true, + platformId: true + }, + where: { userId } + }); + let activities = await this.prismaService.order.findMany({ orderBy: { date: 'desc' }, select: { @@ -38,6 +54,7 @@ export class ExportService { return { meta: { date: new Date().toISOString(), version: environment.version }, + accounts, activities: activities.map( ({ accountId, diff --git a/apps/api/src/app/import/import-data.dto.ts b/apps/api/src/app/import/import-data.dto.ts index f3a0ba8fe..fa954a2c9 100644 --- a/apps/api/src/app/import/import-data.dto.ts +++ b/apps/api/src/app/import/import-data.dto.ts @@ -1,8 +1,15 @@ +import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto'; import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; import { Type } from 'class-transformer'; -import { IsArray, ValidateNested } from 'class-validator'; +import { IsArray, IsOptional, ValidateNested } from 'class-validator'; export class ImportDataDto { + @IsOptional() + @IsArray() + @Type(() => CreateAccountDto) + @ValidateNested({ each: true }) + accounts: CreateAccountDto[]; + @IsArray() @Type(() => CreateOrderDto) @ValidateNested({ each: true }) diff --git a/apps/api/src/app/import/import.controller.ts b/apps/api/src/app/import/import.controller.ts index 2591ab638..93724bf02 100644 --- a/apps/api/src/app/import/import.controller.ts +++ b/apps/api/src/app/import/import.controller.ts @@ -2,6 +2,7 @@ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interce import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; import { ImportResponse } from '@ghostfolio/common/interfaces'; +import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import type { RequestWithUser } from '@ghostfolio/common/types'; import { Body, @@ -38,7 +39,10 @@ export class ImportController { @Body() importData: ImportDataDto, @Query('dryRun') isDryRun?: boolean ): Promise { - if (!this.configurationService.get('ENABLE_FEATURE_IMPORT')) { + if ( + !this.configurationService.get('ENABLE_FEATURE_IMPORT') || + !hasPermission(this.request.user.permissions, permissions.createAccount) + ) { throw new HttpException( getReasonPhrase(StatusCodes.FORBIDDEN), StatusCodes.FORBIDDEN @@ -60,9 +64,10 @@ export class ImportController { try { const activities = await this.importService.import({ - maxActivitiesToImport, isDryRun, + maxActivitiesToImport, userCurrency, + accountsDto: importData.accounts ?? [], activitiesDto: importData.activities, userId: this.request.user.id }); diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index d3be33bbc..37693f160 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -1,4 +1,5 @@ import { AccountService } from '@ghostfolio/api/app/account/account.service'; +import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto'; import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; import { OrderService } from '@ghostfolio/api/app/order/order.service'; @@ -100,18 +101,75 @@ export class ImportService { } public async import({ + accountsDto, activitiesDto, isDryRun = false, maxActivitiesToImport, userCurrency, userId }: { + accountsDto: Partial[]; activitiesDto: Partial[]; isDryRun?: boolean; maxActivitiesToImport: number; userCurrency: string; userId: string; }): Promise { + const accountIdMapping: { [oldAccountId: string]: string } = {}; + + if (!isDryRun && accountsDto?.length) { + const existingAccounts = await this.accountService.accounts({ + where: { + id: { + in: accountsDto.map(({ id }) => { + return id; + }) + } + } + }); + + for (const account of accountsDto) { + // Check if there is any existing account with the same ID + const accountWithSameId = existingAccounts.find( + (existingAccount) => existingAccount.id === account.id + ); + + // If there is no account or if the account belongs to a different user then create a new account + if (!accountWithSameId || accountWithSameId.userId !== userId) { + let oldAccountId: string; + const platformId = account.platformId; + + delete account.platformId; + + if (accountWithSameId) { + oldAccountId = account.id; + delete account.id; + } + + const newAccountObject = { + ...account, + User: { connect: { id: userId } } + }; + + if (platformId) { + Object.assign(newAccountObject, { + Platform: { connect: { id: platformId } } + }); + } + + const newAccount = await this.accountService.createAccount( + newAccountObject, + userId + ); + + // Store the new to old account ID mappings for updating activities + if (accountWithSameId && oldAccountId) { + accountIdMapping[oldAccountId] = newAccount.id; + } + } + } + } + for (const activity of activitiesDto) { if (!activity.dataSource) { if (activity.type === 'ITEM') { @@ -120,6 +178,13 @@ export class ImportService { activity.dataSource = this.dataProviderService.getPrimaryDataSource(); } } + + // If a new account is created, then update the accountId in all activities + if (!isDryRun) { + if (Object.keys(accountIdMapping).includes(activity.accountId)) { + activity.accountId = accountIdMapping[activity.accountId]; + } + } } const assetProfiles = await this.validateActivities({ @@ -128,12 +193,18 @@ export class ImportService { userId }); - const accountIds = (await this.accountService.getAccounts(userId)).map( + const accounts = (await this.accountService.getAccounts(userId)).map( (account) => { - return account.id; + return { id: account.id, name: account.name }; } ); + if (isDryRun) { + accountsDto.forEach(({ id, name }) => { + accounts.push({ id, name }); + }); + } + const activities: Activity[] = []; for (const { @@ -149,11 +220,15 @@ export class ImportService { unitPrice } of activitiesDto) { const date = parseISO((dateString)); - const validatedAccountId = accountIds.includes(accountId) - ? accountId - : undefined; + const validatedAccount = accounts.find(({ id }) => { + return id === accountId; + }); - let order: OrderWithAccount; + let order: + | OrderWithAccount + | (Omit & { + Account?: { id: string; name: string }; + }); if (isDryRun) { order = { @@ -164,7 +239,7 @@ export class ImportService { type, unitPrice, userId, - accountId: validatedAccountId, + accountId: validatedAccount?.id, accountUserId: undefined, createdAt: new Date(), id: uuidv4(), @@ -187,6 +262,7 @@ export class ImportService { url: null, ...assetProfiles[symbol] }, + Account: validatedAccount, symbolProfileId: undefined, updatedAt: new Date() }; @@ -199,7 +275,7 @@ export class ImportService { type, unitPrice, userId, - accountId: validatedAccountId, + accountId: validatedAccount?.id, SymbolProfile: { connectOrCreate: { create: { @@ -221,6 +297,7 @@ export class ImportService { const value = new Big(quantity).mul(unitPrice).toNumber(); + //@ts-ignore activities.push({ ...order, value, diff --git a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts index b7c143c37..7bfa9aa37 100644 --- a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts +++ b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts @@ -11,6 +11,7 @@ import { MatLegacyDialogRef as MatDialogRef } from '@angular/material/legacy-dialog'; import { MatLegacySnackBar as MatSnackBar } from '@angular/material/legacy-snack-bar'; +import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto'; import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; import { DataService } from '@ghostfolio/client/services/data.service'; import { ImportActivitiesService } from '@ghostfolio/client/services/import-activities.service'; @@ -28,6 +29,7 @@ import { ImportActivitiesDialogParams } from './interfaces/interfaces'; templateUrl: 'import-activities-dialog.html' }) export class ImportActivitiesDialog implements OnDestroy { + public accounts: CreateAccountDto[] = []; public activities: Activity[] = []; public details: any[] = []; public errorMessages: string[] = []; @@ -91,9 +93,10 @@ export class ImportActivitiesDialog implements OnDestroy { try { this.snackBar.open('⏳ ' + $localize`Importing data...`); - await this.importActivitiesService.importSelectedActivities( - this.selectedActivities - ); + await this.importActivitiesService.importSelectedActivities({ + accounts: this.accounts, + activities: this.selectedActivities + }); this.snackBar.open( '✅ ' + $localize`Import has been completed`, @@ -163,6 +166,8 @@ export class ImportActivitiesDialog implements OnDestroy { if (file.name.endsWith('.json')) { const content = JSON.parse(fileContent); + this.accounts = content.accounts; + if (!isArray(content.activities)) { if (isArray(content.orders)) { this.handleImportError({ @@ -180,10 +185,13 @@ export class ImportActivitiesDialog implements OnDestroy { } try { - this.activities = await this.importActivitiesService.importJson({ - content: content.activities, - isDryRun: true - }); + const { activities } = + await this.importActivitiesService.importJson({ + accounts: content.accounts, + activities: content.activities, + isDryRun: true + }); + this.activities = activities; } catch (error) { console.error(error); this.handleImportError({ error, activities: content.activities }); @@ -192,11 +200,12 @@ export class ImportActivitiesDialog implements OnDestroy { return; } else if (file.name.endsWith('.csv')) { try { - this.activities = await this.importActivitiesService.importCsv({ + const data = await this.importActivitiesService.importCsv({ fileContent, isDryRun: true, userAccounts: this.data.user.accounts }); + this.activities = data.activities; } catch (error) { console.error(error); this.handleImportError({ diff --git a/apps/client/src/app/services/import-activities.service.ts b/apps/client/src/app/services/import-activities.service.ts index 2e15f367f..3b76b44b1 100644 --- a/apps/client/src/app/services/import-activities.service.ts +++ b/apps/client/src/app/services/import-activities.service.ts @@ -1,5 +1,6 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; +import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto'; import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto'; import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface'; import { Account, DataSource, Type } from '@prisma/client'; @@ -33,7 +34,9 @@ export class ImportActivitiesService { fileContent: string; isDryRun?: boolean; userAccounts: Account[]; - }): Promise { + }): Promise<{ + activities: Activity[]; + }> { const content = csvToJson(fileContent, { dynamicTyping: true, header: true, @@ -55,20 +58,26 @@ export class ImportActivitiesService { }); } - return await this.importJson({ isDryRun, content: activities }); + return await this.importJson({ activities, isDryRun }); } public importJson({ - content, + accounts, + activities, isDryRun = false }: { - content: CreateOrderDto[]; + activities: CreateOrderDto[]; + accounts?: CreateAccountDto[]; isDryRun?: boolean; - }): Promise { + }): Promise<{ + activities: Activity[]; + accounts?: CreateAccountDto[]; + }> { return new Promise((resolve, reject) => { this.postImport( { - activities: content + accounts, + activities }, isDryRun ) @@ -80,22 +89,29 @@ export class ImportActivitiesService { ) .subscribe({ next: (data) => { - resolve(data.activities); + resolve(data); } }); }); } - public importSelectedActivities( - selectedActivities: Activity[] - ): Promise { + public importSelectedActivities({ + accounts, + activities + }: { + accounts: CreateAccountDto[]; + activities: Activity[]; + }): Promise<{ + activities: Activity[]; + accounts?: CreateAccountDto[]; + }> { const importData: CreateOrderDto[] = []; - for (const activity of selectedActivities) { + for (const activity of activities) { importData.push(this.convertToCreateOrderDto(activity)); } - return this.importJson({ content: importData }); + return this.importJson({ accounts, activities: importData }); } private convertToCreateOrderDto({ @@ -347,7 +363,7 @@ export class ImportActivitiesService { } private postImport( - aImportData: { activities: CreateOrderDto[] }, + aImportData: { accounts: CreateAccountDto[]; activities: CreateOrderDto[] }, aIsDryRun = false ) { return this.http.post<{ activities: Activity[] }>( diff --git a/libs/common/src/lib/interfaces/export.interface.ts b/libs/common/src/lib/interfaces/export.interface.ts index 37dbfba79..b142cb2f8 100644 --- a/libs/common/src/lib/interfaces/export.interface.ts +++ b/libs/common/src/lib/interfaces/export.interface.ts @@ -1,10 +1,11 @@ -import { Order } from '@prisma/client'; +import { Account, Order } from '@prisma/client'; export interface Export { meta: { date: string; version: string; }; + accounts: Omit[]; activities: (Omit< Order, | 'accountUserId' diff --git a/test/import/ok-without-accounts.json b/test/import/ok-without-accounts.json new file mode 100644 index 000000000..63961be74 --- /dev/null +++ b/test/import/ok-without-accounts.json @@ -0,0 +1,48 @@ +{ + "meta": { + "date": "2022-04-01T00:00:00.000Z", + "version": "dev" + }, + "activities": [ + { + "fee": 0, + "quantity": 0, + "type": "BUY", + "unitPrice": 0, + "currency": "USD", + "dataSource": "YAHOO", + "date": "2050-06-05T22:00:00.000Z", + "symbol": "MSFT" + }, + { + "fee": 0, + "quantity": 1, + "type": "ITEM", + "unitPrice": 500000, + "currency": "USD", + "dataSource": "MANUAL", + "date": "2021-12-31T22:00:00.000Z", + "symbol": "Penthouse Apartment" + }, + { + "fee": 0, + "quantity": 5, + "type": "DIVIDEND", + "unitPrice": 0.62, + "currency": "USD", + "dataSource": "YAHOO", + "date": "2021-11-16T22:00:00.000Z", + "symbol": "MSFT" + }, + { + "fee": 19, + "quantity": 5, + "type": "BUY", + "unitPrice": 298.58, + "currency": "USD", + "dataSource": "YAHOO", + "date": "2021-09-15T22:00:00.000Z", + "symbol": "MSFT" + } + ] +} diff --git a/test/import/ok.json b/test/import/ok.json index 63961be74..335d2cd8a 100644 --- a/test/import/ok.json +++ b/test/import/ok.json @@ -1,10 +1,23 @@ { "meta": { - "date": "2022-04-01T00:00:00.000Z", + "date": "2023-02-05T00:00:00.000Z", "version": "dev" }, + "accounts": [ + { + "accountType": "SECURITIES", + "balance": 2000, + "currency": "USD", + "id": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0", + "isExcluded": false, + "name": "My Online Trading Account", + "platformId": null + } + ], "activities": [ { + "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0", + "comment": null, "fee": 0, "quantity": 0, "type": "BUY", @@ -15,6 +28,8 @@ "symbol": "MSFT" }, { + "accountId": null, + "comment": null, "fee": 0, "quantity": 1, "type": "ITEM", @@ -25,6 +40,8 @@ "symbol": "Penthouse Apartment" }, { + "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0", + "comment": null, "fee": 0, "quantity": 5, "type": "DIVIDEND", @@ -35,6 +52,8 @@ "symbol": "MSFT" }, { + "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0", + "comment": "My first order", "fee": 19, "quantity": 5, "type": "BUY",