From 7d6a74a67df1c2080a5fb93867605dbe679e27c7 Mon Sep 17 00:00:00 2001 From: Visrut Date: Tue, 9 May 2023 01:15:59 +0530 Subject: [PATCH] Feature/improve check for duplicates in import #1932 (#1940) * Improve check for duplicates in import * Update changelog --- CHANGELOG.md | 6 + apps/api/src/app/import/import.service.ts | 169 ++++++++++++------ .../order/interfaces/activities.interface.ts | 1 + apps/api/src/app/order/order.service.ts | 1 + .../activities-table.component.html | 7 +- .../activities-table.component.ts | 4 +- 6 files changed, 129 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9e05dfcb..035366a4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Improved the preview step of the activities import by unchecking duplicates + ## 1.267.0 - 2023-05-07 ### Added diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index faf71784c..14bdd4c50 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -84,6 +84,7 @@ export class ImportService { feeInBaseCurrency: 0, id: assetProfile.id, isDraft: false, + isDuplicate: false, // TODO: Use evaluated state SymbolProfile: (assetProfile), symbolProfileId: assetProfile.id, type: 'DIVIDEND', @@ -204,9 +205,14 @@ export class ImportService { userId }); + const activitiesMarkedAsDuplicates = await this.markActivitiesAsDuplicates({ + activitiesDto, + userId + }); + const accounts = (await this.accountService.getAccounts(userId)).map( - (account) => { - return { id: account.id, name: account.name }; + ({ id, name }) => { + return { id, name }; } ); @@ -221,16 +227,14 @@ export class ImportService { for (const { accountId, comment, - currency, - dataSource, - date: dateString, + date, fee, + isDuplicate, quantity, - symbol, + SymbolProfile: assetProfile, type, unitPrice - } of activitiesDto) { - const date = parseISO((dateString)); + } of activitiesMarkedAsDuplicates) { const validatedAccount = accounts.find(({ id }) => { return id === accountId; }); @@ -256,29 +260,33 @@ export class ImportService { id: uuidv4(), isDraft: isAfter(date, endOfToday()), SymbolProfile: { - currency, - dataSource, - symbol, - assetClass: null, - assetSubClass: null, - comment: null, - countries: null, - createdAt: undefined, - id: undefined, - isin: null, - name: null, - scraperConfiguration: null, - sectors: null, - symbolMapping: null, - updatedAt: undefined, - url: null, - ...assetProfiles[symbol] + assetClass: assetProfile.assetClass, + assetSubClass: assetProfile.assetSubClass, + comment: assetProfile.comment, + countries: assetProfile.countries, + createdAt: assetProfile.createdAt, + currency: assetProfile.currency, + dataSource: assetProfile.dataSource, + id: assetProfile.id, + isin: assetProfile.isin, + name: assetProfile.name, + scraperConfiguration: assetProfile.scraperConfiguration, + sectors: assetProfile.sectors, + symbol: assetProfile.currency, + symbolMapping: assetProfile.symbolMapping, + updatedAt: assetProfile.updatedAt, + url: assetProfile.url, + ...assetProfiles[assetProfile.symbol] }, Account: validatedAccount, symbolProfileId: undefined, updatedAt: new Date() }; } else { + if (isDuplicate) { + continue; + } + order = await this.orderService.createOrder({ comment, date, @@ -291,14 +299,14 @@ export class ImportService { SymbolProfile: { connectOrCreate: { create: { - currency, - dataSource, - symbol + currency: assetProfile.currency, + dataSource: assetProfile.dataSource, + symbol: assetProfile.symbol }, where: { dataSource_symbol: { - dataSource, - symbol + dataSource: assetProfile.dataSource, + symbol: assetProfile.symbol } } } @@ -313,15 +321,16 @@ export class ImportService { //@ts-ignore activities.push({ ...order, + isDuplicate, value, feeInBaseCurrency: this.exchangeRateDataService.toCurrency( fee, - currency, + assetProfile.currency, userCurrency ), valueInBaseCurrency: this.exchangeRateDataService.toCurrency( value, - currency, + assetProfile.currency, userCurrency ) }); @@ -340,6 +349,78 @@ export class ImportService { return uniqueAccountIds.size === 1; } + private async markActivitiesAsDuplicates({ + activitiesDto, + userId + }: { + activitiesDto: Partial[]; + userId: string; + }): Promise[]> { + const existingActivities = await this.orderService.orders({ + include: { SymbolProfile: true }, + orderBy: { date: 'desc' }, + where: { userId } + }); + + return activitiesDto.map( + ({ + accountId, + comment, + currency, + dataSource, + date: dateString, + fee, + quantity, + symbol, + type, + unitPrice + }) => { + const date = parseISO((dateString)); + const isDuplicate = existingActivities.some((activity) => { + return ( + activity.SymbolProfile.currency === currency && + activity.SymbolProfile.dataSource === dataSource && + isSameDay(activity.date, date) && + activity.fee === fee && + activity.quantity === quantity && + activity.SymbolProfile.symbol === symbol && + activity.type === type && + activity.unitPrice === unitPrice + ); + }); + + return { + accountId, + comment, + date, + fee, + isDuplicate, + quantity, + type, + unitPrice, + SymbolProfile: { + currency, + dataSource, + symbol, + assetClass: null, + assetSubClass: null, + comment: null, + countries: null, + createdAt: undefined, + id: undefined, + isin: null, + name: null, + scraperConfiguration: null, + sectors: null, + symbolMapping: null, + updatedAt: undefined, + url: null + } + }; + } + ); + } + private async validateActivities({ activitiesDto, maxActivitiesToImport, @@ -356,33 +437,11 @@ export class ImportService { const assetProfiles: { [symbol: string]: Partial; } = {}; - const existingActivities = await this.orderService.orders({ - include: { SymbolProfile: true }, - orderBy: { date: 'desc' }, - where: { userId } - }); for (const [ index, - { currency, dataSource, date, fee, quantity, symbol, type, unitPrice } + { currency, dataSource, symbol } ] of activitiesDto.entries()) { - const duplicateActivity = existingActivities.find((activity) => { - return ( - activity.SymbolProfile.currency === currency && - activity.SymbolProfile.dataSource === dataSource && - isSameDay(activity.date, parseISO((date))) && - activity.fee === fee && - activity.quantity === quantity && - activity.SymbolProfile.symbol === symbol && - activity.type === type && - activity.unitPrice === unitPrice - ); - }); - - if (duplicateActivity) { - throw new Error(`activities.${index} is a duplicate activity`); - } - if (dataSource !== 'MANUAL') { const assetProfile = ( await this.dataProviderService.getAssetProfiles([ diff --git a/apps/api/src/app/order/interfaces/activities.interface.ts b/apps/api/src/app/order/interfaces/activities.interface.ts index 33fe8b40d..f10417f07 100644 --- a/apps/api/src/app/order/interfaces/activities.interface.ts +++ b/apps/api/src/app/order/interfaces/activities.interface.ts @@ -6,6 +6,7 @@ export interface Activities { export interface Activity extends OrderWithAccount { feeInBaseCurrency: number; + isDuplicate: boolean; updateAccountBalance?: boolean; value: number; valueInBaseCurrency: number; diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 696f5442e..b3f27c221 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -333,6 +333,7 @@ export class OrderService { order.SymbolProfile.currency, userCurrency ), + isDuplicate: false, valueInBaseCurrency: this.exchangeRateDataService.toCurrency( value, order.SymbolProfile.currency, diff --git a/libs/ui/src/lib/activities-table/activities-table.component.html b/libs/ui/src/lib/activities-table/activities-table.component.html index 4ac853436..45551305b 100644 --- a/libs/ui/src/lib/activities-table/activities-table.component.html +++ b/libs/ui/src/lib/activities-table/activities-table.component.html @@ -76,7 +76,6 @@ diff --git a/libs/ui/src/lib/activities-table/activities-table.component.ts b/libs/ui/src/lib/activities-table/activities-table.component.ts index 1c3fb0367..0ba6e4f58 100644 --- a/libs/ui/src/lib/activities-table/activities-table.component.ts +++ b/libs/ui/src/lib/activities-table/activities-table.component.ts @@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit { public onClickActivity(activity: Activity) { if (this.showCheckbox) { - this.selectedRows.toggle(activity); + if (!activity.isDuplicate) { + this.selectedRows.toggle(activity); + } } else if ( this.hasPermissionToOpenDetails && !activity.isDraft &&