diff --git a/CHANGELOG.md b/CHANGELOG.md index 80987a811..33fa8fd89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Optimized the validation in the activities import by reducing the list to unique asset profiles +- Optimized the data gathering in the activities import + ## 1.295.0 - 2023-07-30 ### Added diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 54d36d8c5..8e3fb2871 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -7,6 +7,7 @@ import { GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { AdminData, AdminMarketData, @@ -116,7 +117,7 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: `${dataSource}-${symbol}` + jobId: getAssetProfileIdentifier({ dataSource, symbol }) } }; }) @@ -152,7 +153,7 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: `${dataSource}-${symbol}` + jobId: getAssetProfileIdentifier({ dataSource, symbol }) } }; }) @@ -185,7 +186,7 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: `${dataSource}-${symbol}` + jobId: getAssetProfileIdentifier({ dataSource, symbol }) } }); } diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 65611ce0d..3d9c2999d 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -8,10 +8,14 @@ import { import { OrderService } from '@ghostfolio/api/app/order/order.service'; import { PlatformService } from '@ghostfolio/api/app/platform/platform.service'; import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service'; +import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; -import { parseDate } from '@ghostfolio/common/helper'; +import { + getAssetProfileIdentifier, + parseDate +} from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { AccountWithPlatform, @@ -21,12 +25,14 @@ import { Injectable } from '@nestjs/common'; import { DataSource, Prisma, SymbolProfile } from '@prisma/client'; import Big from 'big.js'; import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns'; +import { uniqBy } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; @Injectable() export class ImportService { public constructor( private readonly accountService: AccountService, + private readonly dataGatheringService: DataGatheringService, private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly orderService: OrderService, @@ -220,8 +226,7 @@ export class ImportService { const assetProfiles = await this.validateActivities({ activitiesDto, - maxActivitiesToImport, - userId + maxActivitiesToImport }); const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({ @@ -250,10 +255,37 @@ export class ImportService { error, fee, quantity, - SymbolProfile: assetProfile, + SymbolProfile, type, unitPrice } of activitiesExtendedWithErrors) { + const assetProfile = assetProfiles[ + getAssetProfileIdentifier({ + dataSource: SymbolProfile.dataSource, + symbol: SymbolProfile.symbol + }) + ] ?? { + currency: SymbolProfile.currency, + dataSource: SymbolProfile.dataSource, + symbol: SymbolProfile.symbol + }; + const { + assetClass, + assetSubClass, + countries, + createdAt, + currency, + dataSource, + id, + isin, + name, + scraperConfiguration, + sectors, + symbol, + symbolMapping, + url, + updatedAt + } = assetProfile; const validatedAccount = accounts.find(({ id }) => { return id === accountId; }); @@ -279,23 +311,22 @@ export class ImportService { id: uuidv4(), isDraft: isAfter(date, endOfToday()), SymbolProfile: { - assetClass: assetProfile.assetClass, - assetSubClass: assetProfile.assetSubClass, - comment: assetProfile.comment, - countries: assetProfile.countries, - createdAt: assetProfile.createdAt, - currency: assetProfile.currency, - dataSource: assetProfile.dataSource, - id: assetProfile.id, - isin: assetProfile.isin, - name: assetProfile.name, - scraperConfiguration: assetProfile.scraperConfiguration, - sectors: assetProfile.sectors, - symbol: assetProfile.currency, - symbolMapping: assetProfile.symbolMapping, - updatedAt: assetProfile.updatedAt, - url: assetProfile.url, - ...assetProfiles[assetProfile.symbol] + assetClass, + assetSubClass, + countries, + createdAt, + currency, + dataSource, + id, + isin, + name, + scraperConfiguration, + sectors, + symbol, + symbolMapping, + updatedAt, + url, + comment: assetProfile.comment }, Account: validatedAccount, symbolProfileId: undefined, @@ -318,14 +349,14 @@ export class ImportService { SymbolProfile: { connectOrCreate: { create: { - currency: assetProfile.currency, - dataSource: assetProfile.dataSource, - symbol: assetProfile.symbol + currency, + dataSource, + symbol }, where: { dataSource_symbol: { - dataSource: assetProfile.dataSource, - symbol: assetProfile.symbol + dataSource, + symbol } } } @@ -337,24 +368,49 @@ export class ImportService { const value = new Big(quantity).mul(unitPrice).toNumber(); - //@ts-ignore activities.push({ ...order, error, value, feeInBaseCurrency: this.exchangeRateDataService.toCurrency( fee, - assetProfile.currency, + currency, userCurrency ), + //@ts-ignore + SymbolProfile: assetProfile, valueInBaseCurrency: this.exchangeRateDataService.toCurrency( value, - assetProfile.currency, + currency, userCurrency ) }); } + activities.sort((activity1, activity2) => { + return Number(activity1.date) - Number(activity2.date); + }); + + if (!isDryRun) { + // Gather symbol data in the background, if not dry run + const uniqueActivities = uniqBy(activities, ({ SymbolProfile }) => { + return getAssetProfileIdentifier({ + dataSource: SymbolProfile.dataSource, + symbol: SymbolProfile.symbol + }); + }); + + this.dataGatheringService.gatherSymbols( + uniqueActivities.map(({ date, SymbolProfile }) => { + return { + date, + dataSource: SymbolProfile.dataSource, + symbol: SymbolProfile.symbol + }; + }) + ); + } + return activities; } @@ -446,25 +502,30 @@ export class ImportService { private async validateActivities({ activitiesDto, - maxActivitiesToImport, - userId + maxActivitiesToImport }: { activitiesDto: Partial[]; maxActivitiesToImport: number; - userId: string; }) { if (activitiesDto?.length > maxActivitiesToImport) { throw new Error(`Too many activities (${maxActivitiesToImport} at most)`); } const assetProfiles: { - [symbol: string]: Partial; + [assetProfileIdentifier: string]: Partial; } = {}; + const uniqueActivitiesDto = uniqBy( + activitiesDto, + ({ dataSource, symbol }) => { + return getAssetProfileIdentifier({ dataSource, symbol }); + } + ); + for (const [ index, { currency, dataSource, symbol } - ] of activitiesDto.entries()) { + ] of uniqueActivitiesDto.entries()) { if (dataSource !== 'MANUAL') { const assetProfile = ( await this.dataProviderService.getAssetProfiles([ @@ -484,7 +545,8 @@ export class ImportService { ); } - assetProfiles[symbol] = assetProfile; + assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] = + assetProfile; } } diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts index c478860d2..0e617462e 100644 --- a/apps/api/src/app/order/order.controller.ts +++ b/apps/api/src/app/order/order.controller.ts @@ -2,6 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor'; import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; +import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service'; import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; @@ -36,6 +37,7 @@ import { UpdateOrderDto } from './update-order.dto'; export class OrderController { public constructor( private readonly apiService: ApiService, + private readonly dataGatheringService: DataGatheringService, private readonly impersonationService: ImpersonationService, private readonly orderService: OrderService, @Inject(REQUEST) private readonly request: RequestWithUser @@ -123,7 +125,7 @@ export class OrderController { ); } - return this.orderService.createOrder({ + const order = await this.orderService.createOrder({ ...data, date: parseISO(data.date), SymbolProfile: { @@ -144,6 +146,19 @@ export class OrderController { User: { connect: { id: this.request.user.id } }, userId: this.request.user.id }); + + if (!order.isDraft) { + // Gather symbol data in the background, if not draft + this.dataGatheringService.gatherSymbols([ + { + dataSource: data.dataSource, + date: order.date, + symbol: data.symbol + } + ]); + } + + return order; } @Put(':id') diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index be5708a90..0f7da5da3 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -7,6 +7,7 @@ import { GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { Filter } from '@ghostfolio/common/interfaces'; import { OrderWithAccount } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; @@ -117,7 +118,7 @@ export class OrderService { }; } - await this.dataGatheringService.addJobToQueue({ + this.dataGatheringService.addJobToQueue({ data: { dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol @@ -125,25 +126,12 @@ export class OrderService { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}` - } - }); - - const isDraft = - data.type === 'LIABILITY' - ? false - : isAfter(data.date as Date, endOfToday()); - - if (!isDraft) { - // Gather symbol data of order in the background, if not draft - this.dataGatheringService.gatherSymbols([ - { + jobId: getAssetProfileIdentifier({ dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, - date: data.date, symbol: data.SymbolProfile.connectOrCreate.create.symbol - } - ]); - } + }) + } + }); delete data.accountId; delete data.assetClass; @@ -162,6 +150,11 @@ export class OrderService { const orderData: Prisma.OrderCreateInput = data; + const isDraft = + data.type === 'LIABILITY' + ? false + : isAfter(data.date as Date, endOfToday()); + const order = await this.prismaService.order.create({ data: { ...orderData, diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.ts index 865a23aea..aa0a2314e 100644 --- a/apps/api/src/app/redis-cache/redis-cache.service.ts +++ b/apps/api/src/app/redis-cache/redis-cache.service.ts @@ -1,4 +1,5 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common'; @@ -22,7 +23,7 @@ export class RedisCacheService { } public getQuoteKey({ dataSource, symbol }: UniqueAsset) { - return `quote-${dataSource}-${symbol}`; + return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`; } public async remove(key: string) { diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index 72043b36b..e3597f049 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -2,6 +2,7 @@ import { GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; @@ -48,7 +49,7 @@ export class CronService { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: `${dataSource}-${symbol}` + jobId: getAssetProfileIdentifier({ dataSource, symbol }) } }; }) diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index fc77bdc60..20da098ff 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -10,7 +10,11 @@ import { GATHER_HISTORICAL_MARKET_DATA_PROCESS, GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS } from '@ghostfolio/common/config'; -import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; +import { + DATE_FORMAT, + getAssetProfileIdentifier, + resetHours +} from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { InjectQueue } from '@nestjs/bull'; import { Inject, Injectable, Logger } from '@nestjs/common'; @@ -221,7 +225,10 @@ export class DataGatheringService { name: GATHER_HISTORICAL_MARKET_DATA_PROCESS, opts: { ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, - jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}` + jobId: `${getAssetProfileIdentifier({ + dataSource, + symbol + })}-${format(date, DATE_FORMAT)}` } }; }) diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts index bc02acbe6..02b0cc08d 100644 --- a/libs/common/src/lib/helper.ts +++ b/libs/common/src/lib/helper.ts @@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns'; import { de, es, fr, it, nl, pt } from 'date-fns/locale'; import { ghostfolioScraperApiSymbolPrefix, locale } from './config'; -import { Benchmark } from './interfaces'; +import { Benchmark, UniqueAsset } from './interfaces'; import { ColorScheme } from './types'; const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g; @@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number { } } +export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) { + return `${dataSource}-${symbol}`; +} + export function getBackgroundColor(aColorScheme: ColorScheme) { return getCssVariable( aColorScheme === 'DARK' ||