From 403ee2741d2436f9c496da5465163367ef9f4039 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Tue, 10 Sep 2024 20:32:08 +0200 Subject: [PATCH] Feature/set up portfolio snapshot queue (#3725) * Set up portfolio snapshot queue * Update changelog --- CHANGELOG.md | 1 + apps/api/src/app/admin/admin.controller.ts | 2 +- apps/api/src/app/admin/admin.module.ts | 2 +- apps/api/src/app/admin/queue/queue.module.ts | 5 +- apps/api/src/app/admin/queue/queue.service.ts | 36 +++++-- apps/api/src/app/app.module.ts | 4 +- apps/api/src/app/import/import.module.ts | 2 +- apps/api/src/app/import/import.service.ts | 2 +- apps/api/src/app/info/info.module.ts | 2 +- apps/api/src/app/order/order.controller.ts | 2 +- apps/api/src/app/order/order.module.ts | 2 +- apps/api/src/app/order/order.service.ts | 2 +- .../portfolio-calculator.factory.ts | 4 + .../calculator/portfolio-calculator.ts | 75 +++++++++------ ...aln-buy-and-sell-in-two-activities.spec.ts | 26 ++++- ...folio-calculator-baln-buy-and-sell.spec.ts | 26 ++++- .../twr/portfolio-calculator-baln-buy.spec.ts | 26 ++++- ...ator-btcusd-buy-and-sell-partially.spec.ts | 26 ++++- .../twr/portfolio-calculator-fee.spec.ts | 26 ++++- .../portfolio-calculator-googl-buy.spec.ts | 26 ++++- .../twr/portfolio-calculator-item.spec.ts | 26 ++++- .../portfolio-calculator-liability.spec.ts | 31 ++++-- ...-calculator-msft-buy-with-dividend.spec.ts | 26 ++++- .../portfolio-calculator-no-orders.spec.ts | 28 ++++-- ...ulator-novn-buy-and-sell-partially.spec.ts | 26 ++++- ...folio-calculator-novn-buy-and-sell.spec.ts | 26 ++++- .../twr/portfolio-calculator.spec.ts | 9 +- .../api/src/app/portfolio/portfolio.module.ts | 4 +- .../redis-cache/redis-cache.service.mock.ts | 21 +++- apps/api/src/services/cron.service.ts | 2 +- .../data-gathering/data-gathering.module.ts | 2 +- .../data-gathering.processor.ts | 13 ++- .../data-gathering/data-gathering.service.ts | 8 +- .../portfolio-snapshot-queue-job.interface.ts | 7 ++ .../portfolio-snapshot.module.ts | 37 +++++++ .../portfolio-snapshot.processor.ts | 96 +++++++++++++++++++ .../portfolio-snapshot.service.mock.ts | 34 +++++++ .../portfolio-snapshot.service.ts | 31 ++++++ .../app/components/admin-jobs/admin-jobs.html | 2 + libs/common/src/lib/config.ts | 14 ++- 40 files changed, 621 insertions(+), 119 deletions(-) rename apps/api/src/services/{ => queues}/data-gathering/data-gathering.module.ts (93%) rename apps/api/src/services/{ => queues}/data-gathering/data-gathering.processor.ts (95%) rename apps/api/src/services/{ => queues}/data-gathering/data-gathering.service.ts (98%) create mode 100644 apps/api/src/services/queues/portfolio-snapshot/interfaces/portfolio-snapshot-queue-job.interface.ts create mode 100644 apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts create mode 100644 apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts create mode 100644 apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock.ts create mode 100644 apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index db47c8b61..d1d6d787c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Migrated the portfolio snapshot calculation to the queue design pattern - Optimized the asynchronous operations using `Promise.all()` in the info service - Optimized the asynchronous operations using `Promise.all()` in the admin control panel endpoint - Extracted the users from the admin control panel endpoint to a dedicated endpoint diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index da4b5dd7e..e0444d112 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -2,10 +2,10 @@ import { HasPermission } from '@ghostfolio/api/decorators/has-permission.decorat import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'; import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PropertyDto } from '@ghostfolio/api/services/property/property.dto'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.ts index f8cf8e8ac..81c58ff03 100644 --- a/apps/api/src/app/admin/admin.module.ts +++ b/apps/api/src/app/admin/admin.module.ts @@ -4,12 +4,12 @@ import { SubscriptionModule } from '@ghostfolio/api/app/subscription/subscriptio import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; import { ApiModule } from '@ghostfolio/api/services/api/api.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; diff --git a/apps/api/src/app/admin/queue/queue.module.ts b/apps/api/src/app/admin/queue/queue.module.ts index 46ae3b8a5..22d1cefc6 100644 --- a/apps/api/src/app/admin/queue/queue.module.ts +++ b/apps/api/src/app/admin/queue/queue.module.ts @@ -1,4 +1,5 @@ -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; import { Module } from '@nestjs/common'; @@ -7,7 +8,7 @@ import { QueueService } from './queue.service'; @Module({ controllers: [QueueController], - imports: [DataGatheringModule], + imports: [DataGatheringModule, PortfolioSnapshotQueueModule], providers: [QueueService] }) export class QueueModule {} diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index abae3cad1..7e4f0adb7 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -1,5 +1,6 @@ import { DATA_GATHERING_QUEUE, + PORTFOLIO_SNAPSHOT_QUEUE, QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config'; import { AdminJobs } from '@ghostfolio/common/interfaces'; @@ -12,11 +13,19 @@ import { JobStatus, Queue } from 'bull'; export class QueueService { public constructor( @InjectQueue(DATA_GATHERING_QUEUE) - private readonly dataGatheringQueue: Queue + private readonly dataGatheringQueue: Queue, + @InjectQueue(PORTFOLIO_SNAPSHOT_QUEUE) + private readonly portfolioSnapshotQueue: Queue ) {} public async deleteJob(aId: string) { - return (await this.dataGatheringQueue.getJob(aId))?.remove(); + let job = await this.dataGatheringQueue.getJob(aId); + + if (!job) { + job = await this.portfolioSnapshotQueue.getJob(aId); + } + + return job?.remove(); } public async deleteJobs({ @@ -25,15 +34,21 @@ export class QueueService { status?: JobStatus[]; }) { for (const statusItem of status) { - await this.dataGatheringQueue.clean( - 300, - statusItem === 'waiting' ? 'wait' : statusItem - ); + const queueStatus = statusItem === 'waiting' ? 'wait' : statusItem; + + await this.dataGatheringQueue.clean(300, queueStatus); + await this.portfolioSnapshotQueue.clean(300, queueStatus); } } public async executeJob(aId: string) { - return (await this.dataGatheringQueue.getJob(aId))?.promote(); + let job = await this.dataGatheringQueue.getJob(aId); + + if (!job) { + job = await this.portfolioSnapshotQueue.getJob(aId); + } + + return job?.promote(); } public async getJobs({ @@ -43,10 +58,13 @@ export class QueueService { limit?: number; status?: JobStatus[]; }): Promise { - const jobs = await this.dataGatheringQueue.getJobs(status); + const [dataGatheringJobs, portfolioSnapshotJobs] = await Promise.all([ + this.dataGatheringQueue.getJobs(status), + this.portfolioSnapshotQueue.getJobs(status) + ]); const jobsWithState = await Promise.all( - jobs + [...dataGatheringJobs, ...portfolioSnapshotJobs] .filter((job) => { return job; }) diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.ts index ca19d63bc..86d97eaf8 100644 --- a/apps/api/src/app/app.module.ts +++ b/apps/api/src/app/app.module.ts @@ -1,11 +1,12 @@ import { EventsModule } from '@ghostfolio/api/events/events.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; import { CronService } from '@ghostfolio/api/services/cron.service'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module'; import { DEFAULT_LANGUAGE_CODE, @@ -81,6 +82,7 @@ import { UserModule } from './user/user.module'; OrderModule, PlatformModule, PortfolioModule, + PortfolioSnapshotQueueModule, PrismaModule, PropertyModule, RedisCacheModule, diff --git a/apps/api/src/app/import/import.module.ts b/apps/api/src/app/import/import.module.ts index 47a4b5db3..142a939a6 100644 --- a/apps/api/src/app/import/import.module.ts +++ b/apps/api/src/app/import/import.module.ts @@ -7,10 +7,10 @@ import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.mo import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 30ab87069..c42cd300b 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -9,9 +9,9 @@ import { OrderService } from '@ghostfolio/api/app/order/order.service'; import { PlatformService } from '@ghostfolio/api/app/platform/platform.service'; import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH } from '@ghostfolio/common/config'; import { diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.ts index 473a966ad..9b7854160 100644 --- a/apps/api/src/app/info/info.module.ts +++ b/apps/api/src/app/info/info.module.ts @@ -4,10 +4,10 @@ import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.mo import { UserModule } from '@ghostfolio/api/app/user/user.module'; import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { TagModule } from '@ghostfolio/api/services/tag/tag.module'; diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts index f2acac5e4..907335aa0 100644 --- a/apps/api/src/app/order/order.controller.ts +++ b/apps/api/src/app/order/order.controller.ts @@ -4,8 +4,8 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor'; import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { getIntervalFromDateRange } from '@ghostfolio/common/calculation-helper'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, diff --git a/apps/api/src/app/order/order.module.ts b/apps/api/src/app/order/order.module.ts index 55b4cce82..9bc837aa6 100644 --- a/apps/api/src/app/order/order.module.ts +++ b/apps/api/src/app/order/order.module.ts @@ -6,11 +6,11 @@ import { RedactValuesInResponseModule } from '@ghostfolio/api/interceptors/redac import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors/transform-data-source-in-request/transform-data-source-in-request.module'; import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { ApiModule } from '@ghostfolio/api/services/api/api.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index dcbc20e2b..8e69aae6f 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -1,9 +1,9 @@ import { AccountService } from '@ghostfolio/api/app/account/account.service'; import { PortfolioChangedEvent } from '@ghostfolio/api/events/portfolio-changed.event'; import { LogPerformance } from '@ghostfolio/api/interceptors/performance-logging/performance-logging.interceptor'; -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts index b531ffc9d..18738373e 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.factory.ts @@ -3,6 +3,7 @@ import { CurrentRateService } from '@ghostfolio/api/app/portfolio/current-rate.s import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; import { Filter, HistoricalDataItem } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; @@ -22,6 +23,7 @@ export class PortfolioCalculatorFactory { private readonly configurationService: ConfigurationService, private readonly currentRateService: CurrentRateService, private readonly exchangeRateDataService: ExchangeRateDataService, + private readonly portfolioSnapshotService: PortfolioSnapshotService, private readonly redisCacheService: RedisCacheService ) {} @@ -51,6 +53,7 @@ export class PortfolioCalculatorFactory { configurationService: this.configurationService, currentRateService: this.currentRateService, exchangeRateDataService: this.exchangeRateDataService, + portfolioSnapshotService: this.portfolioSnapshotService, redisCacheService: this.redisCacheService }); case PerformanceCalculationType.TWR: @@ -63,6 +66,7 @@ export class PortfolioCalculatorFactory { userId, configurationService: this.configurationService, exchangeRateDataService: this.exchangeRateDataService, + portfolioSnapshotService: this.portfolioSnapshotService, redisCacheService: this.redisCacheService }); default: diff --git a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts index 2938bd734..d2f68e628 100644 --- a/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/calculator/portfolio-calculator.ts @@ -10,8 +10,14 @@ import { LogPerformance } from '@ghostfolio/api/interceptors/performance-logging import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; import { getIntervalFromDateRange } from '@ghostfolio/common/calculation-helper'; -import { CACHE_TTL_INFINITE } from '@ghostfolio/common/config'; +import { + PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME, + PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS, + PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_HIGH, + PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_LOW +} from '@ghostfolio/common/config'; import { DATE_FORMAT, getSum, @@ -34,7 +40,6 @@ import { Logger } from '@nestjs/common'; import { Big } from 'big.js'; import { plainToClass } from 'class-transformer'; import { - addMilliseconds, differenceInDays, eachDayOfInterval, endOfDay, @@ -59,6 +64,7 @@ export abstract class PortfolioCalculator { private endDate: Date; private exchangeRateDataService: ExchangeRateDataService; private filters: Filter[]; + private portfolioSnapshotService: PortfolioSnapshotService; private redisCacheService: RedisCacheService; private snapshot: PortfolioSnapshot; private snapshotPromise: Promise; @@ -74,6 +80,7 @@ export abstract class PortfolioCalculator { currentRateService, exchangeRateDataService, filters, + portfolioSnapshotService, redisCacheService, userId }: { @@ -84,6 +91,7 @@ export abstract class PortfolioCalculator { currentRateService: CurrentRateService; exchangeRateDataService: ExchangeRateDataService; filters: Filter[]; + portfolioSnapshotService: PortfolioSnapshotService; redisCacheService: RedisCacheService; userId: string; }) { @@ -132,6 +140,7 @@ export abstract class PortfolioCalculator { return a.date?.localeCompare(b.date); }); + this.portfolioSnapshotService = portfolioSnapshotService; this.redisCacheService = redisCacheService; this.userId = userId; @@ -153,7 +162,7 @@ export abstract class PortfolioCalculator { ): PortfolioSnapshot; @LogPerformance - private async computeSnapshot(): Promise { + public async computeSnapshot(): Promise { const lastTransactionPoint = last(this.transactionPoints); const transactionPoints = this.transactionPoints?.filter(({ date }) => { @@ -866,29 +875,6 @@ export abstract class PortfolioCalculator { return chartDateMap; } - private async computeAndCacheSnapshot() { - const snapshot = await this.computeSnapshot(); - - const expiration = addMilliseconds( - new Date(), - this.configurationService.get('CACHE_QUOTES_TTL') - ); - - this.redisCacheService.set( - this.redisCacheService.getPortfolioSnapshotKey({ - filters: this.filters, - userId: this.userId - }), - JSON.stringify(({ - expiration: expiration.getTime(), - portfolioSnapshot: snapshot - })), - CACHE_TTL_INFINITE - ); - - return snapshot; - } - @LogPerformance private computeTransactionPoints() { this.transactionPoints = []; @@ -1034,6 +1020,7 @@ export abstract class PortfolioCalculator { let cachedPortfolioSnapshot: PortfolioSnapshot; let isCachedPortfolioSnapshotExpired = false; + const jobId = this.userId; try { const cachedPortfolioSnapshotValue = await this.redisCacheService.get( @@ -1069,11 +1056,43 @@ export abstract class PortfolioCalculator { if (isCachedPortfolioSnapshotExpired) { // Compute in the background - this.computeAndCacheSnapshot(); + this.portfolioSnapshotService.addJobToQueue({ + data: { + filters: this.filters, + userCurrency: this.currency, + userId: this.userId + }, + name: PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME, + opts: { + ...PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS, + jobId, + priority: PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_LOW + } + }); } } else { // Wait for computation - this.snapshot = await this.computeAndCacheSnapshot(); + await this.portfolioSnapshotService.addJobToQueue({ + data: { + filters: this.filters, + userCurrency: this.currency, + userId: this.userId + }, + name: PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME, + opts: { + ...PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS, + jobId, + priority: PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_HIGH + } + }); + + const job = await this.portfolioSnapshotService.getJob(jobId); + + if (job) { + await job.finished(); + } + + await this.initialize(); } } } diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell-in-two-activities.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell-in-two-activities.spec.ts index 1ac86faf0..37499f0e3 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell-in-two-activities.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell-in-two-activities.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -118,14 +136,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell.spec.ts index 54cea8dae..23c594e5b 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy-and-sell.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -103,14 +121,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy.spec.ts index e638073eb..90f6a59d1 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-baln-buy.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -88,14 +106,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-btcusd-buy-and-sell-partially.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-btcusd-buy-and-sell-partially.spec.ts index cc64a540b..e232b42c4 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-btcusd-buy-and-sell-partially.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-btcusd-buy-and-sell-partially.spec.ts @@ -15,6 +15,8 @@ import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cac import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { ExchangeRateDataServiceMock } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service.mock'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -29,6 +31,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -54,7 +68,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -69,12 +84,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -117,14 +135,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-fee.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-fee.spec.ts index 4f4c05b13..fe379a92a 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-fee.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-fee.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -88,14 +106,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'USD', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); expect(portfolioSnapshot).toMatchObject({ currentValueInBaseCurrency: new Big('0'), diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-googl-buy.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-googl-buy.spec.ts index c7cf7e2b3..60fe6dc6b 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-googl-buy.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-googl-buy.spec.ts @@ -15,6 +15,8 @@ import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cac import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { ExchangeRateDataServiceMock } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service.mock'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -29,6 +31,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -54,7 +68,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -69,12 +84,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -101,14 +119,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-item.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-item.spec.ts index a0e62af57..228568374 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-item.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-item.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -88,14 +106,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'USD', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); expect(portfolioSnapshot).toMatchObject({ currentValueInBaseCurrency: new Big('0'), diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-liability.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-liability.spec.ts index 0df8dee48..5fa90e94c 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-liability.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-liability.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -27,6 +29,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -40,7 +54,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -55,12 +70,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -87,17 +105,18 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'USD', userId: userDummyData.id }); - const liabilitiesInBaseCurrency = - await portfolioCalculator.getLiabilitiesInBaseCurrency(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); - expect(liabilitiesInBaseCurrency).toEqual(new Big(3000)); + expect(portfolioSnapshot.totalLiabilitiesWithCurrencyEffect).toEqual( + new Big(3000) + ); }); }); }); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts index 8a1c5a517..97b860400 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-msft-buy-with-dividend.spec.ts @@ -15,6 +15,8 @@ import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cac import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { ExchangeRateDataServiceMock } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service.mock'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -29,6 +31,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -54,7 +68,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -69,12 +84,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -116,14 +134,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'USD', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); expect(portfolioSnapshot).toMatchObject({ errors: [], diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-no-orders.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-no-orders.spec.ts index a25e31dd3..84898490f 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-no-orders.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-no-orders.spec.ts @@ -9,11 +9,11 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; -import { subDays } from 'date-fns'; -import { last } from 'lodash'; jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { return { @@ -24,6 +24,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -37,7 +49,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -52,12 +65,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -66,14 +82,14 @@ describe('PortfolioCalculator', () => { it('with no orders', async () => { jest.useFakeTimers().setSystemTime(parseDate('2021-12-18').getTime()); - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities: [], calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell-partially.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell-partially.spec.ts index 967f8cd1f..30eb79754 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell-partially.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell-partially.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -103,14 +121,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts index 068246eb6..db5aaf6bc 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator-novn-buy-and-sell.spec.ts @@ -14,6 +14,8 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s import { RedisCacheServiceMock } from '@ghostfolio/api/app/redis-cache/redis-cache.service.mock'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PortfolioSnapshotServiceMock } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock'; import { parseDate } from '@ghostfolio/common/helper'; import { Big } from 'big.js'; @@ -28,6 +30,18 @@ jest.mock('@ghostfolio/api/app/portfolio/current-rate.service', () => { }; }); +jest.mock( + '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service', + () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + PortfolioSnapshotService: jest.fn().mockImplementation(() => { + return PortfolioSnapshotServiceMock; + }) + }; + } +); + jest.mock('@ghostfolio/api/app/redis-cache/redis-cache.service', () => { return { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -41,7 +55,8 @@ describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -56,12 +71,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); @@ -103,14 +121,14 @@ describe('PortfolioCalculator', () => { } ]; - const portfolioCalculator = factory.createCalculator({ + const portfolioCalculator = portfolioCalculatorFactory.createCalculator({ activities, calculationType: PerformanceCalculationType.TWR, currency: 'CHF', userId: userDummyData.id }); - const portfolioSnapshot = await portfolioCalculator.getSnapshot(); + const portfolioSnapshot = await portfolioCalculator.computeSnapshot(); const investments = portfolioCalculator.getInvestments(); diff --git a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.spec.ts b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.spec.ts index 536581070..d8431cd83 100644 --- a/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.spec.ts +++ b/apps/api/src/app/portfolio/calculator/twr/portfolio-calculator.spec.ts @@ -3,12 +3,14 @@ import { CurrentRateService } from '@ghostfolio/api/app/portfolio/current-rate.s import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; describe('PortfolioCalculator', () => { let configurationService: ConfigurationService; let currentRateService: CurrentRateService; let exchangeRateDataService: ExchangeRateDataService; - let factory: PortfolioCalculatorFactory; + let portfolioCalculatorFactory: PortfolioCalculatorFactory; + let portfolioSnapshotService: PortfolioSnapshotService; let redisCacheService: RedisCacheService; beforeEach(() => { @@ -23,12 +25,15 @@ describe('PortfolioCalculator', () => { null ); + portfolioSnapshotService = new PortfolioSnapshotService(null); + redisCacheService = new RedisCacheService(null, null); - factory = new PortfolioCalculatorFactory( + portfolioCalculatorFactory = new PortfolioCalculatorFactory( configurationService, currentRateService, exchangeRateDataService, + portfolioSnapshotService, redisCacheService ); }); diff --git a/apps/api/src/app/portfolio/portfolio.module.ts b/apps/api/src/app/portfolio/portfolio.module.ts index ad81e9e15..7ae74ee5f 100644 --- a/apps/api/src/app/portfolio/portfolio.module.ts +++ b/apps/api/src/app/portfolio/portfolio.module.ts @@ -10,12 +10,13 @@ import { TransformDataSourceInRequestModule } from '@ghostfolio/api/interceptors import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptors/transform-data-source-in-response/transform-data-source-in-response.module'; import { ApiModule } from '@ghostfolio/api/services/api/api.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; +import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module'; +import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { Module } from '@nestjs/common'; @@ -40,6 +41,7 @@ import { RulesService } from './rules.service'; MarketDataModule, OrderModule, PerformanceLoggingModule, + PortfolioSnapshotQueueModule, PrismaModule, RedactValuesInResponseModule, RedisCacheModule, diff --git a/apps/api/src/app/redis-cache/redis-cache.service.mock.ts b/apps/api/src/app/redis-cache/redis-cache.service.mock.ts index 094c7e6a0..2779308bd 100644 --- a/apps/api/src/app/redis-cache/redis-cache.service.mock.ts +++ b/apps/api/src/app/redis-cache/redis-cache.service.mock.ts @@ -1,13 +1,28 @@ +import { Filter } from '@ghostfolio/common/interfaces'; + import { Milliseconds } from 'cache-manager'; export const RedisCacheServiceMock = { + cache: new Map(), get: (key: string): Promise => { - return Promise.resolve(null); + const value = RedisCacheServiceMock.cache.get(key) || null; + + return Promise.resolve(value); }, - getPortfolioSnapshotKey: (userId: string): string => { - return `portfolio-snapshot-${userId}`; + getPortfolioSnapshotKey: ({ + filters, + userId + }: { + filters?: Filter[]; + userId: string; + }): string => { + const filtersHash = filters?.length; + + return `portfolio-snapshot-${userId}${filtersHash > 0 ? `-${filtersHash}` : ''}`; }, set: (key: string, value: string, ttl?: Milliseconds): Promise => { + RedisCacheServiceMock.cache.set(key, value); + return Promise.resolve(value); } }; diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index 864891c6a..17e970c1b 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -9,9 +9,9 @@ import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; -import { DataGatheringService } from './data-gathering/data-gathering.service'; import { ExchangeRateDataService } from './exchange-rate-data/exchange-rate-data.service'; import { PropertyService } from './property/property.service'; +import { DataGatheringService } from './queues/data-gathering/data-gathering.service'; import { TwitterBotService } from './twitter-bot/twitter-bot.service'; @Injectable() diff --git a/apps/api/src/services/data-gathering/data-gathering.module.ts b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts similarity index 93% rename from apps/api/src/services/data-gathering/data-gathering.module.ts rename to apps/api/src/services/queues/data-gathering/data-gathering.module.ts index f3ab2fc9c..b51823476 100644 --- a/apps/api/src/services/data-gathering/data-gathering.module.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts @@ -1,11 +1,11 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { DataEnhancerModule } from '@ghostfolio/api/services/data-provider/data-enhancer/data-enhancer.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module'; import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; +import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module'; import { DATA_GATHERING_QUEUE } from '@ghostfolio/common/config'; diff --git a/apps/api/src/services/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts similarity index 95% rename from apps/api/src/services/data-gathering/data-gathering.processor.ts rename to apps/api/src/services/queues/data-gathering/data-gathering.processor.ts index d8a6a7644..62f52d45b 100644 --- a/apps/api/src/services/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts @@ -4,7 +4,7 @@ import { MarketDataService } from '@ghostfolio/api/services/market-data/market-d import { DATA_GATHERING_QUEUE, GATHER_ASSET_PROFILE_PROCESS, - GATHER_HISTORICAL_MARKET_DATA_PROCESS + GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME } from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; @@ -58,7 +58,10 @@ export class DataGatheringProcessor { } } - @Process({ concurrency: 1, name: GATHER_HISTORICAL_MARKET_DATA_PROCESS }) + @Process({ + concurrency: 1, + name: GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME + }) public async gatherHistoricalMarketData(job: Job) { try { const { dataSource, date, symbol } = job.data; @@ -69,7 +72,7 @@ export class DataGatheringProcessor { currentDate, DATE_FORMAT )}`, - `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})` + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` ); const historicalData = await this.dataProviderService.getHistoricalRaw({ @@ -123,12 +126,12 @@ export class DataGatheringProcessor { currentDate, DATE_FORMAT )}`, - `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})` + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` ); } catch (error) { Logger.error( error, - `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})` + `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})` ); throw new Error(error); diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts similarity index 98% rename from apps/api/src/services/data-gathering/data-gathering.service.ts rename to apps/api/src/services/queues/data-gathering/data-gathering.service.ts index 8b8c65a21..72b8ac716 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts @@ -11,8 +11,8 @@ import { DATA_GATHERING_QUEUE_PRIORITY_HIGH, DATA_GATHERING_QUEUE_PRIORITY_LOW, DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, - GATHER_HISTORICAL_MARKET_DATA_PROCESS, - GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, + GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, + GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, PROPERTY_BENCHMARKS } from '@ghostfolio/common/config'; import { @@ -279,9 +279,9 @@ export class DataGatheringService { date, symbol }, - name: GATHER_HISTORICAL_MARKET_DATA_PROCESS, + name: GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME, opts: { - ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, + ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS, priority, jobId: `${getAssetProfileIdentifier({ dataSource, diff --git a/apps/api/src/services/queues/portfolio-snapshot/interfaces/portfolio-snapshot-queue-job.interface.ts b/apps/api/src/services/queues/portfolio-snapshot/interfaces/portfolio-snapshot-queue-job.interface.ts new file mode 100644 index 000000000..24948e211 --- /dev/null +++ b/apps/api/src/services/queues/portfolio-snapshot/interfaces/portfolio-snapshot-queue-job.interface.ts @@ -0,0 +1,7 @@ +import { Filter } from '@ghostfolio/common/interfaces'; + +export interface IPortfolioSnapshotQueueJob { + filters: Filter[]; + userCurrency: string; + userId: string; +} diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts new file mode 100644 index 000000000..331e5849f --- /dev/null +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.module.ts @@ -0,0 +1,37 @@ +import { OrderModule } from '@ghostfolio/api/app/order/order.module'; +import { PortfolioCalculatorFactory } from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator.factory'; +import { CurrentRateService } from '@ghostfolio/api/app/portfolio/current-rate.service'; +import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module'; +import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; +import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; +import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module'; +import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; +import { PortfolioSnapshotService } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.service'; +import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config'; + +import { BullModule } from '@nestjs/bull'; +import { Module } from '@nestjs/common'; + +import { PortfolioSnapshotProcessor } from './portfolio-snapshot.processor'; + +@Module({ + exports: [BullModule, PortfolioSnapshotService], + imports: [ + BullModule.registerQueue({ + name: PORTFOLIO_SNAPSHOT_QUEUE + }), + ConfigurationModule, + DataProviderModule, + ExchangeRateDataModule, + MarketDataModule, + OrderModule, + RedisCacheModule + ], + providers: [ + CurrentRateService, + PortfolioCalculatorFactory, + PortfolioSnapshotProcessor, + PortfolioSnapshotService + ] +}) +export class PortfolioSnapshotQueueModule {} diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts new file mode 100644 index 000000000..f8173559e --- /dev/null +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts @@ -0,0 +1,96 @@ +import { OrderService } from '@ghostfolio/api/app/order/order.service'; +import { + PerformanceCalculationType, + PortfolioCalculatorFactory +} from '@ghostfolio/api/app/portfolio/calculator/portfolio-calculator.factory'; +import { PortfolioSnapshotValue } from '@ghostfolio/api/app/portfolio/interfaces/snapshot-value.interface'; +import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { + CACHE_TTL_INFINITE, + PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME, + PORTFOLIO_SNAPSHOT_QUEUE +} from '@ghostfolio/common/config'; + +import { Process, Processor } from '@nestjs/bull'; +import { Injectable, Logger } from '@nestjs/common'; +import { Job } from 'bull'; +import { addMilliseconds } from 'date-fns'; + +import { IPortfolioSnapshotQueueJob } from './interfaces/portfolio-snapshot-queue-job.interface'; + +@Injectable() +@Processor(PORTFOLIO_SNAPSHOT_QUEUE) +export class PortfolioSnapshotProcessor { + public constructor( + private readonly calculatorFactory: PortfolioCalculatorFactory, + private readonly configurationService: ConfigurationService, + private readonly orderService: OrderService, + private readonly redisCacheService: RedisCacheService + ) {} + + @Process({ concurrency: 1, name: PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME }) + public async calculatePortfolioSnapshot( + job: Job + ) { + try { + const startTime = performance.now(); + + Logger.log( + `Portfolio snapshot calculation of user '${job.data.userId}' has been started`, + `PortfolioSnapshotProcessor (${PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME})` + ); + + const { activities } = + await this.orderService.getOrdersForPortfolioCalculator({ + filters: job.data.filters, + userCurrency: job.data.userCurrency, + userId: job.data.userId + }); + + const portfolioCalculator = this.calculatorFactory.createCalculator({ + activities, + calculationType: PerformanceCalculationType.TWR, + currency: job.data.userCurrency, + filters: job.data.filters, + userId: job.data.userId + }); + + const snapshot = await portfolioCalculator.computeSnapshot(); + + Logger.log( + `Portfolio snapshot calculation of user '${job.data.userId}' has been completed in ${( + (performance.now() - startTime) / + 1000 + ).toFixed(3)} seconds`, + `PortfolioSnapshotProcessor (${PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME})` + ); + + const expiration = addMilliseconds( + new Date(), + this.configurationService.get('CACHE_QUOTES_TTL') + ); + + this.redisCacheService.set( + this.redisCacheService.getPortfolioSnapshotKey({ + filters: job.data.filters, + userId: job.data.userId + }), + JSON.stringify(({ + expiration: expiration.getTime(), + portfolioSnapshot: snapshot + })), + CACHE_TTL_INFINITE + ); + + return snapshot; + } catch (error) { + Logger.error( + error, + `PortfolioSnapshotProcessor (${PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME})` + ); + + throw new Error(error); + } + } +} diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock.ts new file mode 100644 index 000000000..8d7526906 --- /dev/null +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.mock.ts @@ -0,0 +1,34 @@ +import { Job, JobOptions } from 'bull'; +import { setTimeout } from 'timers/promises'; + +import { IPortfolioSnapshotQueueJob } from './interfaces/portfolio-snapshot-queue-job.interface'; + +export const PortfolioSnapshotServiceMock = { + addJobToQueue({ + data, + name, + opts + }: { + data: IPortfolioSnapshotQueueJob; + name: string; + opts?: JobOptions; + }): Promise> { + const mockJob: Partial> = { + finished: async () => { + await setTimeout(100); + + return Promise.resolve(); + } + }; + + this.jobsStore.set(opts?.jobId, mockJob); + + return Promise.resolve(mockJob as Job); + }, + getJob(jobId: string): Promise> { + const job = this.jobsStore.get(jobId); + + return Promise.resolve(job as Job); + }, + jobsStore: new Map>>() +}; diff --git a/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.ts b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.ts new file mode 100644 index 000000000..27ebdee53 --- /dev/null +++ b/apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.service.ts @@ -0,0 +1,31 @@ +import { PORTFOLIO_SNAPSHOT_QUEUE } from '@ghostfolio/common/config'; + +import { InjectQueue } from '@nestjs/bull'; +import { Injectable } from '@nestjs/common'; +import { JobOptions, Queue } from 'bull'; + +import { IPortfolioSnapshotQueueJob } from './interfaces/portfolio-snapshot-queue-job.interface'; + +@Injectable() +export class PortfolioSnapshotService { + public constructor( + @InjectQueue(PORTFOLIO_SNAPSHOT_QUEUE) + private readonly portfolioSnapshotQueue: Queue + ) {} + + public async addJobToQueue({ + data, + name, + opts + }: { + data: IPortfolioSnapshotQueueJob; + name: string; + opts?: JobOptions; + }) { + return this.portfolioSnapshotQueue.add(name, data, opts); + } + + public async getJob(jobId: string) { + return this.portfolioSnapshotQueue.getJob(jobId); + } +} diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.html index e194b2b37..ef46d766e 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.html +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html @@ -35,6 +35,8 @@ Asset Profile } @else if (element.name === 'GATHER_HISTORICAL_MARKET_DATA') { Historical Market Data + } @else if (element.name === 'PORTFOLIO') { + Portfolio Snapshot } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 00e756810..79fdf6d05 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -40,6 +40,10 @@ export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round( DATA_GATHERING_QUEUE_PRIORITY_LOW / 2 ); +export const PORTFOLIO_SNAPSHOT_QUEUE = 'PORTFOLIO_SNAPSHOT_QUEUE'; +export const PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_HIGH = 1; +export const PORTFOLIO_SNAPSHOT_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; + export const DEFAULT_CURRENCY = 'USD'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; export const DEFAULT_LANGUAGE_CODE = 'en'; @@ -76,9 +80,10 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = { }, removeOnComplete: true }; -export const GATHER_HISTORICAL_MARKET_DATA_PROCESS = + +export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME = 'GATHER_HISTORICAL_MARKET_DATA'; -export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = { +export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = { attempts: 12, backoff: { delay: ms('1 minute'), @@ -87,6 +92,11 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = { removeOnComplete: true }; +export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME = 'PORTFOLIO'; +export const PORTFOLIO_SNAPSHOT_PROCESS_JOB_OPTIONS: JobOptions = { + removeOnComplete: true +}; + export const HEADER_KEY_IMPERSONATION = 'Impersonation-Id'; export const HEADER_KEY_TIMEZONE = 'Timezone'; export const HEADER_KEY_TOKEN = 'Authorization';