From 54ea6c84b4ff85151a20ad098600a7cbec4eb8f4 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 24 Jun 2023 13:06:28 +0200 Subject: [PATCH] Feature/add caching for quotes (#2095) * Add caching for quotes * Update changelog --- CHANGELOG.md | 1 + .../portfolio/current-rate.service.spec.ts | 3 +- .../app/redis-cache/redis-cache.service.ts | 5 ++ .../configuration/configuration.service.ts | 1 + .../data-provider/data-provider.module.ts | 2 + .../data-provider/data-provider.service.ts | 53 +++++++++++++++++-- .../interfaces/environment.interface.ts | 1 + 7 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc54f98b4..22c03989b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added the caching for current market prices - Added a loading indicator to the import dividends dialog ### Changed diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 0dd2b3a68..c9711aa7b 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -98,7 +98,8 @@ describe('CurrentRateService', () => { [], null, null, - propertyService + propertyService, + null ); exchangeRateDataService = new ExchangeRateDataService( null, diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.ts index 49f736bc5..fb75460ed 100644 --- a/apps/api/src/app/redis-cache/redis-cache.service.ts +++ b/apps/api/src/app/redis-cache/redis-cache.service.ts @@ -1,4 +1,5 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; +import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common'; import { Cache } from 'cache-manager'; @@ -13,6 +14,10 @@ export class RedisCacheService { return await this.cache.get(key); } + public getQuoteKey({ dataSource, symbol }: UniqueAsset) { + return `quote-${dataSource}-${symbol}`; + } + public async remove(key: string) { await this.cache.del(key); } diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index c5a468905..e522aeccd 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -16,6 +16,7 @@ export class ConfigurationService { default: 'USD' }), BETTER_UPTIME_API_KEY: str({ default: '' }), + CACHE_QUOTES_TTL: num({ default: 1 }), CACHE_TTL: num({ default: 1 }), DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }), DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }), diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 3ba678631..b3a219a50 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -1,3 +1,4 @@ +import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module'; import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module'; import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service'; @@ -26,6 +27,7 @@ import { DataProviderService } from './data-provider.service'; MarketDataModule, PrismaModule, PropertyModule, + RedisCacheModule, SymbolProfileModule ], providers: [ diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index c05fc3e4f..6209b89aa 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -1,3 +1,4 @@ +import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service'; import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; @@ -27,7 +28,8 @@ export class DataProviderService { private readonly dataProviderInterfaces: DataProviderInterface[], private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, - private readonly propertyService: PropertyService + private readonly propertyService: PropertyService, + private readonly redisCacheService: RedisCacheService ) { this.initialize(); } @@ -235,9 +237,43 @@ export class DataProviderService { } = {}; const startTimeTotal = performance.now(); - const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); + // Get items from cache + const itemsToFetch: IDataGatheringItem[] = []; - const promises = []; + for (const { dataSource, symbol } of items) { + const quoteString = await this.redisCacheService.get( + this.redisCacheService.getQuoteKey({ dataSource, symbol }) + ); + + if (quoteString) { + try { + const cachedDataProviderResponse = JSON.parse(quoteString); + response[symbol] = cachedDataProviderResponse; + } catch {} + } + + if (!quoteString) { + itemsToFetch.push({ dataSource, symbol }); + } + } + + const numberOfItemsInCache = Object.keys(response)?.length; + + if (numberOfItemsInCache) { + Logger.debug( + `Fetched ${numberOfItemsInCache} quote${ + numberOfItemsInCache > 1 ? 's' : '' + } from cache in ${((performance.now() - startTimeTotal) / 1000).toFixed( + 3 + )} seconds` + ); + } + + const itemsGroupedByDataSource = groupBy(itemsToFetch, ({ dataSource }) => { + return dataSource; + }); + + const promises: Promise[] = []; for (const [dataSource, dataGatheringItems] of Object.entries( itemsGroupedByDataSource @@ -271,6 +307,15 @@ export class DataProviderService { result )) { response[symbol] = dataProviderResponse; + + this.redisCacheService.set( + this.redisCacheService.getQuoteKey({ + dataSource: DataSource[dataSource], + symbol + }), + JSON.stringify(dataProviderResponse), + this.configurationService.get('CACHE_QUOTES_TTL') + ); } Logger.debug( @@ -283,7 +328,7 @@ export class DataProviderService { ); try { - await this.marketDataService.updateMany({ + this.marketDataService.updateMany({ data: Object.keys(response) .filter((symbol) => { return ( diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 1256f0f2f..88fa4c3f5 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -5,6 +5,7 @@ export interface Environment extends CleanedEnvAccessors { ALPHA_VANTAGE_API_KEY: string; BASE_CURRENCY: string; BETTER_UPTIME_API_KEY: string; + CACHE_QUOTES_TTL: number; CACHE_TTL: number; DATA_SOURCE_EXCHANGE_RATES: string; DATA_SOURCE_IMPORT: string;