From ed735e0b29218efc169414d2ba73dcd1a581286c Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 15 Jul 2023 10:54:19 +0200 Subject: [PATCH] Feature/disable caching in health check endpoints for data providers (#2147) * Disable caching in health check endpoint * Update changelog --- CHANGELOG.md | 6 ++ .../src/app/benchmark/benchmark.service.ts | 6 +- .../src/app/portfolio/current-rate.service.ts | 2 +- .../src/app/portfolio/portfolio.service.ts | 30 ++++---- apps/api/src/app/symbol/symbol.service.ts | 6 +- .../data-provider/data-provider.service.ts | 68 +++++++++++-------- .../exchange-rate-data.service.ts | 6 +- 7 files changed, 72 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22650dc37..344d9f77d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Disabled the caching in the health check endpoints for data providers + ## 1.289.0 - 2023-07-14 ### Changed diff --git a/apps/api/src/app/benchmark/benchmark.service.ts b/apps/api/src/app/benchmark/benchmark.service.ts index 73b48068b..785c2801a 100644 --- a/apps/api/src/app/benchmark/benchmark.service.ts +++ b/apps/api/src/app/benchmark/benchmark.service.ts @@ -66,11 +66,11 @@ export class BenchmarkService { const promises: Promise[] = []; - const quotes = await this.dataProviderService.getQuotes( - benchmarkAssetProfiles.map(({ dataSource, symbol }) => { + const quotes = await this.dataProviderService.getQuotes({ + items: benchmarkAssetProfiles.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }) - ); + }); for (const { dataSource, symbol } of benchmarkAssetProfiles) { promises.push(this.marketDataService.getMax({ dataSource, symbol })); diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 6d48573f7..a4ee317bb 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -38,7 +38,7 @@ export class CurrentRateService { if (includeToday) { promises.push( this.dataProviderService - .getQuotes(dataGatheringItems) + .getQuotes({ items: dataGatheringItems }) .then((dataResultProvider) => { const result: GetValueObject[] = []; for (const dataGatheringItem of dataGatheringItems) { diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 66f3841a4..7d13728d2 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -504,15 +504,17 @@ export class PortfolioService { ); } - const dataGatheringItems = currentPositions.positions.map((position) => { - return { - dataSource: position.dataSource, - symbol: position.symbol - }; - }); + const dataGatheringItems = currentPositions.positions.map( + ({ dataSource, symbol }) => { + return { + dataSource, + symbol + }; + } + ); const [dataProviderResponses, symbolProfiles] = await Promise.all([ - this.dataProviderService.getQuotes(dataGatheringItems), + this.dataProviderService.getQuotes({ items: dataGatheringItems }), this.symbolProfileService.getSymbolProfiles(dataGatheringItems) ]); @@ -897,9 +899,9 @@ export class PortfolioService { ) }; } else { - const currentData = await this.dataProviderService.getQuotes([ - { dataSource: DataSource.YAHOO, symbol: aSymbol } - ]); + const currentData = await this.dataProviderService.getQuotes({ + items: [{ dataSource: DataSource.YAHOO, symbol: aSymbol }] + }); const marketPrice = currentData[aSymbol]?.marketPrice; let historicalData = await this.dataProviderService.getHistorical( @@ -1000,15 +1002,15 @@ export class PortfolioService { (item) => !item.quantity.eq(0) ); - const dataGatheringItem = positions.map((position) => { + const dataGatheringItems = positions.map(({ dataSource, symbol }) => { return { - dataSource: position.dataSource, - symbol: position.symbol + dataSource, + symbol }; }); const [dataProviderResponses, symbolProfiles] = await Promise.all([ - this.dataProviderService.getQuotes(dataGatheringItem), + this.dataProviderService.getQuotes({ items: dataGatheringItems }), this.symbolProfileService.getSymbolProfiles( positions.map(({ dataSource, symbol }) => { return { dataSource, symbol }; diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index bc626a97f..5eacbb1b0 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -27,9 +27,9 @@ export class SymbolService { dataGatheringItem: IDataGatheringItem; includeHistoricalData?: number; }): Promise { - const quotes = await this.dataProviderService.getQuotes([ - dataGatheringItem - ]); + const quotes = await this.dataProviderService.getQuotes({ + items: [dataGatheringItem] + }); const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {}; if (dataGatheringItem.dataSource && marketPrice >= 0) { diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index b4f4e10b5..557699495 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -3,7 +3,6 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.in import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { - IDataGatheringItem, IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; @@ -12,6 +11,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; +import { UniqueAsset } from '@ghostfolio/common/interfaces'; import type { Granularity, UserWithSettings } from '@ghostfolio/common/types'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; @@ -45,12 +45,15 @@ export class DataProviderService { const dataProvider = this.getDataProvider(dataSource); const symbol = dataProvider.getTestSymbol(); - const quotes = await this.getQuotes([ - { - dataSource, - symbol - } - ]); + const quotes = await this.getQuotes({ + items: [ + { + dataSource, + symbol + } + ], + useCache: false + }); if (quotes[symbol]?.marketPrice > 0) { return true; @@ -59,14 +62,16 @@ export class DataProviderService { return false; } - public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{ + public async getAssetProfiles(items: UniqueAsset[]): Promise<{ [symbol: string]: Partial; }> { const response: { [symbol: string]: Partial; } = {}; - const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); + const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => { + return dataSource; + }); const promises = []; @@ -127,7 +132,7 @@ export class DataProviderService { } public async getHistorical( - aItems: IDataGatheringItem[], + aItems: UniqueAsset[], aGranularity: Granularity = 'month', from: Date, to: Date @@ -155,11 +160,11 @@ export class DataProviderService { )}'` : ''; - const dataSources = aItems.map((item) => { - return item.dataSource; + const dataSources = aItems.map(({ dataSource }) => { + return dataSource; }); - const symbols = aItems.map((item) => { - return item.symbol; + const symbols = aItems.map(({ symbol }) => { + return symbol; }); try { @@ -192,7 +197,7 @@ export class DataProviderService { } public async getHistoricalRaw( - aDataGatheringItems: IDataGatheringItem[], + aDataGatheringItems: UniqueAsset[], from: Date, to: Date ): Promise<{ @@ -229,7 +234,13 @@ export class DataProviderService { return result; } - public async getQuotes(items: IDataGatheringItem[]): Promise<{ + public async getQuotes({ + items, + useCache = true + }: { + items: UniqueAsset[]; + useCache?: boolean; + }): Promise<{ [symbol: string]: IDataProviderResponse; }> { const response: { @@ -238,23 +249,24 @@ export class DataProviderService { const startTimeTotal = performance.now(); // Get items from cache - const itemsToFetch: IDataGatheringItem[] = []; + const itemsToFetch: UniqueAsset[] = []; for (const { dataSource, symbol } of items) { - const quoteString = await this.redisCacheService.get( - this.redisCacheService.getQuoteKey({ dataSource, symbol }) - ); + if (useCache) { + const quoteString = await this.redisCacheService.get( + this.redisCacheService.getQuoteKey({ dataSource, symbol }) + ); - if (quoteString) { - try { - const cachedDataProviderResponse = JSON.parse(quoteString); - response[symbol] = cachedDataProviderResponse; - } catch {} + if (quoteString) { + try { + const cachedDataProviderResponse = JSON.parse(quoteString); + response[symbol] = cachedDataProviderResponse; + continue; + } catch {} + } } - if (!quoteString) { - itemsToFetch.push({ dataSource, symbol }); - } + itemsToFetch.push({ dataSource, symbol }); } const numberOfItemsInCache = Object.keys(response)?.length; diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 448933b42..9e1daa224 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -64,11 +64,11 @@ export class ExchangeRateDataService { if (Object.keys(result).length !== this.currencyPairs.length) { // Load currencies directly from data provider as a fallback // if historical data is not fully available - const quotes = await this.dataProviderService.getQuotes( - this.currencyPairs.map(({ dataSource, symbol }) => { + const quotes = await this.dataProviderService.getQuotes({ + items: this.currencyPairs.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }) - ); + }); for (const symbol of Object.keys(quotes)) { if (isNumber(quotes[symbol].marketPrice)) {