From dec1d89c5c5a662e99b524443a66f56295434e2f Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:02:38 +0100 Subject: [PATCH] Feature/increase timeout in data provider and enhancer health check endpoint (#2621) * Increase timeout in health check endpoint (data enhancer and provider) * Update changelog --- CHANGELOG.md | 8 +++++--- .../data-provider/alpha-vantage/alpha-vantage.service.ts | 3 +++ .../services/data-provider/coingecko/coingecko.service.ts | 4 +++- .../data-provider/data-enhancer/data-enhancer.service.ts | 2 ++ .../data-enhancer/openfigi/openfigi.service.ts | 4 +++- .../data-enhancer/trackinsight/trackinsight.service.ts | 4 +++- .../data-enhancer/yahoo-finance/yahoo-finance.service.ts | 8 +++++++- .../src/services/data-provider/data-provider.service.ts | 6 +++++- .../eod-historical-data/eod-historical-data.service.ts | 4 +++- .../financial-modeling-prep.service.ts | 4 +++- .../data-provider/google-sheets/google-sheets.service.ts | 3 +++ .../data-provider/interfaces/data-enhancer.interface.ts | 2 ++ .../data-provider/interfaces/data-provider.interface.ts | 2 ++ .../src/services/data-provider/manual/manual.service.ts | 2 ++ .../services/data-provider/rapid-api/rapid-api.service.ts | 2 ++ .../data-provider/yahoo-finance/yahoo-finance.service.ts | 7 ++++++- 16 files changed, 54 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d42cccdc3..e6e55c6c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Improved the import of historical market data in the admin control panel +- Increased the timeout in the health check endpoint for data enhancers +- Increased the timeout in the health check endpoint for data providers - Removed the account type from the `Account` database schema ## 2.19.0 - 2023-11-06 @@ -368,7 +370,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added health check endpoints for data enhancers +- Added a health check endpoint for data enhancers ### Changed @@ -544,7 +546,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Improved the usability of the login dialog -- Disabled the caching in the health check endpoints for data providers +- Disabled the caching in the health check endpoint for data providers - Improved the content of the Frequently Asked Questions (FAQ) page - Upgraded `prisma` from version `4.15.0` to `4.16.2` @@ -932,7 +934,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a fallback to historical market data if a data provider does not provide live data - Added a general health check endpoint -- Added health check endpoints for data providers +- Added a health check endpoint for data providers ### Changed diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index fbbfffc67..2b03f519d 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -5,6 +5,7 @@ import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; +import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; @@ -106,8 +107,10 @@ export class AlphaVantageService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { return {}; diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts index 3e93d42bf..6ed67bf8b 100644 --- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts +++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts @@ -135,8 +135,10 @@ export class CoinGeckoService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { const response: { [symbol: string]: IDataProviderResponse } = {}; @@ -150,7 +152,7 @@ export class CoinGeckoService implements DataProviderInterface { setTimeout(() => { abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); + }, requestTimeout); const quotes = await got( `${this.URL}/simple/price?ids=${symbols.join( diff --git a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts index e5038c7c6..d9b509f42 100644 --- a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts @@ -2,6 +2,7 @@ import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/in import { HttpException, Inject, Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; +import ms from 'ms'; @Injectable() export class DataEnhancerService { @@ -24,6 +25,7 @@ export class DataEnhancerService { try { const assetProfile = await dataEnhancer.enhance({ + requestTimeout: ms('30 seconds'), response: { assetClass: 'EQUITY', assetSubClass: 'ETF' diff --git a/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts b/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts index 363cbb167..dda30ba1a 100644 --- a/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts @@ -15,9 +15,11 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface { ) {} public async enhance({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, response, symbol }: { + requestTimeout?: number; response: Partial; symbol: string; }): Promise> { @@ -45,7 +47,7 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface { setTimeout(() => { abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); + }, requestTimeout); const mappings = await got .post(`${OpenFigiDataEnhancerService.baseUrl}/v3/mapping`, { diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts index 36eb22dad..4de08fcef 100644 --- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts @@ -21,9 +21,11 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface { }; public async enhance({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, response, symbol }: { + requestTimeout?: number; response: Partial; symbol: string; }): Promise> { @@ -37,7 +39,7 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface { setTimeout(() => { abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); + }, requestTimeout); const profile = await got( `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`, diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index d46af66a7..67fc0a9c1 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -1,6 +1,10 @@ import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service'; import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; -import { DEFAULT_CURRENCY, UNKNOWN_KEY } from '@ghostfolio/common/config'; +import { + DEFAULT_CURRENCY, + DEFAULT_REQUEST_TIMEOUT, + UNKNOWN_KEY +} from '@ghostfolio/common/config'; import { isCurrency } from '@ghostfolio/common/helper'; import { Injectable, Logger } from '@nestjs/common'; import { @@ -72,9 +76,11 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { } public async enhance({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, response, symbol }: { + requestTimeout?: number; response: Partial; symbol: string; }): Promise> { diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 7a998eeb3..ef5143475 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -17,6 +17,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; import { format, isValid } from 'date-fns'; import { groupBy, isEmpty, isNumber } from 'lodash'; +import ms from 'ms'; @Injectable() export class DataProviderService { @@ -52,6 +53,7 @@ export class DataProviderService { symbol } ], + requestTimeout: ms('30 seconds'), useCache: false }); @@ -236,9 +238,11 @@ export class DataProviderService { public async getQuotes({ items, + requestTimeout, useCache = true }: { items: UniqueAsset[]; + requestTimeout?: number; useCache?: boolean; }): Promise<{ [symbol: string]: IDataProviderResponse; @@ -312,7 +316,7 @@ export class DataProviderService { ); const promise = Promise.resolve( - dataProvider.getQuotes({ symbols: symbolsChunk }) + dataProvider.getQuotes({ requestTimeout, symbols: symbolsChunk }) ); promises.push( diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index 37c37e389..a582a953d 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -132,8 +132,10 @@ export class EodHistoricalDataService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { let response: { [symbol: string]: IDataProviderResponse } = {}; @@ -151,7 +153,7 @@ export class EodHistoricalDataService implements DataProviderInterface { setTimeout(() => { abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); + }, requestTimeout); const realTimeResponse = await got( `${this.URL}/real-time/${eodHistoricalDataSymbols[0]}?api_token=${ diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts index b08bc099e..c74cfdc00 100644 --- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts +++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts @@ -114,8 +114,10 @@ export class FinancialModelingPrepService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { const response: { [symbol: string]: IDataProviderResponse } = {}; @@ -129,7 +131,7 @@ export class FinancialModelingPrepService implements DataProviderInterface { setTimeout(() => { abortController.abort(); - }, DEFAULT_REQUEST_TIMEOUT); + }, requestTimeout); const response = await got( `${this.URL}/quote/${symbols.join(',')}?apikey=${this.apiKey}`, diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts index a541fcd12..6dc8dc80b 100644 --- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts +++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts @@ -7,6 +7,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; +import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config'; import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; @@ -100,8 +101,10 @@ export class GoogleSheetsService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { const response: { [symbol: string]: IDataProviderResponse } = {}; diff --git a/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts b/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts index 9c6db9196..73e0cc69d 100644 --- a/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts @@ -2,9 +2,11 @@ import { SymbolProfile } from '@prisma/client'; export interface DataEnhancerInterface { enhance({ + requestTimeout, response, symbol }: { + requestTimeout?: number; response: Partial; symbol: string; }): Promise>; diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index f4daeb108..a27c8544c 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -37,8 +37,10 @@ export interface DataProviderInterface { getName(): DataSource; getQuotes({ + requestTimeout, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }>; diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index bea4b60ce..1464a526d 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -134,8 +134,10 @@ export class ManualService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { const response: { [symbol: string]: IDataProviderResponse } = {}; diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index ce188ffe0..5e833cbf1 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -88,8 +88,10 @@ export class RapidApiService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { if (symbols.length <= 0) { diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 2f69448f9..96bffa7ea 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -6,7 +6,10 @@ import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; -import { DEFAULT_CURRENCY } from '@ghostfolio/common/config'; +import { + DEFAULT_CURRENCY, + DEFAULT_REQUEST_TIMEOUT +} from '@ghostfolio/common/config'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; @@ -157,8 +160,10 @@ export class YahooFinanceService implements DataProviderInterface { } public async getQuotes({ + requestTimeout = DEFAULT_REQUEST_TIMEOUT, symbols }: { + requestTimeout?: number; symbols: string[]; }): Promise<{ [symbol: string]: IDataProviderResponse }> { const response: { [symbol: string]: IDataProviderResponse } = {};