Feature/increase timeout in data provider and enhancer health check endpoint (#2621)

* Increase timeout in health check endpoint (data enhancer and provider)

* Update changelog

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Improved the import of historical market data in the admin control panel
+- Increased the timeout in the health check endpoint for data enhancers
+- Increased the timeout in the health check endpoint for data providers
 - Removed the account type from the `Account` database schema
 
 ## 2.19.0 - 2023-11-06
@@ -368,7 +370,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added health check endpoints for data enhancers
+- Added a health check endpoint for data enhancers
 
 ### Changed
 
@@ -544,7 +546,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Improved the usability of the login dialog
-- Disabled the caching in the health check endpoints for data providers
+- Disabled the caching in the health check endpoint for data providers
 - Improved the content of the Frequently Asked Questions (FAQ) page
 - Upgraded `prisma` from version `4.15.0` to `4.16.2`
 
@@ -932,7 +934,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added a fallback to historical market data if a data provider does not provide live data
 - Added a general health check endpoint
-- Added health check endpoints for data providers
+- Added a health check endpoint for data providers
 
 ### Changed
 

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -5,6 +5,7 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
@@ -106,8 +107,10 @@ export class AlphaVantageService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     return {};

apps/api/src/services/data-provider/coingecko/coingecko.service.ts

@@ -135,8 +135,10 @@ export class CoinGeckoService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     const response: { [symbol: string]: IDataProviderResponse } = {};
@@ -150,7 +152,7 @@ export class CoinGeckoService implements DataProviderInterface {
 
       setTimeout(() => {
         abortController.abort();
-      }, DEFAULT_REQUEST_TIMEOUT);
+      }, requestTimeout);
 
       const quotes = await got(
         `${this.URL}/simple/price?ids=${symbols.join(

apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts

@@ -2,6 +2,7 @@ import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/in
 import { HttpException, Inject, Injectable } from '@nestjs/common';
 import { Prisma } from '@prisma/client';
 import { StatusCodes, getReasonPhrase } from 'http-status-codes';
+import ms from 'ms';
 
 @Injectable()
 export class DataEnhancerService {
@@ -24,6 +25,7 @@ export class DataEnhancerService {
 
     try {
       const assetProfile = await dataEnhancer.enhance({
+        requestTimeout: ms('30 seconds'),
         response: {
           assetClass: 'EQUITY',
           assetSubClass: 'ETF'

apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts

@@ -15,9 +15,11 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
   ) {}
 
   public async enhance({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     response,
     symbol
   }: {
+    requestTimeout?: number;
     response: Partial<SymbolProfile>;
     symbol: string;
   }): Promise<Partial<SymbolProfile>> {
@@ -45,7 +47,7 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
 
     setTimeout(() => {
       abortController.abort();
-    }, DEFAULT_REQUEST_TIMEOUT);
+    }, requestTimeout);
 
     const mappings = await got
       .post(`${OpenFigiDataEnhancerService.baseUrl}/v3/mapping`, {

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -21,9 +21,11 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
   };
 
   public async enhance({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     response,
     symbol
   }: {
+    requestTimeout?: number;
     response: Partial<SymbolProfile>;
     symbol: string;
   }): Promise<Partial<SymbolProfile>> {
@@ -37,7 +39,7 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
 
     setTimeout(() => {
       abortController.abort();
-    }, DEFAULT_REQUEST_TIMEOUT);
+    }, requestTimeout);
 
     const profile = await got(
       `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts

@@ -1,6 +1,10 @@
 import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
-import { DEFAULT_CURRENCY, UNKNOWN_KEY } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DEFAULT_REQUEST_TIMEOUT,
+  UNKNOWN_KEY
+} from '@ghostfolio/common/config';
 import { isCurrency } from '@ghostfolio/common/helper';
 import { Injectable, Logger } from '@nestjs/common';
 import {
@@ -72,9 +76,11 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
   }
 
   public async enhance({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     response,
     symbol
   }: {
+    requestTimeout?: number;
     response: Partial<SymbolProfile>;
     symbol: string;
   }): Promise<Partial<SymbolProfile>> {

apps/api/src/services/data-provider/data-provider.service.ts

@@ -17,6 +17,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
 import { format, isValid } from 'date-fns';
 import { groupBy, isEmpty, isNumber } from 'lodash';
+import ms from 'ms';
 
 @Injectable()
 export class DataProviderService {
@@ -52,6 +53,7 @@ export class DataProviderService {
           symbol
         }
       ],
+      requestTimeout: ms('30 seconds'),
       useCache: false
     });
 
@@ -236,9 +238,11 @@ export class DataProviderService {
 
   public async getQuotes({
     items,
+    requestTimeout,
     useCache = true
   }: {
     items: UniqueAsset[];
+    requestTimeout?: number;
     useCache?: boolean;
   }): Promise<{
     [symbol: string]: IDataProviderResponse;
@@ -312,7 +316,7 @@ export class DataProviderService {
         );
 
         const promise = Promise.resolve(
-          dataProvider.getQuotes({ symbols: symbolsChunk })
+          dataProvider.getQuotes({ requestTimeout, symbols: symbolsChunk })
         );
 
         promises.push(

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -132,8 +132,10 @@ export class EodHistoricalDataService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     let response: { [symbol: string]: IDataProviderResponse } = {};
@@ -151,7 +153,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
 
       setTimeout(() => {
         abortController.abort();
-      }, DEFAULT_REQUEST_TIMEOUT);
+      }, requestTimeout);
 
       const realTimeResponse = await got(
         `${this.URL}/real-time/${eodHistoricalDataSymbols[0]}?api_token=${

apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts

@@ -114,8 +114,10 @@ export class FinancialModelingPrepService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     const response: { [symbol: string]: IDataProviderResponse } = {};
@@ -129,7 +131,7 @@ export class FinancialModelingPrepService implements DataProviderInterface {
 
       setTimeout(() => {
         abortController.abort();
-      }, DEFAULT_REQUEST_TIMEOUT);
+      }, requestTimeout);
 
       const response = await got(
         `${this.URL}/quote/${symbols.join(',')}?apikey=${this.apiKey}`,

apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts

@@ -7,6 +7,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
@@ -100,8 +101,10 @@ export class GoogleSheetsService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     const response: { [symbol: string]: IDataProviderResponse } = {};

apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts

@@ -2,9 +2,11 @@ import { SymbolProfile } from '@prisma/client';
 
 export interface DataEnhancerInterface {
   enhance({
+    requestTimeout,
     response,
     symbol
   }: {
+    requestTimeout?: number;
     response: Partial<SymbolProfile>;
     symbol: string;
   }): Promise<Partial<SymbolProfile>>;

apps/api/src/services/data-provider/interfaces/data-provider.interface.ts

@@ -37,8 +37,10 @@ export interface DataProviderInterface {
   getName(): DataSource;
 
   getQuotes({
+    requestTimeout,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }>;
 

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -134,8 +134,10 @@ export class ManualService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     const response: { [symbol: string]: IDataProviderResponse } = {};

apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts

@@ -88,8 +88,10 @@ export class RapidApiService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     if (symbols.length <= 0) {

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -6,7 +6,10 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DEFAULT_REQUEST_TIMEOUT
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
@@ -157,8 +160,10 @@ export class YahooFinanceService implements DataProviderInterface {
   }
 
   public async getQuotes({
+    requestTimeout = DEFAULT_REQUEST_TIMEOUT,
     symbols
   }: {
+    requestTimeout?: number;
     symbols: string[];
   }): Promise<{ [symbol: string]: IDataProviderResponse }> {
     const response: { [symbol: string]: IDataProviderResponse } = {};