From 9f2a49a1c700a5fef9455883a3a0b03468dd5a83 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 30 Jul 2022 15:48:45 +0200 Subject: [PATCH] Feature/introduce max number of symbols per data provider request (#1111) * Introduce maximum number of symbols per request * Set log level settings * Update changelog --- CHANGELOG.md | 7 +++ apps/api/src/main.ts | 7 ++- .../data-provider/data-provider.service.ts | 55 +++++++++++++++---- .../eod-historical-data.service.ts | 6 ++ .../interfaces/data-provider.interface.ts | 2 + .../yahoo-finance/yahoo-finance.service.ts | 4 ++ 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f90870cd..28c77e173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Improved the performance of data provider requests by introducing a maximum number of symbols per request (chunk size) +- Changed the log level settings + ## 1.175.0 - 29.07.2022 ### Added diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 7b4ea21e9..fd8237cd2 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -5,7 +5,12 @@ import { AppModule } from './app/app.module'; import { environment } from './environments/environment'; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create(AppModule, { + logger: + process.env.NODE_ENV === 'production' + ? ['error', 'log', 'warn'] + : ['debug', 'error', 'log', 'verbose', 'warn'] + }); app.enableCors(); app.enableVersioning({ defaultVersion: '1', diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index fd44f2426..7f10dc3a0 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -168,6 +168,7 @@ export class DataProviderService { const response: { [symbol: string]: IDataProviderResponse; } = {}; + const startTimeTotal = performance.now(); const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource); @@ -176,25 +177,59 @@ export class DataProviderService { for (const [dataSource, dataGatheringItems] of Object.entries( itemsGroupedByDataSource )) { + const dataProvider = this.getDataProvider(DataSource[dataSource]); + const symbols = dataGatheringItems.map((dataGatheringItem) => { return dataGatheringItem.symbol; }); - const promise = Promise.resolve( - this.getDataProvider(DataSource[dataSource]).getQuotes(symbols) - ); + const maximumNumberOfSymbolsPerRequest = + dataProvider.getMaxNumberOfSymbolsPerRequest?.() ?? + Number.MAX_SAFE_INTEGER; + for ( + let i = 0; + i < symbols.length; + i += maximumNumberOfSymbolsPerRequest + ) { + const startTimeDataSource = performance.now(); + + const symbolsChunk = symbols.slice( + i, + i + maximumNumberOfSymbolsPerRequest + ); - promises.push( - promise.then((result) => { - for (const [symbol, dataProviderResponse] of Object.entries(result)) { - response[symbol] = dataProviderResponse; - } - }) - ); + const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk)); + + promises.push( + promise.then((result) => { + for (const [symbol, dataProviderResponse] of Object.entries( + result + )) { + response[symbol] = dataProviderResponse; + } + + Logger.debug( + `Fetched ${symbolsChunk.length} quotes from ${dataSource} in ${( + (performance.now() - startTimeDataSource) / + 1000 + ).toFixed(3)} seconds` + ); + }) + ); + } } await Promise.all(promises); + Logger.debug('------------------------------------------------'); + Logger.debug( + `Fetched ${items.length} quotes in ${( + (performance.now() - startTimeTotal) / + 1000 + ).toFixed(3)} seconds` + ); + Logger.debug('================================================'); + return response; } diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index 1dd3c7aff..c87c6ec3e 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -81,6 +81,12 @@ export class EodHistoricalDataService implements DataProviderInterface { } } + public getMaxNumberOfSymbolsPerRequest() { + // It is not recommended using more than 15-20 tickers per request + // https://eodhistoricaldata.com/financial-apis/live-realtime-stocks-api + return 20; + } + public getName(): DataSource { return DataSource.EOD_HISTORICAL_DATA; } diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index 16cf44603..6719f3099 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -20,6 +20,8 @@ export interface DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }>; // TODO: Return only one symbol + getMaxNumberOfSymbolsPerRequest?(): number; + getName(): DataSource; getQuotes( diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index bddd529c2..d6a6a8412 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -208,6 +208,10 @@ export class YahooFinanceService implements DataProviderInterface { } } + public getMaxNumberOfSymbolsPerRequest() { + return 50; + } + public getName(): DataSource { return DataSource.YAHOO; }