diff --git a/CHANGELOG.md b/CHANGELOG.md index b3717916d..54b752d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Moved the data enhancer calls from the data provider (`get()`) to the data gathering service to reduce traffic to 3rd party data providers +- Changed the profile data gathering from every 12 hours to once every weekend + ## 1.64.0 - 21.10.2021 ### Added diff --git a/apps/api/src/app/cache/cache.module.ts b/apps/api/src/app/cache/cache.module.ts index 663d9926b..b346ee444 100644 --- a/apps/api/src/app/cache/cache.module.ts +++ b/apps/api/src/app/cache/cache.module.ts @@ -1,6 +1,7 @@ import { CacheService } from '@ghostfolio/api/app/cache/cache.service'; import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module'; @@ -10,7 +11,12 @@ import { Module } from '@nestjs/common'; import { CacheController } from './cache.controller'; @Module({ - imports: [DataProviderModule, ExchangeRateDataModule, RedisCacheModule], + imports: [ + DataGatheringModule, + DataProviderModule, + ExchangeRateDataModule, + RedisCacheModule + ], controllers: [CacheController], providers: [ CacheService, diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.ts index d2fb4ff38..0db322a85 100644 --- a/apps/api/src/app/info/info.module.ts +++ b/apps/api/src/app/info/info.module.ts @@ -1,4 +1,5 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module'; @@ -11,6 +12,7 @@ import { InfoService } from './info.service'; @Module({ imports: [ + DataGatheringModule, DataProviderModule, ExchangeRateDataModule, JwtModule.register({ diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 5521faf04..1f3f89309 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -72,7 +72,7 @@ describe('CurrentRateService', () => { let marketDataService: MarketDataService; beforeAll(async () => { - dataProviderService = new DataProviderService(null, [], [], null); + dataProviderService = new DataProviderService(null, [], null); exchangeRateDataService = new ExchangeRateDataService(null, null); marketDataService = new MarketDataService(null); diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index 01ee5444d..be29d5a7b 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -18,7 +18,11 @@ export class CronService { @Cron(CronExpression.EVERY_12_HOURS) public async runEveryTwelveHours() { - await this.dataGatheringService.gatherProfileData(); await this.exchangeRateDataService.loadCurrencies(); } + + @Cron(CronExpression.EVERY_WEEKEND) + public async runEveryWeekend() { + await this.dataGatheringService.gatherProfileData(); + } } diff --git a/apps/api/src/services/data-gathering.module.ts b/apps/api/src/services/data-gathering.module.ts index adf48a33b..85f2f4983 100644 --- a/apps/api/src/services/data-gathering.module.ts +++ b/apps/api/src/services/data-gathering.module.ts @@ -1,5 +1,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; +import { DataEnhancerModule } from '@ghostfolio/api/services/data-provider/data-enhancer/data-enhancer.module'; import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; import { Module } from '@nestjs/common'; @@ -9,11 +10,12 @@ import { ExchangeRateDataModule } from './exchange-rate-data.module'; @Module({ imports: [ ConfigurationModule, + DataEnhancerModule, DataProviderModule, ExchangeRateDataModule, PrismaModule ], providers: [DataGatheringService], - exports: [DataGatheringService] + exports: [DataEnhancerModule, DataGatheringService] }) export class DataGatheringModule {} diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 35a9a4e63..65e08fcf5 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -3,7 +3,7 @@ import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config'; import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; import { differenceInHours, @@ -18,6 +18,7 @@ import { import { ConfigurationService } from './configuration.service'; import { DataProviderService } from './data-provider/data-provider.service'; import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; +import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface'; import { ExchangeRateDataService } from './exchange-rate-data.service'; import { IDataGatheringItem } from './interfaces/interfaces'; import { PrismaService } from './prisma.service'; @@ -26,6 +27,8 @@ import { PrismaService } from './prisma.service'; export class DataGatheringService { public constructor( private readonly configurationService: ConfigurationService, + @Inject('DataEnhancers') + private readonly dataEnhancers: DataEnhancerInterface[], private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, private readonly ghostfolioScraperApi: GhostfolioScraperApiService, @@ -130,9 +133,19 @@ export class DataGatheringService { const currentData = await this.dataProviderService.get(dataGatheringItems); - for (const [ - symbol, - { + for (const [symbol, response] of Object.entries(currentData)) { + for (const dataEnhancer of this.dataEnhancers) { + try { + currentData[symbol] = await dataEnhancer.enhance({ + response, + symbol + }); + } catch (error) { + console.error(`Failed to enhance data for symbol ${symbol}`, error); + } + } + + const { assetClass, assetSubClass, countries, @@ -140,8 +153,8 @@ export class DataGatheringService { dataSource, name, sectors - } - ] of Object.entries(currentData)) { + } = currentData[symbol]; + try { await this.prismaService.symbolProfile.upsert({ create: { diff --git a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts new file mode 100644 index 000000000..9d4c0704d --- /dev/null +++ b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts @@ -0,0 +1,15 @@ +import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service'; +import { Module } from '@nestjs/common'; + +@Module({ + exports: ['DataEnhancers', TrackinsightDataEnhancerService], + providers: [ + { + inject: [TrackinsightDataEnhancerService], + provide: 'DataEnhancers', + useFactory: (trackinsight) => [trackinsight] + }, + TrackinsightDataEnhancerService + ] +}) +export class DataEnhancerModule {} diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 234d4d694..cacf2bf80 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -1,6 +1,5 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; -import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; @@ -17,13 +16,7 @@ import { DataProviderService } from './data-provider.service'; DataProviderService, GhostfolioScraperApiService, RakutenRapidApiService, - TrackinsightDataEnhancerService, YahooFinanceService, - { - inject: [TrackinsightDataEnhancerService], - provide: 'DataEnhancers', - useFactory: (trackinsight) => [trackinsight] - }, { inject: [ AlphaVantageService, diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index ff975251a..b70b8dd59 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -1,6 +1,5 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; -import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; import { IDataGatheringItem, @@ -19,8 +18,6 @@ import { isEmpty } from 'lodash'; export class DataProviderService { public constructor( private readonly configurationService: ConfigurationService, - @Inject('DataEnhancers') - private readonly dataEnhancers: DataEnhancerInterface[], @Inject('DataProviderInterfaces') private readonly dataProviderInterfaces: DataProviderInterface[], private readonly prismaService: PrismaService @@ -42,20 +39,7 @@ export class DataProviderService { const promises = []; for (const symbol of Object.keys(response)) { - let promise = Promise.resolve(response[symbol]); - for (const dataEnhancer of this.dataEnhancers) { - promise = promise.then((currentResponse) => - dataEnhancer - .enhance({ symbol, response: currentResponse }) - .catch((error) => { - console.error( - `Failed to enhance data for symbol ${symbol}`, - error - ); - return currentResponse; - }) - ); - } + const promise = Promise.resolve(response[symbol]); promises.push( promise.then((currentResponse) => (response[symbol] = currentResponse)) );