diff --git a/CHANGELOG.md b/CHANGELOG.md index 70522fc33..c4887507d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Respected the data source attribute of the transactions model in the data management for historical data + ## 1.8.0 - 24.05.2021 ### Added diff --git a/apps/api/src/app/experimental/experimental.controller.ts b/apps/api/src/app/experimental/experimental.controller.ts index cdddcee4e..f8535ac57 100644 --- a/apps/api/src/app/experimental/experimental.controller.ts +++ b/apps/api/src/app/experimental/experimental.controller.ts @@ -37,7 +37,9 @@ export class ExperimentalController { ); } - return benchmarks; + return benchmarks.map(({ symbol }) => { + return symbol; + }); } @Get('benchmarks/:symbol') diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 5f772375b..b933b054f 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -2,7 +2,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { OrderWithAccount } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { Order, Prisma } from '@prisma/client'; +import { DataSource, Order, Prisma } from '@prisma/client'; import { CacheService } from '../cache/cache.service'; import { RedisCacheService } from '../redis-cache/redis-cache.service'; @@ -53,6 +53,7 @@ export class OrderService { // Gather symbol data of order in the background this.dataGatheringService.gatherSymbols([ { + dataSource: data.dataSource, date: data.date, symbol: data.symbol } @@ -90,6 +91,7 @@ export class OrderService { // Gather symbol data of order in the background this.dataGatheringService.gatherSymbols([ { + dataSource: data.dataSource, date: data.date, symbol: data.symbol } diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index b13f4b88b..e45fae214 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -11,6 +11,7 @@ import { import { DateRange, RequestWithUser } from '@ghostfolio/common/types'; import { Inject, Injectable } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; +import { DataSource } from '@prisma/client'; import { add, format, @@ -289,7 +290,7 @@ export class PortfolioService { if (isEmpty(historicalData)) { historicalData = await this.dataProviderService.getHistoricalRaw( - [aSymbol], + [{ dataSource: DataSource.YAHOO, symbol: aSymbol }], portfolio.getMinDate(), new Date() ); diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index df84076ab..96ba41735 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -5,6 +5,7 @@ import { resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; import { differenceInHours, format, @@ -18,6 +19,7 @@ import { import { ConfigurationService } from './configuration.service'; import { DataProviderService } from './data-provider.service'; import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; +import { IDataGatheringItem } from './interfaces/interfaces'; import { PrismaService } from './prisma.service'; @Injectable() @@ -115,15 +117,13 @@ export class DataGatheringService { } } - public async gatherSymbols( - aSymbolsWithStartDate: { date: Date; symbol: string }[] - ) { + public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { let hasError = false; - for (const { date, symbol } of aSymbolsWithStartDate) { + for (const { dataSource, date, symbol } of aSymbolsWithStartDate) { try { const historicalData = await this.dataProviderService.getHistoricalRaw( - [symbol], + [{ dataSource, symbol }], date, new Date() ); @@ -184,20 +184,24 @@ export class DataGatheringService { } } - public async getCustomSymbolsToGather(startDate?: Date) { + public async getCustomSymbolsToGather( + startDate?: Date + ): Promise { const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations(); return scraperConfigurations.map((scraperConfiguration) => { return { + dataSource: DataSource.GHOSTFOLIO, date: startDate, symbol: scraperConfiguration.symbol }; }); } - private getBenchmarksToGather(startDate: Date) { - const benchmarksToGather = benchmarks.map((symbol) => { + private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] { + const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => { return { + dataSource, symbol, date: startDate }; @@ -205,6 +209,7 @@ export class DataGatheringService { if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) { benchmarksToGather.push({ + dataSource: DataSource.RAKUTEN, date: startDate, symbol: 'GF.FEAR_AND_GREED_INDEX' }); @@ -213,16 +218,16 @@ export class DataGatheringService { return benchmarksToGather; } - private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> { + private async getSymbols7D(): Promise { const startDate = subDays(resetHours(new Date()), 7); const distinctOrders = await this.prisma.order.findMany({ distinct: ['symbol'], orderBy: [{ symbol: 'asc' }], - select: { symbol: true } + select: { dataSource: true, symbol: true } }); - const distinctOrdersWithDate = distinctOrders + const distinctOrdersWithDate: IDataGatheringItem[] = distinctOrders .filter((distinctOrder) => { return !isGhostfolioScraperApiSymbol(distinctOrder.symbol); }) @@ -233,12 +238,15 @@ export class DataGatheringService { }; }); - const currencyPairsToGather = currencyPairs.map((symbol) => { - return { - symbol, - date: startDate - }; - }); + const currencyPairsToGather = currencyPairs.map( + ({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: startDate + }; + } + ); const customSymbolsToGather = await this.getCustomSymbolsToGather( startDate @@ -252,24 +260,27 @@ export class DataGatheringService { ]; } - private async getSymbolsMax() { + private async getSymbolsMax(): Promise { const startDate = new Date(getUtc('2015-01-01')); const customSymbolsToGather = await this.getCustomSymbolsToGather( startDate ); - const currencyPairsToGather = currencyPairs.map((symbol) => { - return { - symbol, - date: startDate - }; - }); + const currencyPairsToGather = currencyPairs.map( + ({ dataSource, symbol }) => { + return { + dataSource, + symbol, + date: startDate + }; + } + ); const distinctOrders = await this.prisma.order.findMany({ distinct: ['symbol'], orderBy: [{ date: 'asc' }], - select: { date: true, symbol: true } + select: { dataSource: true, date: true, symbol: true } }); return [ diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index 2183a514f..42b47fac5 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -1,6 +1,4 @@ -import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { - isCrypto, isGhostfolioScraperApiSymbol, isRakutenRapidApiSymbol } from '@ghostfolio/common/helper'; @@ -16,6 +14,7 @@ import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakute import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service'; import { DataProviderInterface } from './interfaces/data-provider.interface'; import { + IDataGatheringItem, IDataProviderHistoricalResponse, IDataProviderResponse } from './interfaces/interfaces'; @@ -121,79 +120,53 @@ export class DataProviderService implements DataProviderInterface { } public async getHistoricalRaw( - aSymbols: string[], + aDataGatheringItems: IDataGatheringItem[], from: Date, to: Date ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { - const filteredSymbols = aSymbols.filter((symbol) => { - return !isGhostfolioScraperApiSymbol(symbol); - }); - - const dataOfYahoo = await this.yahooFinanceService.getHistorical( - filteredSymbols, - undefined, - from, - to - ); - - if (aSymbols.length === 1) { - const symbol = aSymbols[0]; - - if ( - isCrypto(symbol) && - this.configurationService.get('ALPHA_VANTAGE_API_KEY') - ) { - // Merge data from Yahoo with data from Alpha Vantage - const dataOfAlphaVantage = await this.alphaVantageService.getHistorical( - [symbol], - undefined, - from, - to - ); - - return { - [symbol]: { - ...dataOfYahoo[symbol], - ...dataOfAlphaVantage[symbol] - } - }; - } else if (isGhostfolioScraperApiSymbol(symbol)) { - const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical( - [symbol], - undefined, - from, - to - ); + const result: { + [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; + } = {}; - return dataOfGhostfolioScraperApi; - } else if ( - isRakutenRapidApiSymbol(symbol) && - this.configurationService.get('RAKUTEN_RAPID_API_KEY') - ) { - const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical( - [symbol], - undefined, - from, - to + const promises: Promise<{ + data: { [date: string]: IDataProviderHistoricalResponse }; + symbol: string; + }>[] = []; + for (const { dataSource, symbol } of aDataGatheringItems) { + const dataProvider = this.getDataProvider(dataSource); + if (dataProvider.canHandle(symbol)) { + promises.push( + dataProvider + .getHistorical([symbol], undefined, from, to) + .then((data) => ({ data: data?.[symbol], symbol })) ); - - return dataOfRakutenRapidApi; } } - return dataOfYahoo; + const allData = await Promise.all(promises); + for (const { data, symbol } of allData) { + result[symbol] = data; + } + + return result; } public async search(aSymbol: string) { - return this.getDataProvider().search(aSymbol); + return this.getDataProvider( + this.configurationService.get('DATA_SOURCES')[0] + ).search(aSymbol); } - private getDataProvider() { - switch (this.configurationService.get('DATA_SOURCES')[0]) { + private getDataProvider(providerName: DataSource) { + switch (providerName) { case DataSource.ALPHA_VANTAGE: return this.alphaVantageService; + case DataSource.GHOSTFOLIO: + return this.ghostfolioScraperApiService; + case DataSource.RAKUTEN: + return this.rakutenRapidApiService; case DataSource.YAHOO: return this.yahooFinanceService; default: diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 3046737e5..64e0b6c99 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -24,6 +24,10 @@ export class AlphaVantageService implements DataProviderInterface { }); } + public canHandle(symbol: string) { + return this.configurationService.get('ALPHA_VANTAGE_API_KEY'); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index b6423f4a2..8b8e4290d 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -1,4 +1,7 @@ -import { getYesterday } from '@ghostfolio/common/helper'; +import { + getYesterday, + isGhostfolioScraperApiSymbol +} from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; @@ -21,6 +24,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface { public constructor(private prisma: PrismaService) {} + public canHandle(symbol: string) { + return isGhostfolioScraperApiSymbol(symbol); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index 9f3943a8e..7af775f8b 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -1,4 +1,8 @@ -import { getToday, getYesterday } from '@ghostfolio/common/helper'; +import { + getToday, + getYesterday, + isRakutenRapidApiSymbol +} from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; @@ -24,6 +28,13 @@ export class RakutenRapidApiService implements DataProviderInterface { private readonly configurationService: ConfigurationService ) {} + public canHandle(symbol: string) { + return ( + isRakutenRapidApiSymbol(symbol) && + this.configurationService.get('RAKUTEN_RAPID_API_KEY') + ); + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 19a969cbd..b22d7e7cf 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -28,6 +28,10 @@ export class YahooFinanceService implements DataProviderInterface { public constructor() {} + public canHandle(symbol: string) { + return true; + } + public async get( aSymbols: string[] ): Promise<{ [symbol: string]: IDataProviderResponse }> { diff --git a/apps/api/src/services/interfaces/data-provider.interface.ts b/apps/api/src/services/interfaces/data-provider.interface.ts index cf861ad7b..b36be4cb4 100644 --- a/apps/api/src/services/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/interfaces/data-provider.interface.ts @@ -7,6 +7,8 @@ import { } from './interfaces'; export interface DataProviderInterface { + canHandle(symbol: string): boolean; + get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>; getHistorical( diff --git a/apps/api/src/services/interfaces/interfaces.ts b/apps/api/src/services/interfaces/interfaces.ts index 82f1381c3..bea217384 100644 --- a/apps/api/src/services/interfaces/interfaces.ts +++ b/apps/api/src/services/interfaces/interfaces.ts @@ -65,6 +65,12 @@ export interface IDataProviderResponse { url?: string; } +export interface IDataGatheringItem { + dataSource: DataSource; + date?: Date; + symbol: string; +} + export type Industry = typeof Industry[keyof typeof Industry]; export type MarketState = typeof MarketState[keyof typeof MarketState]; diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 66191843d..1383d08f6 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -1,13 +1,17 @@ +import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { Currency } from '@prisma/client'; +import { DataSource } from '@prisma/client'; export const baseCurrency = Currency.CHF; -export const benchmarks = ['VOO']; +export const benchmarks: Partial[] = [ + { dataSource: DataSource.YAHOO, symbol: 'VOO' } +]; -export const currencyPairs = [ - `${Currency.USD}${Currency.EUR}`, - `${Currency.USD}${Currency.GBP}`, - `${Currency.USD}${Currency.CHF}` +export const currencyPairs: Partial[] = [ + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.EUR}` }, + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.GBP}` }, + { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.CHF}` } ]; export const ghostfolioScraperApiSymbolPrefix = '_GF_';