Feature/respect data source in data gathering (#107)

* Respect data source in data gathering

* Update changelog

* optimize fetching from multiple data sources (#123)

* optimize fetching from multiple data sources

* improve performance by executing data gathering promises in parallel

* removed unused imports

* rename hasHistoricalData to canHandle

* Sort imports

* Clean up

Co-authored-by: Valentin Zickner <3200232+vzickner@users.noreply.github.com>

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Respected the data source attribute of the transactions model in the data management for historical data
+
 ## 1.8.0 - 24.05.2021
 
 ### Added

apps/api/src/app/experimental/experimental.controller.ts

@@ -37,7 +37,9 @@ export class ExperimentalController {
       );
     }
 
-    return benchmarks;
+    return benchmarks.map(({ symbol }) => {
+      return symbol;
+    });
   }
 
   @Get('benchmarks/:symbol')

apps/api/src/app/order/order.service.ts

@@ -2,7 +2,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { OrderWithAccount } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
-import { Order, Prisma } from '@prisma/client';
+import { DataSource, Order, Prisma } from '@prisma/client';
 
 import { CacheService } from '../cache/cache.service';
 import { RedisCacheService } from '../redis-cache/redis-cache.service';
@@ -53,6 +53,7 @@ export class OrderService {
     // Gather symbol data of order in the background
     this.dataGatheringService.gatherSymbols([
       {
+        dataSource: data.dataSource,
         date: <Date>data.date,
         symbol: data.symbol
       }
@@ -90,6 +91,7 @@ export class OrderService {
     // Gather symbol data of order in the background
     this.dataGatheringService.gatherSymbols([
       {
+        dataSource: <DataSource>data.dataSource,
         date: <Date>data.date,
         symbol: <string>data.symbol
       }

apps/api/src/app/portfolio/portfolio.service.ts

@@ -11,6 +11,7 @@ import {
 import { DateRange, RequestWithUser } from '@ghostfolio/common/types';
 import { Inject, Injectable } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
+import { DataSource } from '@prisma/client';
 import {
   add,
   format,
@@ -289,7 +290,7 @@ export class PortfolioService {
 
       if (isEmpty(historicalData)) {
         historicalData = await this.dataProviderService.getHistoricalRaw(
-          [aSymbol],
+          [{ dataSource: DataSource.YAHOO, symbol: aSymbol }],
           portfolio.getMinDate(),
           new Date()
         );

apps/api/src/services/data-gathering.service.ts

@@ -5,6 +5,7 @@ import {
   resetHours
 } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
 import {
   differenceInHours,
   format,
@@ -18,6 +19,7 @@ import {
 import { ConfigurationService } from './configuration.service';
 import { DataProviderService } from './data-provider.service';
 import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { IDataGatheringItem } from './interfaces/interfaces';
 import { PrismaService } from './prisma.service';
 
 @Injectable()
@@ -115,15 +117,13 @@ export class DataGatheringService {
     }
   }
 
-  public async gatherSymbols(
-    aSymbolsWithStartDate: { date: Date; symbol: string }[]
-  ) {
+  public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
     let hasError = false;
 
-    for (const { date, symbol } of aSymbolsWithStartDate) {
+    for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
       try {
         const historicalData = await this.dataProviderService.getHistoricalRaw(
-          [symbol],
+          [{ dataSource, symbol }],
           date,
           new Date()
         );
@@ -184,20 +184,24 @@ export class DataGatheringService {
     }
   }
 
-  public async getCustomSymbolsToGather(startDate?: Date) {
+  public async getCustomSymbolsToGather(
+    startDate?: Date
+  ): Promise<IDataGatheringItem[]> {
     const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations();
 
     return scraperConfigurations.map((scraperConfiguration) => {
       return {
+        dataSource: DataSource.GHOSTFOLIO,
         date: startDate,
         symbol: scraperConfiguration.symbol
       };
     });
   }
 
-  private getBenchmarksToGather(startDate: Date) {
-    const benchmarksToGather = benchmarks.map((symbol) => {
+  private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
+    const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
       return {
+        dataSource,
         symbol,
         date: startDate
       };
@@ -205,6 +209,7 @@ export class DataGatheringService {
 
     if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) {
       benchmarksToGather.push({
+        dataSource: DataSource.RAKUTEN,
         date: startDate,
         symbol: 'GF.FEAR_AND_GREED_INDEX'
       });
@@ -213,16 +218,16 @@ export class DataGatheringService {
     return benchmarksToGather;
   }
 
-  private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
+  private async getSymbols7D(): Promise<IDataGatheringItem[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
     const distinctOrders = await this.prisma.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ symbol: 'asc' }],
-      select: { symbol: true }
+      select: { dataSource: true, symbol: true }
     });
 
-    const distinctOrdersWithDate = distinctOrders
+    const distinctOrdersWithDate: IDataGatheringItem[] = distinctOrders
       .filter((distinctOrder) => {
         return !isGhostfolioScraperApiSymbol(distinctOrder.symbol);
       })
@@ -233,12 +238,15 @@ export class DataGatheringService {
         };
       });
 
-    const currencyPairsToGather = currencyPairs.map((symbol) => {
+    const currencyPairsToGather = currencyPairs.map(
+      ({ dataSource, symbol }) => {
         return {
+          dataSource,
           symbol,
           date: startDate
         };
-    });
+      }
+    );
 
     const customSymbolsToGather = await this.getCustomSymbolsToGather(
       startDate
@@ -252,24 +260,27 @@ export class DataGatheringService {
     ];
   }
 
-  private async getSymbolsMax() {
+  private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
     const startDate = new Date(getUtc('2015-01-01'));
 
     const customSymbolsToGather = await this.getCustomSymbolsToGather(
       startDate
     );
 
-    const currencyPairsToGather = currencyPairs.map((symbol) => {
+    const currencyPairsToGather = currencyPairs.map(
+      ({ dataSource, symbol }) => {
         return {
+          dataSource,
           symbol,
           date: startDate
         };
-    });
+      }
+    );
 
     const distinctOrders = await this.prisma.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ date: 'asc' }],
-      select: { date: true, symbol: true }
+      select: { dataSource: true, date: true, symbol: true }
     });
 
     return [

apps/api/src/services/data-provider.service.ts

@@ -1,6 +1,4 @@
-import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import {
-  isCrypto,
   isGhostfolioScraperApiSymbol,
   isRakutenRapidApiSymbol
 } from '@ghostfolio/common/helper';
@@ -16,6 +14,7 @@ import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakute
 import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
 import { DataProviderInterface } from './interfaces/data-provider.interface';
 import {
+  IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from './interfaces/interfaces';
@@ -121,79 +120,53 @@ export class DataProviderService implements DataProviderInterface {
   }
 
   public async getHistoricalRaw(
-    aSymbols: string[],
+    aDataGatheringItems: IDataGatheringItem[],
     from: Date,
     to: Date
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
-    const filteredSymbols = aSymbols.filter((symbol) => {
-      return !isGhostfolioScraperApiSymbol(symbol);
-    });
+    const result: {
+      [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+    } = {};
 
-    const dataOfYahoo = await this.yahooFinanceService.getHistorical(
-      filteredSymbols,
-      undefined,
-      from,
-      to
+    const promises: Promise<{
+      data: { [date: string]: IDataProviderHistoricalResponse };
+      symbol: string;
+    }>[] = [];
+    for (const { dataSource, symbol } of aDataGatheringItems) {
+      const dataProvider = this.getDataProvider(dataSource);
+      if (dataProvider.canHandle(symbol)) {
+        promises.push(
+          dataProvider
+            .getHistorical([symbol], undefined, from, to)
+            .then((data) => ({ data: data?.[symbol], symbol }))
         );
-
-    if (aSymbols.length === 1) {
-      const symbol = aSymbols[0];
-
-      if (
-        isCrypto(symbol) &&
-        this.configurationService.get('ALPHA_VANTAGE_API_KEY')
-      ) {
-        // Merge data from Yahoo with data from Alpha Vantage
-        const dataOfAlphaVantage = await this.alphaVantageService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
-
-        return {
-          [symbol]: {
-            ...dataOfYahoo[symbol],
-            ...dataOfAlphaVantage[symbol]
       }
-        };
-      } else if (isGhostfolioScraperApiSymbol(symbol)) {
-        const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
-
-        return dataOfGhostfolioScraperApi;
-      } else if (
-        isRakutenRapidApiSymbol(symbol) &&
-        this.configurationService.get('RAKUTEN_RAPID_API_KEY')
-      ) {
-        const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical(
-          [symbol],
-          undefined,
-          from,
-          to
-        );
-
-        return dataOfRakutenRapidApi;
     }
+
+    const allData = await Promise.all(promises);
+    for (const { data, symbol } of allData) {
+      result[symbol] = data;
     }
 
-    return dataOfYahoo;
+    return result;
   }
 
   public async search(aSymbol: string) {
-    return this.getDataProvider().search(aSymbol);
+    return this.getDataProvider(
+      this.configurationService.get('DATA_SOURCES')[0]
+    ).search(aSymbol);
   }
 
-  private getDataProvider() {
-    switch (this.configurationService.get('DATA_SOURCES')[0]) {
+  private getDataProvider(providerName: DataSource) {
+    switch (providerName) {
       case DataSource.ALPHA_VANTAGE:
         return this.alphaVantageService;
+      case DataSource.GHOSTFOLIO:
+        return this.ghostfolioScraperApiService;
+      case DataSource.RAKUTEN:
+        return this.rakutenRapidApiService;
       case DataSource.YAHOO:
         return this.yahooFinanceService;
       default:

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -24,6 +24,10 @@ export class AlphaVantageService implements DataProviderInterface {
     });
   }
 
+  public canHandle(symbol: string) {
+    return this.configurationService.get('ALPHA_VANTAGE_API_KEY');
+  }
+
   public async get(
     aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -1,4 +1,7 @@
-import { getYesterday } from '@ghostfolio/common/helper';
+import {
+  getYesterday,
+  isGhostfolioScraperApiSymbol
+} from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
@@ -21,6 +24,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
 
   public constructor(private prisma: PrismaService) {}
 
+  public canHandle(symbol: string) {
+    return isGhostfolioScraperApiSymbol(symbol);
+  }
+
   public async get(
     aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {

apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts

@@ -1,4 +1,8 @@
-import { getToday, getYesterday } from '@ghostfolio/common/helper';
+import {
+  getToday,
+  getYesterday,
+  isRakutenRapidApiSymbol
+} from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
@@ -24,6 +28,13 @@ export class RakutenRapidApiService implements DataProviderInterface {
     private readonly configurationService: ConfigurationService
   ) {}
 
+  public canHandle(symbol: string) {
+    return (
+      isRakutenRapidApiSymbol(symbol) &&
+      this.configurationService.get('RAKUTEN_RAPID_API_KEY')
+    );
+  }
+
   public async get(
     aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -28,6 +28,10 @@ export class YahooFinanceService implements DataProviderInterface {
 
   public constructor() {}
 
+  public canHandle(symbol: string) {
+    return true;
+  }
+
   public async get(
     aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {

apps/api/src/services/interfaces/data-provider.interface.ts

@@ -7,6 +7,8 @@ import {
 } from './interfaces';
 
 export interface DataProviderInterface {
+  canHandle(symbol: string): boolean;
+
   get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>;
 
   getHistorical(

apps/api/src/services/interfaces/interfaces.ts

@@ -65,6 +65,12 @@ export interface IDataProviderResponse {
   url?: string;
 }
 
+export interface IDataGatheringItem {
+  dataSource: DataSource;
+  date?: Date;
+  symbol: string;
+}
+
 export type Industry = typeof Industry[keyof typeof Industry];
 
 export type MarketState = typeof MarketState[keyof typeof MarketState];

libs/common/src/lib/config.ts

@@ -1,13 +1,17 @@
+import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { Currency } from '@prisma/client';
+import { DataSource } from '@prisma/client';
 
 export const baseCurrency = Currency.CHF;
 
-export const benchmarks = ['VOO'];
+export const benchmarks: Partial<IDataGatheringItem>[] = [
+  { dataSource: DataSource.YAHOO, symbol: 'VOO' }
+];
 
-export const currencyPairs = [
-  `${Currency.USD}${Currency.EUR}`,
-  `${Currency.USD}${Currency.GBP}`,
-  `${Currency.USD}${Currency.CHF}`
+export const currencyPairs: Partial<IDataGatheringItem>[] = [
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.EUR}` },
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.GBP}` },
+  { dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.CHF}` }
 ];
 
 export const ghostfolioScraperApiSymbolPrefix = '_GF_';