Feature/eliminate redundant storage of historical exchange rates (#500)

* Eliminate redundant storage of historical exchange rates

* Clean up experimental API

* Update changelog

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Removed the experimental API
+
+### Fixed
+
+- Eliminated the redundant storage of historical exchange rates
+
 ## 1.82.0 - 28.11.2021
 
 ### Added

apps/api/src/app/app.module.ts

@@ -19,7 +19,6 @@ import { AdminModule } from './admin/admin.module';
 import { AppController } from './app.controller';
 import { AuthModule } from './auth/auth.module';
 import { CacheModule } from './cache/cache.module';
-import { ExperimentalModule } from './experimental/experimental.module';
 import { ExportModule } from './export/export.module';
 import { ImportModule } from './import/import.module';
 import { InfoModule } from './info/info.module';
@@ -42,7 +41,6 @@ import { UserModule } from './user/user.module';
     DataGatheringModule,
     DataProviderModule,
     ExchangeRateDataModule,
-    ExperimentalModule,
     ExportModule,
     ImportModule,
     InfoModule,

apps/api/src/app/experimental/create-order.dto.ts

@@ -1,22 +0,0 @@
-import { Type } from '@prisma/client';
-import { IsISO8601, IsNumber, IsString } from 'class-validator';
-
-export class CreateOrderDto {
-  @IsString()
-  currency: string;
-
-  @IsISO8601()
-  date: string;
-
-  @IsNumber()
-  quantity: number;
-
-  @IsString()
-  symbol: string;
-
-  @IsString()
-  type: Type;
-
-  @IsNumber()
-  unitPrice: number;
-}

apps/api/src/app/experimental/experimental.controller.ts

@@ -1,69 +0,0 @@
-import { baseCurrency, benchmarks } from '@ghostfolio/common/config';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
-import { isApiTokenAuthorized } from '@ghostfolio/common/permissions';
-import type { RequestWithUser } from '@ghostfolio/common/types';
-import {
-  Body,
-  Controller,
-  Get,
-  Headers,
-  HttpException,
-  Inject,
-  Param,
-  Post
-} from '@nestjs/common';
-import { REQUEST } from '@nestjs/core';
-import { parse } from 'date-fns';
-import { StatusCodes, getReasonPhrase } from 'http-status-codes';
-
-import { CreateOrderDto } from './create-order.dto';
-import { ExperimentalService } from './experimental.service';
-import { Data } from './interfaces/data.interface';
-
-@Controller('experimental')
-export class ExperimentalController {
-  public constructor(
-    private readonly experimentalService: ExperimentalService,
-    @Inject(REQUEST) private readonly request: RequestWithUser
-  ) {}
-
-  @Get('benchmarks')
-  public async getBenchmarks(
-    @Headers('Authorization') apiToken: string
-  ): Promise<string[]> {
-    if (!isApiTokenAuthorized(apiToken)) {
-      throw new HttpException(
-        getReasonPhrase(StatusCodes.FORBIDDEN),
-        StatusCodes.FORBIDDEN
-      );
-    }
-
-    return benchmarks.map(({ symbol }) => {
-      return symbol;
-    });
-  }
-
-  @Get('benchmarks/:symbol')
-  public async getBenchmark(
-    @Headers('Authorization') apiToken: string,
-    @Param('symbol') symbol: string
-  ): Promise<{ date: Date; marketPrice: number }[]> {
-    if (!isApiTokenAuthorized(apiToken)) {
-      throw new HttpException(
-        getReasonPhrase(StatusCodes.FORBIDDEN),
-        StatusCodes.FORBIDDEN
-      );
-    }
-
-    const marketData = await this.experimentalService.getBenchmark(symbol);
-
-    if (marketData?.length === 0) {
-      throw new HttpException(
-        getReasonPhrase(StatusCodes.NOT_FOUND),
-        StatusCodes.NOT_FOUND
-      );
-    }
-
-    return marketData;
-  }
-}

apps/api/src/app/experimental/experimental.module.ts

@@ -1,23 +0,0 @@
-import { AccountService } from '@ghostfolio/api/app/account/account.service';
-import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
-import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
-import { Module } from '@nestjs/common';
-
-import { ExperimentalController } from './experimental.controller';
-import { ExperimentalService } from './experimental.service';
-
-@Module({
-  imports: [
-    ConfigurationModule,
-    DataProviderModule,
-    ExchangeRateDataModule,
-    RedisCacheModule,
-    PrismaModule
-  ],
-  controllers: [ExperimentalController],
-  providers: [AccountService, ExperimentalService]
-})
-export class ExperimentalModule {}

apps/api/src/app/experimental/experimental.service.ts

@@ -1,23 +0,0 @@
-import { AccountService } from '@ghostfolio/api/app/account/account.service';
-import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { Injectable } from '@nestjs/common';
-
-@Injectable()
-export class ExperimentalService {
-  public constructor(
-    private readonly accountService: AccountService,
-    private readonly dataProviderService: DataProviderService,
-    private readonly exchangeRateDataService: ExchangeRateDataService,
-    private readonly prismaService: PrismaService
-  ) {}
-
-  public async getBenchmark(aSymbol: string) {
-    return this.prismaService.marketData.findMany({
-      orderBy: { date: 'asc' },
-      select: { date: true, marketPrice: true },
-      where: { symbol: aSymbol }
-    });
-  }
-}

apps/api/src/app/experimental/interfaces/data.interface.ts

@@ -1,4 +0,0 @@
-export interface Data {
-  currency: string;
-  value: number;
-}

apps/api/src/services/data-gathering.service.ts

@@ -1,8 +1,5 @@
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import {
+import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config';
-  benchmarks,
-  ghostfolioFearAndGreedIndexSymbol
-} from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
@@ -370,13 +367,7 @@ export class DataGatheringService {
   }
 
   private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
-    const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
+    const benchmarksToGather: IDataGatheringItem[] = [];
-      return {
-        dataSource,
-        symbol,
-        date: startDate
-      };
-    });
 
     if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) {
       benchmarksToGather.push({

apps/api/src/services/exchange-rate-data.service.ts

@@ -1,7 +1,6 @@
 import { baseCurrency } from '@ghostfolio/common/config';
 import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
 import { Injectable, Logger } from '@nestjs/common';
-import { DataSource } from '@prisma/client';
 import { format } from 'date-fns';
 import { isEmpty, isNumber, uniq } from 'lodash';
 
@@ -40,7 +39,10 @@ export class ExchangeRateDataService {
       currency2,
       dataSource
     } of this.prepareCurrencyPairs(this.currencies)) {
-      this.addCurrencyPairs({ currency1, currency2, dataSource });
+      this.currencyPairs.push({
+        dataSource,
+        symbol: `${currency1}${currency2}`
+      });
     }
 
     await this.loadCurrencies();
@@ -86,7 +88,7 @@ export class ExchangeRateDataService {
       };
     });
 
-    this.currencyPairs.forEach(({ symbol }) => {
+    Object.keys(resultExtended).forEach((symbol) => {
       const [currency1, currency2] = symbol.match(/.{1,3}/g);
       const date = format(getYesterday(), DATE_FORMAT);
 
@@ -146,25 +148,6 @@ export class ExchangeRateDataService {
     return aValue;
   }
 
-  private addCurrencyPairs({
-    currency1,
-    currency2,
-    dataSource
-  }: {
-    currency1: string;
-    currency2: string;
-    dataSource: DataSource;
-  }) {
-    this.currencyPairs.push({
-      dataSource,
-      symbol: `${currency1}${currency2}`
-    });
-    this.currencyPairs.push({
-      dataSource,
-      symbol: `${currency2}${currency1}`
-    });
-  }
-
   private async prepareCurrencies(): Promise<string[]> {
     const currencies: string[] = [];
 

apps/client/src/app/components/admin-market-data/admin-market-data.html

@@ -6,7 +6,8 @@
           <tr class="mat-header-row">
             <th class="mat-header-cell px-1 py-2 text-right" i18n>#</th>
             <th class="mat-header-cell px-1 py-2" i18n>Symbol</th>
-            <th class="mat-header-cell px-1 py-2" i18n>First transaction</th>
+            <th class="mat-header-cell px-1 py-2" i18n>Data Source</th>
+            <th class="mat-header-cell px-1 py-2" i18n>First Transaction</th>
           </tr>
         </thead>
         <tbody>
@@ -17,6 +18,7 @@
             >
               <td class="mat-cell px-1 py-2 text-right">{{ i + 1 }}</td>
               <td class="mat-cell px-1 py-2">{{ item.symbol }}</td>
+              <td class="mat-cell px-1 py-2">{{ item.dataSource}}</td>
               <td class="mat-cell px-1 py-2">
                 {{ (item.date | date: defaultDateFormat) ?? '' }}
               </td>

libs/common/src/lib/config.ts

@@ -1,12 +1,5 @@
-import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
-import { DataSource } from '@prisma/client';
-
 export const baseCurrency = 'USD';
 
-export const benchmarks: Partial<IDataGatheringItem>[] = [
-  { dataSource: DataSource.YAHOO, symbol: 'VOO' }
-];
-
 export const ghostfolioScraperApiSymbolPrefix = '_GF_';
 export const ghostfolioCashSymbol = `${ghostfolioScraperApiSymbolPrefix}CASH`;
 export const ghostfolioFearAndGreedIndexSymbol = `${ghostfolioScraperApiSymbolPrefix}FEAR_AND_GREED_INDEX`;

libs/common/src/lib/permissions.ts

@@ -1,9 +1,5 @@
 import { Role } from '@prisma/client';
 
-export function isApiTokenAuthorized(aApiToken: string) {
-  return aApiToken === 'Bearer fc804dead6ff45b98da4e5530f6aa3cb';
-}
-
 export const permissions = {
   accessAdminControl: 'accessAdminControl',
   accessFearAndGreedIndex: 'accessFearAndGreedIndex',