Feature/add caching for quotes (#2095)

* Add caching for quotes

* Update changelog

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added the caching for current market prices
 - Added a loading indicator to the import dividends dialog
 
 ### Changed

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -98,7 +98,8 @@ describe('CurrentRateService', () => {
       [],
       null,
       null,
-      propertyService
+      propertyService,
+      null
     );
     exchangeRateDataService = new ExchangeRateDataService(
       null,

apps/api/src/app/redis-cache/redis-cache.service.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
 import { Cache } from 'cache-manager';
 
@@ -13,6 +14,10 @@ export class RedisCacheService {
     return await this.cache.get(key);
   }
 
+  public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
+    return `quote-${dataSource}-${symbol}`;
+  }
+
   public async remove(key: string) {
     await this.cache.del(key);
   }

apps/api/src/services/configuration/configuration.service.ts

@@ -16,6 +16,7 @@ export class ConfigurationService {
         default: 'USD'
       }),
       BETTER_UPTIME_API_KEY: str({ default: '' }),
+      CACHE_QUOTES_TTL: num({ default: 1 }),
       CACHE_TTL: num({ default: 1 }),
       DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
       DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),

apps/api/src/services/data-provider/data-provider.module.ts

@@ -1,3 +1,4 @@
+import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
 import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
@@ -26,6 +27,7 @@ import { DataProviderService } from './data-provider.service';
     MarketDataModule,
     PrismaModule,
     PropertyModule,
+    RedisCacheModule,
     SymbolProfileModule
   ],
   providers: [

apps/api/src/services/data-provider/data-provider.service.ts

@@ -1,3 +1,4 @@
+import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
@@ -27,7 +28,8 @@ export class DataProviderService {
     private readonly dataProviderInterfaces: DataProviderInterface[],
     private readonly marketDataService: MarketDataService,
     private readonly prismaService: PrismaService,
-    private readonly propertyService: PropertyService
+    private readonly propertyService: PropertyService,
+    private readonly redisCacheService: RedisCacheService
   ) {
     this.initialize();
   }
@@ -235,9 +237,43 @@ export class DataProviderService {
     } = {};
     const startTimeTotal = performance.now();
 
-    const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+    // Get items from cache
+    const itemsToFetch: IDataGatheringItem[] = [];
 
-    const promises = [];
+    for (const { dataSource, symbol } of items) {
+      const quoteString = await this.redisCacheService.get(
+        this.redisCacheService.getQuoteKey({ dataSource, symbol })
+      );
+
+      if (quoteString) {
+        try {
+          const cachedDataProviderResponse = JSON.parse(quoteString);
+          response[symbol] = cachedDataProviderResponse;
+        } catch {}
+      }
+
+      if (!quoteString) {
+        itemsToFetch.push({ dataSource, symbol });
+      }
+    }
+
+    const numberOfItemsInCache = Object.keys(response)?.length;
+
+    if (numberOfItemsInCache) {
+      Logger.debug(
+        `Fetched ${numberOfItemsInCache} quote${
+          numberOfItemsInCache > 1 ? 's' : ''
+        } from cache in ${((performance.now() - startTimeTotal) / 1000).toFixed(
+          3
+        )} seconds`
+      );
+    }
+
+    const itemsGroupedByDataSource = groupBy(itemsToFetch, ({ dataSource }) => {
+      return dataSource;
+    });
+
+    const promises: Promise<any>[] = [];
 
     for (const [dataSource, dataGatheringItems] of Object.entries(
       itemsGroupedByDataSource
@@ -271,6 +307,15 @@ export class DataProviderService {
               result
             )) {
               response[symbol] = dataProviderResponse;
+
+              this.redisCacheService.set(
+                this.redisCacheService.getQuoteKey({
+                  dataSource: DataSource[dataSource],
+                  symbol
+                }),
+                JSON.stringify(dataProviderResponse),
+                this.configurationService.get('CACHE_QUOTES_TTL')
+              );
             }
 
             Logger.debug(
@@ -283,7 +328,7 @@ export class DataProviderService {
             );
 
             try {
-              await this.marketDataService.updateMany({
+              this.marketDataService.updateMany({
                 data: Object.keys(response)
                   .filter((symbol) => {
                     return (

apps/api/src/services/interfaces/environment.interface.ts

@@ -5,6 +5,7 @@ export interface Environment extends CleanedEnvAccessors {
   ALPHA_VANTAGE_API_KEY: string;
   BASE_CURRENCY: string;
   BETTER_UPTIME_API_KEY: string;
+  CACHE_QUOTES_TTL: number;
   CACHE_TTL: number;
   DATA_SOURCE_EXCHANGE_RATES: string;
   DATA_SOURCE_IMPORT: string;