Feature/add data source as unique constraint to market data schema (#1889)

* Add dataSource as unique constraint to MarketData schema

* Update changelog

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added `dataSource` as a unique constraint to the `MarketData` database schema
+
 ### Fixed
 
 - Removed the unnecessary sort header of the comment column in the historical market data table of the admin control panel

apps/api/src/app/admin/admin.controller.ts

@@ -319,7 +319,8 @@ export class AdminController {
     return this.marketDataService.updateMarketData({
       data: { ...data, dataSource },
       where: {
-        date_symbol: {
+        dataSource_date_symbol: {
+          dataSource,
           date,
           symbol
         }

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -91,6 +91,7 @@ describe('CurrentRateService', () => {
       null,
       [],
       null,
+      null,
       propertyService
     );
     exchangeRateDataService = new ExchangeRateDataService(

apps/api/src/services/data-gathering.service.ts

@@ -102,7 +102,7 @@ export class DataGatheringService {
             symbol
           },
           update: { marketPrice },
-          where: { date_symbol: { date, symbol } }
+          where: { dataSource_date_symbol: { dataSource, date, symbol } }
         });
       }
     } catch (error) {

apps/api/src/services/data-provider/data-provider.module.ts

@@ -7,20 +7,22 @@ import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/goog
 import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
 import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';
 import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
 import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { DataEnhancerModule } from './data-enhancer/data-enhancer.module';
 import { YahooFinanceDataEnhancerService } from './data-enhancer/yahoo-finance/yahoo-finance.service';
 import { DataProviderService } from './data-provider.service';
-import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 
 @Module({
   imports: [
     ConfigurationModule,
     CryptocurrencyModule,
     DataEnhancerModule,
+    MarketDataModule,
     PrismaModule,
     PropertyModule,
     SymbolProfileModule

apps/api/src/services/data-provider/data-provider.service.ts

@@ -6,6 +6,7 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UserWithSettings } from '@ghostfolio/common/types';
@@ -25,6 +26,7 @@ export class DataProviderService {
     private readonly configurationService: ConfigurationService,
     @Inject('DataProviderInterfaces')
     private readonly dataProviderInterfaces: DataProviderInterface[],
+    private readonly marketDataService: MarketDataService,
     private readonly prismaService: PrismaService,
     private readonly propertyService: PropertyService
   ) {
@@ -276,35 +278,23 @@ export class DataProviderService {
             );
 
             try {
-              const date = getStartOfUtcDate(new Date());
-
-              // Upsert quotes by imitating missing upsertMany functionality
-              // with $transaction
-              const upsertPromises = Object.keys(response)
-                .filter((symbol) => {
-                  return (
-                    isNumber(response[symbol].marketPrice) &&
-                    response[symbol].marketPrice > 0
-                  );
-                })
-                .map((symbol) =>
-                  this.prismaService.marketData.upsert({
-                    create: {
-                      date,
+              await this.marketDataService.updateMany({
+                data: Object.keys(response)
+                  .filter((symbol) => {
+                    return (
+                      isNumber(response[symbol].marketPrice) &&
+                      response[symbol].marketPrice > 0
+                    );
+                  })
+                  .map((symbol) => {
+                    return {
                       symbol,
                       dataSource: response[symbol].dataSource,
+                      date: getStartOfUtcDate(new Date()),
                       marketPrice: response[symbol].marketPrice
-                    },
-                    update: {
-                      marketPrice: response[symbol].marketPrice
-                    },
-                    where: {
-                      date_symbol: { date, symbol }
-                    }
+                    };
                   })
-                );
-
-              await this.prismaService.$transaction(upsertPromises);
+              });
             } catch {}
           })
         );

apps/api/src/services/market-data.service.ts

@@ -102,11 +102,46 @@ export class MarketDataService {
       where,
       create: {
         dataSource: data.dataSource,
-        date: where.date_symbol.date,
+        date: where.dataSource_date_symbol.date,
         marketPrice: data.marketPrice,
-        symbol: where.date_symbol.symbol
+        symbol: where.dataSource_date_symbol.symbol
       },
       update: { marketPrice: data.marketPrice }
     });
   }
+
+  /**
+   * Upsert market data by imitating missing upsertMany functionality
+   * with $transaction
+   */
+  public async updateMany({
+    data
+  }: {
+    data: Prisma.MarketDataUpdateInput[];
+  }): Promise<MarketData[]> {
+    const upsertPromises = data.map(
+      ({ dataSource, date, marketPrice, symbol }) => {
+        return this.prismaService.marketData.upsert({
+          create: {
+            dataSource: <DataSource>dataSource,
+            date: <Date>date,
+            marketPrice: <number>marketPrice,
+            symbol: <string>symbol
+          },
+          update: {
+            marketPrice: <number>marketPrice
+          },
+          where: {
+            dataSource_date_symbol: {
+              dataSource: <DataSource>dataSource,
+              date: <Date>date,
+              symbol: <string>symbol
+            }
+          }
+        });
+      }
+    );
+
+    return this.prismaService.$transaction(upsertPromises);
+  }
 }

prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql

@@ -0,0 +1,5 @@
+-- DropIndex
+DROP INDEX "MarketData_date_symbol_key";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "MarketData_dataSource_date_symbol_key" ON "MarketData"("dataSource", "date", "symbol");

prisma/schema.prisma

@@ -66,7 +66,7 @@ model MarketData {
   symbol      String
   marketPrice Float
 
-  @@unique([date, symbol])
+  @@unique([dataSource, date, symbol])
   @@index([symbol])
 }