From cddea0401f7da19c9f77787b2cafa214d17f6a31 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 23 Apr 2023 11:13:08 +0200 Subject: [PATCH] Feature/add data source as unique constraint to market data schema (#1889) * Add dataSource as unique constraint to MarketData schema * Update changelog --- CHANGELOG.md | 4 ++ apps/api/src/app/admin/admin.controller.ts | 3 +- .../portfolio/current-rate.service.spec.ts | 1 + .../src/services/data-gathering.service.ts | 2 +- .../data-provider/data-provider.module.ts | 4 +- .../data-provider/data-provider.service.ts | 40 +++++++------------ apps/api/src/services/market-data.service.ts | 39 +++++++++++++++++- .../migration.sql | 5 +++ prisma/schema.prisma | 2 +- 9 files changed, 69 insertions(+), 31 deletions(-) create mode 100644 prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index b6238cf3d..b727ca33d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `dataSource` as a unique constraint to the `MarketData` database schema + ### Fixed - Removed the unnecessary sort header of the comment column in the historical market data table of the admin control panel diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index bcd96a2f1..81fd25308 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -319,7 +319,8 @@ export class AdminController { return this.marketDataService.updateMarketData({ data: { ...data, dataSource }, where: { - date_symbol: { + dataSource_date_symbol: { + dataSource, date, symbol } diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index a4eb10156..f0e4ea4a9 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -91,6 +91,7 @@ describe('CurrentRateService', () => { null, [], null, + null, propertyService ); exchangeRateDataService = new ExchangeRateDataService( diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 5de70c925..e4f8c0b38 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -102,7 +102,7 @@ export class DataGatheringService { symbol }, update: { marketPrice }, - where: { date_symbol: { date, symbol } } + where: { dataSource_date_symbol: { dataSource, date, symbol } } }); } } catch (error) { diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index cfcd70d6b..b27d5faed 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -7,20 +7,22 @@ import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/goog import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; +import { MarketDataModule } from '@ghostfolio/api/services/market-data.module'; import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; +import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module'; import { Module } from '@nestjs/common'; import { DataEnhancerModule } from './data-enhancer/data-enhancer.module'; import { YahooFinanceDataEnhancerService } from './data-enhancer/yahoo-finance/yahoo-finance.service'; import { DataProviderService } from './data-provider.service'; -import { PropertyModule } from '@ghostfolio/api/services/property/property.module'; @Module({ imports: [ ConfigurationModule, CryptocurrencyModule, DataEnhancerModule, + MarketDataModule, PrismaModule, PropertyModule, SymbolProfileModule diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 2076fafd6..0823af704 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -6,6 +6,7 @@ import { IDataProviderHistoricalResponse, IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; +import { MarketDataService } from '@ghostfolio/api/services/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { UserWithSettings } from '@ghostfolio/common/types'; @@ -25,6 +26,7 @@ export class DataProviderService { private readonly configurationService: ConfigurationService, @Inject('DataProviderInterfaces') private readonly dataProviderInterfaces: DataProviderInterface[], + private readonly marketDataService: MarketDataService, private readonly prismaService: PrismaService, private readonly propertyService: PropertyService ) { @@ -276,35 +278,23 @@ export class DataProviderService { ); try { - const date = getStartOfUtcDate(new Date()); - - // Upsert quotes by imitating missing upsertMany functionality - // with $transaction - const upsertPromises = Object.keys(response) - .filter((symbol) => { - return ( - isNumber(response[symbol].marketPrice) && - response[symbol].marketPrice > 0 - ); - }) - .map((symbol) => - this.prismaService.marketData.upsert({ - create: { - date, + await this.marketDataService.updateMany({ + data: Object.keys(response) + .filter((symbol) => { + return ( + isNumber(response[symbol].marketPrice) && + response[symbol].marketPrice > 0 + ); + }) + .map((symbol) => { + return { symbol, dataSource: response[symbol].dataSource, + date: getStartOfUtcDate(new Date()), marketPrice: response[symbol].marketPrice - }, - update: { - marketPrice: response[symbol].marketPrice - }, - where: { - date_symbol: { date, symbol } - } + }; }) - ); - - await this.prismaService.$transaction(upsertPromises); + }); } catch {} }) ); diff --git a/apps/api/src/services/market-data.service.ts b/apps/api/src/services/market-data.service.ts index fbea8e104..ced802a3f 100644 --- a/apps/api/src/services/market-data.service.ts +++ b/apps/api/src/services/market-data.service.ts @@ -102,11 +102,46 @@ export class MarketDataService { where, create: { dataSource: data.dataSource, - date: where.date_symbol.date, + date: where.dataSource_date_symbol.date, marketPrice: data.marketPrice, - symbol: where.date_symbol.symbol + symbol: where.dataSource_date_symbol.symbol }, update: { marketPrice: data.marketPrice } }); } + + /** + * Upsert market data by imitating missing upsertMany functionality + * with $transaction + */ + public async updateMany({ + data + }: { + data: Prisma.MarketDataUpdateInput[]; + }): Promise { + const upsertPromises = data.map( + ({ dataSource, date, marketPrice, symbol }) => { + return this.prismaService.marketData.upsert({ + create: { + dataSource: dataSource, + date: date, + marketPrice: marketPrice, + symbol: symbol + }, + update: { + marketPrice: marketPrice + }, + where: { + dataSource_date_symbol: { + dataSource: dataSource, + date: date, + symbol: symbol + } + } + }); + } + ); + + return this.prismaService.$transaction(upsertPromises); + } } diff --git a/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql new file mode 100644 index 000000000..7d4210bae --- /dev/null +++ b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql @@ -0,0 +1,5 @@ +-- DropIndex +DROP INDEX "MarketData_date_symbol_key"; + +-- CreateIndex +CREATE UNIQUE INDEX "MarketData_dataSource_date_symbol_key" ON "MarketData"("dataSource", "date", "symbol"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index dd46e52b6..6e099d5ee 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -66,7 +66,7 @@ model MarketData { symbol String marketPrice Float - @@unique([date, symbol]) + @@unique([dataSource, date, symbol]) @@index([symbol]) }