Feature/add state to market data database schema (#1893)

* Add state (CLOSE / INTRADAY) to MarketData

* Update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Introduced a new button to delete all activities from the portfolio activities page
+- Added `state` to the `MarketData` database schema to distinguish `CLOSE` and `INTRADAY` in the data gathering
 
 ## 1.260.0 - 2023-04-23
 

apps/api/src/app/admin/admin.controller.ts

@@ -317,7 +317,7 @@ export class AdminController {
     const date = new Date(dateString);
 
     return this.marketDataService.updateMarketData({
-      data: { ...data, dataSource },
+      data: { marketPrice: data.marketPrice, state: 'CLOSE' },
       where: {
         dataSource_date_symbol: {
           dataSource,

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -18,7 +18,8 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
             createdAt: date,
             dataSource: DataSource.YAHOO,
             id: 'aefcbe3a-ee10-4c4f-9f2d-8ffad7b05584',
-            marketPrice: 1847.839966
+            marketPrice: 1847.839966,
+            state: 'CLOSE'
           });
         },
         getRange: ({
@@ -37,6 +38,7 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
               date: dateRangeStart,
               id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d',
               marketPrice: 1841.823902,
+              state: 'CLOSE',
               symbol: symbols[0]
             },
             {
@@ -45,6 +47,7 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
               date: dateRangeEnd,
               id: '082d6893-df27-4c91-8a5d-092e84315b56',
               marketPrice: 1847.839966,
+              state: 'CLOSE',
               symbol: symbols[0]
             }
           ]);

apps/api/src/services/data-gathering/data-gathering.processor.ts

@@ -6,8 +6,9 @@ import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { Process, Processor } from '@nestjs/bull';
 import { Injectable, Logger } from '@nestjs/common';
 import { Job } from 'bull';
@@ -28,7 +29,7 @@ export class DataGatheringProcessor {
   public constructor(
     private readonly dataGatheringService: DataGatheringService,
     private readonly dataProviderService: DataProviderService,
-    private readonly prismaService: PrismaService
+    private readonly marketDataService: MarketDataService
   ) {}
 
   @Process(GATHER_ASSET_PROFILE_PROCESS)
@@ -83,19 +84,17 @@ export class DataGatheringProcessor {
 
         if (lastMarketPrice) {
           try {
-            await this.prismaService.marketData.create({
+            await this.marketDataService.updateMarketData({
               data: {
-                dataSource,
+                marketPrice: lastMarketPrice,
-                symbol,
+                state: 'CLOSE'
-                date: new Date(
+              },
-                  Date.UTC(
+              where: {
-                    getYear(currentDate),
+                dataSource_date_symbol: {
-                    getMonth(currentDate),
+                  dataSource,
-                    getDate(currentDate),
+                  symbol,
-                    0
+                  date: getStartOfUtcDate(currentDate)
-                  )
+                }
-                ),
-                marketPrice: lastMarketPrice
               }
             });
           } catch {}

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -271,7 +271,8 @@ export class DataGatheringService {
         by: ['symbol'],
         orderBy: [{ symbol: 'asc' }],
         where: {
-          date: { gt: startDate }
+          date: { gt: startDate },
+          state: 'CLOSE'
         }
       })
     )

apps/api/src/services/data-provider/data-provider.service.ts

@@ -291,7 +291,8 @@ export class DataProviderService {
                       symbol,
                       dataSource: response[symbol].dataSource,
                       date: getStartOfUtcDate(new Date()),
-                      marketPrice: response[symbol].marketPrice
+                      marketPrice: response[symbol].marketPrice,
+                      state: 'INTRADAY'
                     };
                   })
               });

apps/api/src/services/market-data/market-data.service.ts

@@ -5,7 +5,12 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { resetHours } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
-import { DataSource, MarketData, Prisma } from '@prisma/client';
+import {
+  DataSource,
+  MarketData,
+  MarketDataState,
+  Prisma
+} from '@prisma/client';
 
 @Injectable()
 export class MarketDataService {
@@ -92,7 +97,9 @@ export class MarketDataService {
   }
 
   public async updateMarketData(params: {
-    data: { dataSource: DataSource } & UpdateMarketDataDto;
+    data: {
+      state: MarketDataState;
+    } & UpdateMarketDataDto;
     where: Prisma.MarketDataWhereUniqueInput;
   }): Promise<MarketData> {
     const { data, where } = params;
@@ -100,12 +107,13 @@ export class MarketDataService {
     return this.prismaService.marketData.upsert({
       where,
       create: {
-        dataSource: data.dataSource,
+        dataSource: where.dataSource_date_symbol.dataSource,
         date: where.dataSource_date_symbol.date,
         marketPrice: data.marketPrice,
+        state: data.state,
         symbol: where.dataSource_date_symbol.symbol
       },
-      update: { marketPrice: data.marketPrice }
+      update: { marketPrice: data.marketPrice, state: data.state }
     });
   }
 
@@ -119,16 +127,18 @@ export class MarketDataService {
     data: Prisma.MarketDataUpdateInput[];
   }): Promise<MarketData[]> {
     const upsertPromises = data.map(
-      ({ dataSource, date, marketPrice, symbol }) => {
+      ({ dataSource, date, marketPrice, symbol, state }) => {
         return this.prismaService.marketData.upsert({
           create: {
             dataSource: <DataSource>dataSource,
             date: <Date>date,
             marketPrice: <number>marketPrice,
+            state: <MarketDataState>state,
             symbol: <string>symbol
           },
           update: {
-            marketPrice: <number>marketPrice
+            marketPrice: <number>marketPrice,
+            state: <MarketDataState>state
           },
           where: {
             dataSource_date_symbol: {

prisma/migrations/20230424194009_added_state_to_market_data/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "MarketDataState" AS ENUM ('CLOSE', 'INTRADAY');
+
+-- AlterTable
+ALTER TABLE "MarketData" ADD COLUMN "state" "MarketDataState" NOT NULL DEFAULT 'CLOSE';

prisma/schema.prisma

@@ -59,12 +59,13 @@ model AuthDevice {
 }
 
 model MarketData {
-  createdAt   DateTime   @default(now())
+  createdAt   DateTime        @default(now())
   dataSource  DataSource
   date        DateTime
-  id          String     @id @default(uuid())
+  id          String          @id @default(uuid())
-  symbol      String
   marketPrice Float
+  state       MarketDataState @default(CLOSE)
+  symbol      String
 
   @@unique([dataSource, date, symbol])
   @@index([symbol])
@@ -214,9 +215,9 @@ enum DataSource {
   YAHOO
 }
 
-enum ViewMode {
+enum MarketDataState {
-  DEFAULT
+  CLOSE
-  ZEN
+  INTRADAY
 }
 
 enum Provider {
@@ -237,3 +238,8 @@ enum Type {
   ITEM
   SELL
 }
+
+enum ViewMode {
+  DEFAULT
+  ZEN
+}