Feature/import historical data (#2448)

* Import historical data for an asset

* Update changelog

---------

Co-authored-by: Thomas <4159106+dtslvr@users.noreply.github.com>

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added support to transfer a part of the cash balance from one to another account
 - Extended the markets overview by benchmarks (date of last all time high)
+- Added support to import historical market data in the admin control panel
 
 ### Changed
 

apps/api/src/app/admin/admin.controller.ts

@@ -43,6 +43,7 @@ import { StatusCodes, getReasonPhrase } from 'http-status-codes';
 
 import { AdminService } from './admin.service';
 import { UpdateAssetProfileDto } from './update-asset-profile.dto';
+import { UpdateBulkMarketDataDto } from './update-bulk-market-data.dto';
 import { UpdateMarketDataDto } from './update-market-data.dto';
 
 @Controller('admin')
@@ -313,6 +314,43 @@ export class AdminController {
     return this.adminService.getMarketDataBySymbol({ dataSource, symbol });
   }
 
+  @Post('market-data/:dataSource/:symbol')
+  @UseGuards(AuthGuard('jwt'))
+  public async updateMarketData(
+    @Body() data: UpdateBulkMarketDataDto,
+    @Param('dataSource') dataSource: DataSource,
+    @Param('symbol') symbol: string
+  ) {
+    if (
+      !hasPermission(
+        this.request.user.permissions,
+        permissions.accessAdminControl
+      )
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    const dataBulkUpdate: Prisma.MarketDataUpdateInput[] = data.marketData.map(
+      ({ date, marketPrice }) => ({
+        dataSource,
+        date,
+        marketPrice,
+        symbol,
+        state: 'CLOSE'
+      })
+    );
+
+    return this.marketDataService.updateMany({
+      data: dataBulkUpdate
+    });
+  }
+
+  /**
+   * @deprecated
+   */
   @Put('market-data/:dataSource/:symbol/:dateString')
   @UseGuards(AuthGuard('jwt'))
   public async update(

apps/api/src/app/admin/update-bulk-market-data.dto.ts

@@ -0,0 +1,11 @@
+import { Type } from 'class-transformer';
+import { ArrayNotEmpty, IsArray, isNotEmptyObject } from 'class-validator';
+
+import { UpdateMarketDataDto } from './update-market-data.dto';
+
+export class UpdateBulkMarketDataDto {
+  @ArrayNotEmpty()
+  @IsArray()
+  @Type(() => UpdateMarketDataDto)
+  marketData: UpdateMarketDataDto[];
+}

apps/api/src/app/admin/update-market-data.dto.ts

@@ -1,6 +1,10 @@
-import { IsNumber } from 'class-validator';
+import { IsDate, IsNumber, IsOptional } from 'class-validator';
 
 export class UpdateMarketDataDto {
+  @IsDate()
+  @IsOptional()
+  date?: Date;
+
   @IsNumber()
   marketPrice: number;
 }

apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts

@@ -177,7 +177,7 @@ export class AdminMarketDataDetailComponent implements OnChanges, OnInit {
     dialogRef
       .afterClosed()
       .pipe(takeUntil(this.unsubscribeSubject))
-      .subscribe(({ withRefresh }) => {
+      .subscribe(({ withRefresh } = { withRefresh: false }) => {
         this.marketDataChanged.next(withRefresh);
       });
   }

apps/client/src/app/components/admin-market-data/admin-market-data.component.ts

@@ -342,7 +342,7 @@ export class AdminMarketDataComponent
         dialogRef
           .afterClosed()
           .pipe(takeUntil(this.unsubscribeSubject))
-          .subscribe(({ dataSource, symbol }) => {
+          .subscribe(({ dataSource, symbol } = {}) => {
             if (dataSource && symbol) {
               this.adminService
                 .addAssetProfile({ dataSource, symbol })

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -11,12 +11,15 @@ import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
 import { UpdateAssetProfileDto } from '@ghostfolio/api/app/admin/update-asset-profile.dto';
 import { AdminService } from '@ghostfolio/client/services/admin.service';
 import { DataService } from '@ghostfolio/client/services/data.service';
+import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import {
   AdminMarketDataDetails,
   UniqueAsset
 } from '@ghostfolio/common/interfaces';
 import { translate } from '@ghostfolio/ui/i18n';
 import { MarketData, SymbolProfile } from '@prisma/client';
+import { format, parseISO } from 'date-fns';
+import { parse as csvToJson } from 'papaparse';
 import { Subject } from 'rxjs';
 import { takeUntil } from 'rxjs/operators';
 
@@ -42,12 +45,17 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   public countries: {
     [code: string]: { name: string; value: number };
   };
+  public historicalDataAsCsvString: string;
   public isBenchmark = false;
   public marketDataDetails: MarketData[] = [];
   public sectors: {
     [name: string]: { name: string; value: number };
   };
 
+  private static readonly HISTORICAL_DATA_TEMPLATE = `date;marketPrice\n${format(
+    new Date(),
+    DATE_FORMAT
+  )};123.45`;
   private unsubscribeSubject = new Subject<void>();
 
   public constructor(
@@ -66,6 +74,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   }
 
   public initialize() {
+    this.historicalDataAsCsvString =
+      AssetProfileDialog.HISTORICAL_DATA_TEMPLATE;
+
     this.adminService
       .fetchAdminMarketDataBySymbol({
         dataSource: this.data.dataSource,
@@ -134,6 +145,29 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       .subscribe(() => {});
   }
 
+  public onImportHistoricalData() {
+    const marketData = csvToJson(this.historicalDataAsCsvString, {
+      dynamicTyping: true,
+      header: true,
+      skipEmptyLines: true
+    }).data;
+
+    this.adminService
+      .postMarketData({
+        dataSource: this.data.dataSource,
+        marketData: {
+          marketData: marketData.map(({ date, marketPrice }) => {
+            return { marketPrice, date: parseISO(date) };
+          })
+        },
+        symbol: this.data.symbol
+      })
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe(() => {
+        this.initialize();
+      });
+  }
+
   public onMarketDataChanged(withRefresh: boolean = false) {
     if (withRefresh) {
       this.initialize();

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -51,6 +51,36 @@
       [symbol]="data.symbol"
       (marketDataChanged)="onMarketDataChanged($event)"
     ></gf-admin-market-data-detail>
+
+    <div class="mt-3">
+      <mat-form-field appearance="outline" class="w-100 without-hint">
+        <mat-label>
+          <ng-container i18n>Historical Data</ng-container> (CSV)
+        </mat-label>
+        <textarea
+          cdkAutosizeMaxRows="5"
+          cdkTextareaAutosize
+          matInput
+          placeholder="e.g. 20230601;1.61"
+          type="text"
+          [ngModelOptions]="{standalone: true}"
+          [(ngModel)]="historicalDataAsCsvString"
+          (keyup.enter)="$event.stopPropagation()"
+        ></textarea>
+      </mat-form-field>
+    </div>
+
+    <div class="d-flex justify-content-end mt-2">
+      <button
+        color="accent"
+        mat-flat-button
+        type="button"
+        (click)="onImportHistoricalData()"
+      >
+        <ng-container i18n>Import</ng-container>
+      </button>
+    </div>
+
     <div class="row">
       <div class="col-6 mb-3">
         <gf-value i18n size="medium" [value]="assetProfile?.symbol"

apps/client/src/app/services/admin.service.ts

@@ -1,6 +1,7 @@
 import { HttpClient, HttpParams } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { UpdateAssetProfileDto } from '@ghostfolio/api/app/admin/update-asset-profile.dto';
+import { UpdateBulkMarketDataDto } from '@ghostfolio/api/app/admin/update-bulk-market-data.dto';
 import { UpdateMarketDataDto } from '@ghostfolio/api/app/admin/update-market-data.dto';
 import { CreatePlatformDto } from '@ghostfolio/api/app/platform/create-platform.dto';
 import { UpdatePlatformDto } from '@ghostfolio/api/app/platform/update-platform.dto';
@@ -214,6 +215,20 @@ export class AdminService {
     );
   }
 
+  public postMarketData({
+    dataSource,
+    marketData,
+    symbol
+  }: {
+    dataSource: DataSource;
+    marketData: UpdateBulkMarketDataDto;
+    symbol: string;
+  }) {
+    const url = `/api/v1/admin/market-data/${dataSource}/${symbol}`;
+
+    return this.http.post<MarketData>(url, marketData);
+  }
+
   public postPlatform(aPlatform: CreatePlatformDto) {
     return this.http.post<Platform>(`/api/v1/platform`, aPlatform);
   }