Feature/support data gathering by symbol and date (#532)

* Support data gathering by symbol and date

* Update changelog

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Extended the data gathering by symbol endpoint with an optional date
+
 ### Changed
 
 - Upgraded `Nx` from version `13.2.2` to `13.3.0`

apps/api/src/app/admin/admin.controller.ts

@@ -21,7 +21,8 @@ import {
 } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
 import { AuthGuard } from '@nestjs/passport';
-import { DataSource } from '@prisma/client';
+import { DataSource, MarketData } from '@prisma/client';
+import { isDate, isValid } from 'date-fns';
 import { StatusCodes, getReasonPhrase } from 'http-status-codes';
 
 import { AdminService } from './admin.service';
@@ -73,6 +74,26 @@ export class AdminController {
     return;
   }
 
+  @Post('gather/profile-data')
+  @UseGuards(AuthGuard('jwt'))
+  public async gatherProfileData(): Promise<void> {
+    if (
+      !hasPermission(
+        this.request.user.permissions,
+        permissions.accessAdminControl
+      )
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    this.dataGatheringService.gatherProfileData();
+
+    return;
+  }
+
   @Post('gather/:dataSource/:symbol')
   @UseGuards(AuthGuard('jwt'))
   public async gatherSymbol(
@@ -96,9 +117,13 @@ export class AdminController {
     return;
   }
 
-  @Post('gather/profile-data')
+  @Post('gather/:dataSource/:symbol/:dateString')
   @UseGuards(AuthGuard('jwt'))
-  public async gatherProfileData(): Promise<void> {
+  public async gatherSymbolForDate(
+    @Param('dataSource') dataSource: DataSource,
+    @Param('dateString') dateString: string,
+    @Param('symbol') symbol: string
+  ): Promise<MarketData> {
     if (
       !hasPermission(
         this.request.user.permissions,
@@ -111,9 +136,20 @@ export class AdminController {
       );
     }
 
-    this.dataGatheringService.gatherProfileData();
+    const date = new Date(dateString);
 
-    return;
+    if (!isDate(date)) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.BAD_REQUEST),
+        StatusCodes.BAD_REQUEST
+      );
+    }
+
+    return this.dataGatheringService.gatherSymbolForDate({
+      dataSource,
+      date,
+      symbol
+    });
   }
 
   @Get('market-data')

apps/api/src/services/data-gathering.service.ts

@@ -6,7 +6,7 @@ import {
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { Inject, Injectable, Logger } from '@nestjs/common';
-import { DataSource } from '@prisma/client';
+import { DataSource, MarketData } from '@prisma/client';
 import {
   differenceInHours,
   format,
@@ -181,6 +181,44 @@ export class DataGatheringService {
     }
   }
 
+  public async gatherSymbolForDate({
+    dataSource,
+    date,
+    symbol
+  }: {
+    dataSource: DataSource;
+    date: Date;
+    symbol: string;
+  }) {
+    try {
+      const historicalData = await this.dataProviderService.getHistoricalRaw(
+        [{ dataSource, symbol }],
+        date,
+        date
+      );
+
+      const marketPrice =
+        historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
+
+      if (marketPrice) {
+        return await this.prismaService.marketData.upsert({
+          create: {
+            dataSource,
+            date,
+            marketPrice,
+            symbol
+          },
+          update: { marketPrice },
+          where: { date_symbol: { date, symbol } }
+        });
+      }
+    } catch (error) {
+      Logger.error(error);
+    } finally {
+      return undefined;
+    }
+  }
+
   public async gatherProfileData(aDataGatheringItems?: IDataGatheringItem[]) {
     Logger.log('Profile data gathering has been started.');
     console.time('data-gathering-profile');

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -8,7 +8,7 @@ import { AssetClass, AssetSubClass, DataSource } from '@prisma/client';
 import * as bent from 'bent';
 import Big from 'big.js';
 import { countries } from 'countries-list';
-import { format } from 'date-fns';
+import { addDays, format, isSameDay } from 'date-fns';
 import * as yahooFinance from 'yahoo-finance';
 
 import {
@@ -135,6 +135,10 @@ export class YahooFinanceService implements DataProviderInterface {
       return {};
     }
 
+    if (isSameDay(from, to)) {
+      to = addDays(to, 1);
+    }
+
     const yahooFinanceSymbols = aSymbols.map((symbol) => {
       return this.convertToYahooFinanceSymbol(symbol);
     });

apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.html

@@ -5,20 +5,25 @@
       <div
         *ngFor="let dayItem of days; let i = index"
         class="day"
-        [title]="
-          (itemByMonth.key + '-' + (i + 1 < 10 ? '0' + (i + 1) : i + 1)
-            | date: defaultDateFormat) ?? ''
-        "
         [ngClass]="{
-          valid: isDateOfInterest(
+          'cursor-pointer valid': isDateOfInterest(
             itemByMonth.key + '-' + (i + 1 < 10 ? '0' + (i + 1) : i + 1)
           ),
-          'available cursor-pointer':
-            marketDataByMonth[itemByMonth.key][i + 1]?.day === i + 1
+          available:
+            marketDataByMonth[itemByMonth.key][
+              i + 1 < 10 ? '0' + (i + 1) : i + 1
+            ]?.day ===
+            i + 1
         }"
+        [title]="
+          (itemByMonth.key + '-' + (i + 1 < 10 ? '0' + (i + 1) : i + 1)
+            | date: defaultDateFormat) ?? ''
+        "
         (click)="
-          marketDataByMonth[itemByMonth.key][i + 1] &&
-            onOpenMarketDataDetail(marketDataByMonth[itemByMonth.key][i + 1])
+          onOpenMarketDataDetail({
+            day: i + 1 < 10 ? '0' + (i + 1) : i + 1,
+            yearMonth: itemByMonth.key
+          })
         "
       ></div>
     </div>

apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts

@@ -8,7 +8,7 @@ import {
 import { MatDialog } from '@angular/material/dialog';
 import { DEFAULT_DATE_FORMAT } from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
-import { MarketData } from '@prisma/client';
+import { DataSource, MarketData } from '@prisma/client';
 import { format, isBefore, isValid, parse } from 'date-fns';
 import { DeviceDetectorService } from 'ngx-device-detector';
 import { Subject, takeUntil } from 'rxjs';
@@ -22,7 +22,9 @@ import { MarketDataDetailDialog } from './market-data-detail-dialog/market-data-
   templateUrl: './admin-market-data-detail.component.html'
 })
 export class AdminMarketDataDetailComponent implements OnChanges, OnInit {
+  @Input() dataSource: DataSource;
   @Input() marketData: MarketData[];
+  @Input() symbol: string;
 
   public days = Array(31);
   public defaultDateFormat = DEFAULT_DATE_FORMAT;
@@ -53,7 +55,9 @@ export class AdminMarketDataDetailComponent implements OnChanges, OnInit {
         this.marketDataByMonth[key] = {};
       }
 
-      this.marketDataByMonth[key][currentDay] = {
+      this.marketDataByMonth[key][
+        currentDay < 10 ? `0${currentDay}` : currentDay
+      ] = {
         ...marketDataItem,
         day: currentDay
       };
@@ -66,12 +70,21 @@ export class AdminMarketDataDetailComponent implements OnChanges, OnInit {
     return isValid(date) && isBefore(date, new Date());
   }
 
-  public onOpenMarketDataDetail({ date, marketPrice, symbol }: MarketData) {
+  public onOpenMarketDataDetail({
+    day,
+    yearMonth
+  }: {
+    day: string;
+    yearMonth: string;
+  }) {
+    const marketPrice = this.marketDataByMonth[yearMonth]?.[day]?.marketPrice;
+
     const dialogRef = this.dialog.open(MarketDataDetailDialog, {
       data: {
         marketPrice,
-        symbol,
-        date: format(date, DEFAULT_DATE_FORMAT)
+        dataSource: this.dataSource,
+        date: new Date(`${yearMonth}-${day}`),
+        symbol: this.symbol
       },
       height: this.deviceType === 'mobile' ? '97.5vh' : '80vh',
       width: this.deviceType === 'mobile' ? '100vw' : '50rem'

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/interfaces/interfaces.ts

@@ -1,5 +1,8 @@
+import { DataSource } from '@prisma/client';
+
 export interface MarketDataDetailDialogParams {
-  date: string;
+  dataSource: DataSource;
+  date: Date;
   marketPrice: number;
   symbol: string;
 }

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.component.ts

@@ -1,11 +1,14 @@
 import {
   ChangeDetectionStrategy,
+  ChangeDetectorRef,
   Component,
   Inject,
   OnDestroy
 } from '@angular/core';
 import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
-import { Subject } from 'rxjs';
+import { AdminService } from '@ghostfolio/client/services/admin.service';
+import { MarketData } from '@prisma/client';
+import { Subject, takeUntil } from 'rxjs';
 
 import { MarketDataDetailDialogParams } from './interfaces/interfaces';
 
@@ -20,6 +23,8 @@ export class MarketDataDetailDialog implements OnDestroy {
   private unsubscribeSubject = new Subject<void>();
 
   public constructor(
+    private adminService: AdminService,
+    private changeDetectorRef: ChangeDetectorRef,
     public dialogRef: MatDialogRef<MarketDataDetailDialog>,
     @Inject(MAT_DIALOG_DATA) public data: MarketDataDetailDialogParams
   ) {}
@@ -30,6 +35,21 @@ export class MarketDataDetailDialog implements OnDestroy {
     this.dialogRef.close();
   }
 
+  public onGatherData() {
+    this.adminService
+      .gatherSymbol({
+        dataSource: this.data.dataSource,
+        date: this.data.date,
+        symbol: this.data.symbol
+      })
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe((marketData: MarketData) => {
+        this.data.marketPrice = marketData.marketPrice;
+
+        this.changeDetectorRef.markForCheck();
+      });
+  }
+
   public ngOnDestroy() {
     this.unsubscribeSubject.next();
     this.unsubscribeSubject.complete();

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html

@@ -1,15 +1,29 @@
 <form class="d-flex flex-column h-100">
   <h1 mat-dialog-title i18n>Details for {{ data.symbol }}</h1>
   <div class="flex-grow-1" mat-dialog-content>
-    <div>
+    <div class="mb-3">
       <mat-form-field appearance="outline" class="w-100">
         <mat-label i18n>Date</mat-label>
-        <input matInput name="date" readonly [(ngModel)]="data.date" />
+        <input
+          disabled
+          matInput
+          name="date"
+          [matDatepicker]="date"
+          [(ngModel)]="data.date"
+        />
+        <mat-datepicker-toggle matSuffix [for]="date">
+          <ion-icon
+            class="text-muted"
+            matDatepickerToggleIcon
+            name="calendar-clear-outline"
+          ></ion-icon>
+        </mat-datepicker-toggle>
+        <mat-datepicker #date disabled="true"></mat-datepicker>
       </mat-form-field>
     </div>
-    <div>
-      <mat-form-field appearance="outline" class="w-100">
-        <mat-label i18n>MarketPrice</mat-label>
+    <div class="align-items-center d-flex">
+      <mat-form-field appearance="outline" class="flex-grow-1 mr-2">
+        <mat-label i18n>Market Price</mat-label>
         <input
           matInput
           name="marketPrice"
@@ -17,6 +31,9 @@
           [(ngModel)]="data.marketPrice"
         />
       </mat-form-field>
+      <button color="accent" i18n mat-flat-button (click)="onGatherData()">
+        Gather Data
+      </button>
     </div>
   </div>
   <div class="justify-content-end" mat-dialog-actions>

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.module.ts

@@ -2,6 +2,7 @@ import { CommonModule } from '@angular/common';
 import { CUSTOM_ELEMENTS_SCHEMA, NgModule } from '@angular/core';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { MatButtonModule } from '@angular/material/button';
+import { MatDatepickerModule } from '@angular/material/datepicker';
 import { MatDialogModule } from '@angular/material/dialog';
 import { MatFormFieldModule } from '@angular/material/form-field';
 import { MatInputModule } from '@angular/material/input';
@@ -15,6 +16,7 @@ import { MarketDataDetailDialog } from './market-data-detail-dialog.component';
     CommonModule,
     FormsModule,
     MatButtonModule,
+    MatDatepickerModule,
     MatDialogModule,
     MatFormFieldModule,
     MatInputModule,

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss

@@ -3,5 +3,21 @@
 
   .mat-dialog-content {
     max-height: unset;
+
+    .mat-form-field-appearance-outline {
+      ::ng-deep {
+        .mat-form-field-suffix {
+          top: -0.3rem;
+        }
+
+        .mat-form-field-wrapper {
+          padding-bottom: 0;
+        }
+      }
+
+      ion-icon {
+        font-size: 130%;
+      }
+    }
   }
 }

apps/client/src/app/components/admin-market-data/admin-market-data.html

@@ -47,7 +47,9 @@
               <td></td>
               <td colspan="4">
                 <gf-admin-market-data-detail
+                  [dataSource]="item.dataSource"
                   [marketData]="marketDataDetails"
+                  [symbol]="item.symbol"
                 ></gf-admin-market-data-detail>
               </td>
             </tr>

apps/client/src/app/services/admin.service.ts

@@ -1,6 +1,8 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
-import { DataSource } from '@prisma/client';
+import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DataSource, MarketData } from '@prisma/client';
+import { format } from 'date-fns';
 
 @Injectable({
   providedIn: 'root'
@@ -18,14 +20,19 @@ export class AdminService {
 
   public gatherSymbol({
     dataSource,
+    date,
     symbol
   }: {
     dataSource: DataSource;
+    date?: Date;
     symbol: string;
   }) {
-    return this.http.post<void>(
-      `/api/admin/gather/${dataSource}/${symbol}`,
-      {}
-    );
+    let url = `/api/admin/gather/${dataSource}/${symbol}`;
+
+    if (date) {
+      url = `${url}/${format(date, DATE_FORMAT)}`;
+    }
+
+    return this.http.post<MarketData | void>(url, {});
   }
 }