From ebee851b2349f6796748f0949afd4f9140ff4512 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 11 Dec 2021 11:45:13 +0100 Subject: [PATCH] Feature/support data gathering by symbol and date (#532) * Support data gathering by symbol and date * Update changelog --- CHANGELOG.md | 4 ++ apps/api/src/app/admin/admin.controller.ts | 46 +++++++++++++++++-- .../src/services/data-gathering.service.ts | 40 +++++++++++++++- .../yahoo-finance/yahoo-finance.service.ts | 6 ++- .../admin-market-data-detail.component.html | 23 ++++++---- .../admin-market-data-detail.component.ts | 23 ++++++++-- .../interfaces/interfaces.ts | 5 +- .../market-data-detail-dialog.component.ts | 22 ++++++++- .../market-data-detail-dialog.html | 27 +++++++++-- .../market-data-detail-dialog.module.ts | 2 + .../market-data-detail-dialog.scss | 16 +++++++ .../admin-market-data/admin-market-data.html | 2 + apps/client/src/app/services/admin.service.ts | 17 +++++-- 13 files changed, 200 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3261c74e8..e3c984066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Extended the data gathering by symbol endpoint with an optional date + ### Changed - Upgraded `Nx` from version `13.2.2` to `13.3.0` diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 1de70e588..2c2130da3 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -21,7 +21,8 @@ import { } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; -import { DataSource } from '@prisma/client'; +import { DataSource, MarketData } from '@prisma/client'; +import { isDate, isValid } from 'date-fns'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; import { AdminService } from './admin.service'; @@ -73,6 +74,26 @@ export class AdminController { return; } + @Post('gather/profile-data') + @UseGuards(AuthGuard('jwt')) + public async gatherProfileData(): Promise { + if ( + !hasPermission( + this.request.user.permissions, + permissions.accessAdminControl + ) + ) { + throw new HttpException( + getReasonPhrase(StatusCodes.FORBIDDEN), + StatusCodes.FORBIDDEN + ); + } + + this.dataGatheringService.gatherProfileData(); + + return; + } + @Post('gather/:dataSource/:symbol') @UseGuards(AuthGuard('jwt')) public async gatherSymbol( @@ -96,9 +117,13 @@ export class AdminController { return; } - @Post('gather/profile-data') + @Post('gather/:dataSource/:symbol/:dateString') @UseGuards(AuthGuard('jwt')) - public async gatherProfileData(): Promise { + public async gatherSymbolForDate( + @Param('dataSource') dataSource: DataSource, + @Param('dateString') dateString: string, + @Param('symbol') symbol: string + ): Promise { if ( !hasPermission( this.request.user.permissions, @@ -111,9 +136,20 @@ export class AdminController { ); } - this.dataGatheringService.gatherProfileData(); + const date = new Date(dateString); - return; + if (!isDate(date)) { + throw new HttpException( + getReasonPhrase(StatusCodes.BAD_REQUEST), + StatusCodes.BAD_REQUEST + ); + } + + return this.dataGatheringService.gatherSymbolForDate({ + dataSource, + date, + symbol + }); } @Get('market-data') diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 48ce87e02..203a496d6 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -6,7 +6,7 @@ import { } from '@ghostfolio/common/config'; import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper'; import { Inject, Injectable, Logger } from '@nestjs/common'; -import { DataSource } from '@prisma/client'; +import { DataSource, MarketData } from '@prisma/client'; import { differenceInHours, format, @@ -181,6 +181,44 @@ export class DataGatheringService { } } + public async gatherSymbolForDate({ + dataSource, + date, + symbol + }: { + dataSource: DataSource; + date: Date; + symbol: string; + }) { + try { + const historicalData = await this.dataProviderService.getHistoricalRaw( + [{ dataSource, symbol }], + date, + date + ); + + const marketPrice = + historicalData[symbol][format(date, DATE_FORMAT)].marketPrice; + + if (marketPrice) { + return await this.prismaService.marketData.upsert({ + create: { + dataSource, + date, + marketPrice, + symbol + }, + update: { marketPrice }, + where: { date_symbol: { date, symbol } } + }); + } + } catch (error) { + Logger.error(error); + } finally { + return undefined; + } + } + public async gatherProfileData(aDataGatheringItems?: IDataGatheringItem[]) { Logger.log('Profile data gathering has been started.'); console.time('data-gathering-profile'); diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 4079b437b..ec4640d77 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -8,7 +8,7 @@ import { AssetClass, AssetSubClass, DataSource } from '@prisma/client'; import * as bent from 'bent'; import Big from 'big.js'; import { countries } from 'countries-list'; -import { format } from 'date-fns'; +import { addDays, format, isSameDay } from 'date-fns'; import * as yahooFinance from 'yahoo-finance'; import { @@ -135,6 +135,10 @@ export class YahooFinanceService implements DataProviderInterface { return {}; } + if (isSameDay(from, to)) { + to = addDays(to, 1); + } + const yahooFinanceSymbols = aSymbols.map((symbol) => { return this.convertToYahooFinanceSymbol(symbol); }); diff --git a/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.html b/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.html index 2c67da932..6aab2161b 100644 --- a/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.html +++ b/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.html @@ -5,20 +5,25 @@
diff --git a/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts b/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts index 83f4d65a3..3c6db767a 100644 --- a/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts +++ b/apps/client/src/app/components/admin-market-data-detail/admin-market-data-detail.component.ts @@ -8,7 +8,7 @@ import { import { MatDialog } from '@angular/material/dialog'; import { DEFAULT_DATE_FORMAT } from '@ghostfolio/common/config'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; -import { MarketData } from '@prisma/client'; +import { DataSource, MarketData } from '@prisma/client'; import { format, isBefore, isValid, parse } from 'date-fns'; import { DeviceDetectorService } from 'ngx-device-detector'; import { Subject, takeUntil } from 'rxjs'; @@ -22,7 +22,9 @@ import { MarketDataDetailDialog } from './market-data-detail-dialog/market-data- templateUrl: './admin-market-data-detail.component.html' }) export class AdminMarketDataDetailComponent implements OnChanges, OnInit { + @Input() dataSource: DataSource; @Input() marketData: MarketData[]; + @Input() symbol: string; public days = Array(31); public defaultDateFormat = DEFAULT_DATE_FORMAT; @@ -53,7 +55,9 @@ export class AdminMarketDataDetailComponent implements OnChanges, OnInit { this.marketDataByMonth[key] = {}; } - this.marketDataByMonth[key][currentDay] = { + this.marketDataByMonth[key][ + currentDay < 10 ? `0${currentDay}` : currentDay + ] = { ...marketDataItem, day: currentDay }; @@ -66,12 +70,21 @@ export class AdminMarketDataDetailComponent implements OnChanges, OnInit { return isValid(date) && isBefore(date, new Date()); } - public onOpenMarketDataDetail({ date, marketPrice, symbol }: MarketData) { + public onOpenMarketDataDetail({ + day, + yearMonth + }: { + day: string; + yearMonth: string; + }) { + const marketPrice = this.marketDataByMonth[yearMonth]?.[day]?.marketPrice; + const dialogRef = this.dialog.open(MarketDataDetailDialog, { data: { marketPrice, - symbol, - date: format(date, DEFAULT_DATE_FORMAT) + dataSource: this.dataSource, + date: new Date(`${yearMonth}-${day}`), + symbol: this.symbol }, height: this.deviceType === 'mobile' ? '97.5vh' : '80vh', width: this.deviceType === 'mobile' ? '100vw' : '50rem' diff --git a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/interfaces/interfaces.ts b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/interfaces/interfaces.ts index a7defb817..a6f543789 100644 --- a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/interfaces/interfaces.ts +++ b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/interfaces/interfaces.ts @@ -1,5 +1,8 @@ +import { DataSource } from '@prisma/client'; + export interface MarketDataDetailDialogParams { - date: string; + dataSource: DataSource; + date: Date; marketPrice: number; symbol: string; } diff --git a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.component.ts b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.component.ts index 4ad1ebaa3..0ce14aba6 100644 --- a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.component.ts @@ -1,11 +1,14 @@ import { ChangeDetectionStrategy, + ChangeDetectorRef, Component, Inject, OnDestroy } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import { Subject } from 'rxjs'; +import { AdminService } from '@ghostfolio/client/services/admin.service'; +import { MarketData } from '@prisma/client'; +import { Subject, takeUntil } from 'rxjs'; import { MarketDataDetailDialogParams } from './interfaces/interfaces'; @@ -20,6 +23,8 @@ export class MarketDataDetailDialog implements OnDestroy { private unsubscribeSubject = new Subject(); public constructor( + private adminService: AdminService, + private changeDetectorRef: ChangeDetectorRef, public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: MarketDataDetailDialogParams ) {} @@ -30,6 +35,21 @@ export class MarketDataDetailDialog implements OnDestroy { this.dialogRef.close(); } + public onGatherData() { + this.adminService + .gatherSymbol({ + dataSource: this.data.dataSource, + date: this.data.date, + symbol: this.data.symbol + }) + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe((marketData: MarketData) => { + this.data.marketPrice = marketData.marketPrice; + + this.changeDetectorRef.markForCheck(); + }); + } + public ngOnDestroy() { this.unsubscribeSubject.next(); this.unsubscribeSubject.complete(); diff --git a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html index 65b3578fe..48bb2a7b0 100644 --- a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html +++ b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html @@ -1,15 +1,29 @@

Details for {{ data.symbol }}

-
+
Date - + + + + +
-
- - MarketPrice +
+ + Market Price +
diff --git a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.module.ts b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.module.ts index 9a7235c88..be72b917c 100644 --- a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.module.ts +++ b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.module.ts @@ -2,6 +2,7 @@ import { CommonModule } from '@angular/common'; import { CUSTOM_ELEMENTS_SCHEMA, NgModule } from '@angular/core'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { MatButtonModule } from '@angular/material/button'; +import { MatDatepickerModule } from '@angular/material/datepicker'; import { MatDialogModule } from '@angular/material/dialog'; import { MatFormFieldModule } from '@angular/material/form-field'; import { MatInputModule } from '@angular/material/input'; @@ -15,6 +16,7 @@ import { MarketDataDetailDialog } from './market-data-detail-dialog.component'; CommonModule, FormsModule, MatButtonModule, + MatDatepickerModule, MatDialogModule, MatFormFieldModule, MatInputModule, diff --git a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss index ce1c7d599..91922978c 100644 --- a/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss +++ b/apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss @@ -3,5 +3,21 @@ .mat-dialog-content { max-height: unset; + + .mat-form-field-appearance-outline { + ::ng-deep { + .mat-form-field-suffix { + top: -0.3rem; + } + + .mat-form-field-wrapper { + padding-bottom: 0; + } + } + + ion-icon { + font-size: 130%; + } + } } } diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.html index cadcc7858..15bab021f 100644 --- a/apps/client/src/app/components/admin-market-data/admin-market-data.html +++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html @@ -47,7 +47,9 @@ diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts index 53d76af74..7c9f9ff04 100644 --- a/apps/client/src/app/services/admin.service.ts +++ b/apps/client/src/app/services/admin.service.ts @@ -1,6 +1,8 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { DataSource } from '@prisma/client'; +import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DataSource, MarketData } from '@prisma/client'; +import { format } from 'date-fns'; @Injectable({ providedIn: 'root' @@ -18,14 +20,19 @@ export class AdminService { public gatherSymbol({ dataSource, + date, symbol }: { dataSource: DataSource; + date?: Date; symbol: string; }) { - return this.http.post( - `/api/admin/gather/${dataSource}/${symbol}`, - {} - ); + let url = `/api/admin/gather/${dataSource}/${symbol}`; + + if (date) { + url = `${url}/${format(date, DATE_FORMAT)}`; + } + + return this.http.post(url, {}); } }