From 6996e5a140711623403c6e3bf6f0300fdec8706a Mon Sep 17 00:00:00 2001 From: Thomas <4159106+dtslvr@users.noreply.github.com> Date: Sat, 24 Jul 2021 21:13:48 +0200 Subject: [PATCH] Feature/add data gathering for symbol profile data (#228) * Implement profile data gathering * Update changelog --- CHANGELOG.md | 5 ++ apps/api/src/app/admin/admin.controller.ts | 21 ++++++ apps/api/src/app/order/order.service.ts | 2 + apps/api/src/services/cron.service.ts | 1 + .../src/services/data-gathering.service.ts | 74 ++++++++++++++++++- .../api/src/services/data-provider.service.ts | 22 +++++- .../app/pages/admin/admin-page.component.ts | 7 ++ .../src/app/pages/admin/admin-page.html | 59 ++++++++++----- apps/client/src/app/services/admin.service.ts | 4 + .../migration.sql | 2 + prisma/schema.prisma | 1 + 11 files changed, 171 insertions(+), 27 deletions(-) create mode 100644 prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index ca08bea38..615c6b8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Extended the data management by symbol profile data +- Added a currency attribute to the symbol profile model + ### Changed - Improved the style of the active page in the navigation on desktop diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 6593bf68d..fdece8df7 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -61,8 +61,29 @@ export class AdminController { ); } + await this.dataGatheringService.gatherProfileData(); this.dataGatheringService.gatherMax(); return; } + + @Post('gather/profile-data') + @UseGuards(AuthGuard('jwt')) + public async gatherProfileData(): Promise { + if ( + !hasPermission( + getPermissions(this.request.user.role), + permissions.accessAdminControl + ) + ) { + throw new HttpException( + getReasonPhrase(StatusCodes.FORBIDDEN), + StatusCodes.FORBIDDEN + ); + } + + this.dataGatheringService.gatherProfileData(); + + return; + } } diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 93fd8a706..b7c4d0b29 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -62,6 +62,8 @@ export class OrderService { ]); } + this.dataGatheringService.gatherProfileData([data.symbol]); + await this.cacheService.flush(aUserId); return this.prisma.order.create({ diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index b63d7d01f..01ee5444d 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -18,6 +18,7 @@ export class CronService { @Cron(CronExpression.EVERY_12_HOURS) public async runEveryTwelveHours() { + await this.dataGatheringService.gatherProfileData(); await this.exchangeRateDataService.loadCurrencies(); } } diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 5b1b35435..2fbc4655a 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -2,6 +2,7 @@ import { benchmarks, currencyPairs } from '@ghostfolio/common/config'; import { getUtc, isGhostfolioScraperApiSymbol, + isRakutenRapidApiSymbol, resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; @@ -37,7 +38,7 @@ export class DataGatheringService { if (isDataGatheringNeeded) { console.log('7d data gathering has been started.'); - console.time('data-gathering'); + console.time('7d-data-gathering'); await this.prisma.property.create({ data: { @@ -70,7 +71,7 @@ export class DataGatheringService { }); console.log('7d data gathering has been completed.'); - console.timeEnd('data-gathering'); + console.timeEnd('7d-data-gathering'); } } @@ -81,7 +82,7 @@ export class DataGatheringService { if (!isDataGatheringLocked) { console.log('Max data gathering has been started.'); - console.time('data-gathering'); + console.time('max-data-gathering'); await this.prisma.property.create({ data: { @@ -114,10 +115,56 @@ export class DataGatheringService { }); console.log('Max data gathering has been completed.'); - console.timeEnd('data-gathering'); + console.timeEnd('max-data-gathering'); } } + public async gatherProfileData(aSymbols?: string[]) { + console.log('Profile data gathering has been started.'); + console.time('profile-data-gathering'); + + let symbols = aSymbols; + + if (!symbols) { + const dataGatheringItems = await this.getSymbolsProfileData(); + symbols = dataGatheringItems.map((dataGatheringItem) => { + return dataGatheringItem.symbol; + }); + } + + const currentData = await this.dataProviderService.get(symbols); + + for (const [symbol, { currency, dataSource, name }] of Object.entries( + currentData + )) { + try { + await this.prisma.symbolProfile.upsert({ + create: { + currency, + dataSource, + name, + symbol + }, + update: { + currency, + name + }, + where: { + dataSource_symbol: { + dataSource, + symbol + } + } + }); + } catch (error) { + console.error(`${symbol}: ${error?.meta?.cause}`); + } + } + + console.log('Profile data gathering has been completed.'); + console.timeEnd('profile-data-gathering'); + } + public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { let hasError = false; @@ -303,6 +350,25 @@ export class DataGatheringService { ]; } + private async getSymbolsProfileData(): Promise { + const startDate = subDays(resetHours(new Date()), 7); + + const distinctOrders = await this.prisma.order.findMany({ + distinct: ['symbol'], + orderBy: [{ symbol: 'asc' }], + select: { dataSource: true, symbol: true } + }); + + return [...this.getBenchmarksToGather(startDate), ...distinctOrders].filter( + (distinctOrder) => { + return ( + distinctOrder.dataSource !== DataSource.GHOSTFOLIO && + distinctOrder.dataSource !== DataSource.RAKUTEN + ); + } + ); + } + private async isDataGatheringNeeded() { const lastDataGathering = await this.prisma.property.findUnique({ where: { key: 'LAST_DATA_GATHERING' } diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index 5d3b7de3c..cc32876c7 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -46,7 +46,10 @@ export class DataProviderService { } const yahooFinanceSymbols = aSymbols.filter((symbol) => { - return !isGhostfolioScraperApiSymbol(symbol); + return ( + !isGhostfolioScraperApiSymbol(symbol) && + !isRakutenRapidApiSymbol(symbol) + ); }); const response = await this.yahooFinanceService.get(yahooFinanceSymbols); @@ -57,13 +60,24 @@ export class DataProviderService { for (const symbol of ghostfolioScraperApiSymbols) { if (symbol) { - const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get( - [symbol] - ); + const ghostfolioScraperApiResult = + await this.ghostfolioScraperApiService.get([symbol]); response[symbol] = ghostfolioScraperApiResult[symbol]; } } + const rakutenRapidApiSymbols = aSymbols.filter((symbol) => { + return isRakutenRapidApiSymbol(symbol); + }); + + for (const symbol of rakutenRapidApiSymbols) { + if (symbol) { + const rakutenRapidApiResult = + await this.ghostfolioScraperApiService.get([symbol]); + response[symbol] = rakutenRapidApiResult[symbol]; + } + } + return response; } diff --git a/apps/client/src/app/pages/admin/admin-page.component.ts b/apps/client/src/app/pages/admin/admin-page.component.ts index 3eebc14cd..bc8669237 100644 --- a/apps/client/src/app/pages/admin/admin-page.component.ts +++ b/apps/client/src/app/pages/admin/admin-page.component.ts @@ -85,6 +85,13 @@ export class AdminPageComponent implements OnDestroy, OnInit { } } + public onGatherProfileData() { + this.adminService + .gatherProfileData() + .pipe(takeUntil(this.unsubscribeSubject)) + .subscribe(() => {}); + } + public formatDistanceToNow(aDateString: string) { if (aDateString) { const distanceString = formatDistanceToNowStrict(parseISO(aDateString), { diff --git a/apps/client/src/app/pages/admin/admin-page.html b/apps/client/src/app/pages/admin/admin-page.html index da46d62cb..d1b7f4c59 100644 --- a/apps/client/src/app/pages/admin/admin-page.html +++ b/apps/client/src/app/pages/admin/admin-page.html @@ -27,25 +27,46 @@ >
- - +
+ +
+
+ +
+
+ +
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts index 3e4768404..92516f5df 100644 --- a/apps/client/src/app/services/admin.service.ts +++ b/apps/client/src/app/services/admin.service.ts @@ -10,4 +10,8 @@ export class AdminService { public gatherMax() { return this.http.post(`/api/admin/gather/max`, {}); } + + public gatherProfileData() { + return this.http.post(`/api/admin/gather/profile-data`, {}); + } } diff --git a/prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql b/prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql new file mode 100644 index 000000000..a99c3c1d9 --- /dev/null +++ b/prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "SymbolProfile" ADD COLUMN "currency" "Currency"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e00b95245..c3665c5fd 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -117,6 +117,7 @@ model Settings { model SymbolProfile { countries Json? createdAt DateTime @default(now()) + currency Currency? dataSource DataSource id String @id @default(uuid()) name String?