diff --git a/CHANGELOG.md b/CHANGELOG.md index b4277d54b..e2c23a299 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a logo to the log on the server start +- Added the data gathering progress to the log and the admin control panel ## 1.74.0 - 11.11.2021 diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts index be78d668a..b5c2c2d91 100644 --- a/apps/api/src/app/admin/admin.service.ts +++ b/apps/api/src/app/admin/admin.service.ts @@ -20,6 +20,8 @@ export class AdminService { public async get(): Promise { return { + dataGatheringProgress: + await this.dataGatheringService.getDataGatheringProgress(), exchangeRates: this.exchangeRateDataService .getCurrencies() .filter((currency) => { @@ -58,7 +60,7 @@ export class AdminService { return 'IN_PROGRESS'; } - return null; + return undefined; } private async getUsersWithAnalytics(): Promise { diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 6a6d063d9..9541fce4a 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -18,7 +18,6 @@ import { import { ConfigurationService } from './configuration.service'; import { DataProviderService } from './data-provider/data-provider.service'; -import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface'; import { ExchangeRateDataService } from './exchange-rate-data.service'; import { IDataGatheringItem } from './interfaces/interfaces'; @@ -26,13 +25,14 @@ import { PrismaService } from './prisma.service'; @Injectable() export class DataGatheringService { + private dataGatheringProgress: number; + public constructor( private readonly configurationService: ConfigurationService, @Inject('DataEnhancers') private readonly dataEnhancers: DataEnhancerInterface[], private readonly dataProviderService: DataProviderService, private readonly exchangeRateDataService: ExchangeRateDataService, - private readonly ghostfolioScraperApi: GhostfolioScraperApiService, private readonly prismaService: PrismaService, private readonly symbolProfileService: SymbolProfileService ) {} @@ -204,8 +204,11 @@ export class DataGatheringService { public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { let hasError = false; + let symbolCounter = 0; for (const { dataSource, date, symbol } of aSymbolsWithStartDate) { + this.dataGatheringProgress = symbolCounter / aSymbolsWithStartDate.length; + try { const historicalData = await this.dataProviderService.getHistoricalRaw( [{ dataSource, symbol }], @@ -263,6 +266,16 @@ export class DataGatheringService { hasError = true; Logger.error(error); } + + if (symbolCounter > 0 && symbolCounter % 100 === 0) { + Logger.log( + `Data gathering progress: ${( + this.dataGatheringProgress * 100 + ).toFixed(2)}%` + ); + } + + symbolCounter += 1; } await this.exchangeRateDataService.initialize(); @@ -272,6 +285,16 @@ export class DataGatheringService { } } + public async getDataGatheringProgress() { + const isInProgress = await this.getIsInProgress(); + + if (isInProgress) { + return this.dataGatheringProgress; + } + + return undefined; + } + public async getIsInProgress() { return await this.prismaService.property.findUnique({ where: { key: 'LOCKED_DATA_GATHERING' } diff --git a/apps/client/src/app/pages/admin/admin-page.component.ts b/apps/client/src/app/pages/admin/admin-page.component.ts index 662405840..b61562192 100644 --- a/apps/client/src/app/pages/admin/admin-page.component.ts +++ b/apps/client/src/app/pages/admin/admin-page.component.ts @@ -22,6 +22,7 @@ import { takeUntil } from 'rxjs/operators'; }) export class AdminPageComponent implements OnDestroy, OnInit { public dataGatheringInProgress: boolean; + public dataGatheringProgress: number; public defaultDateFormat = DEFAULT_DATE_FORMAT; public exchangeRates: { label1: string; label2: string; value: number }[]; public lastDataGathering: string; @@ -134,12 +135,14 @@ export class AdminPageComponent implements OnDestroy, OnInit { .pipe(takeUntil(this.unsubscribeSubject)) .subscribe( ({ + dataGatheringProgress, exchangeRates, lastDataGathering, transactionCount, userCount, users }) => { + this.dataGatheringProgress = dataGatheringProgress; this.exchangeRates = exchangeRates; this.users = users; diff --git a/apps/client/src/app/pages/admin/admin-page.html b/apps/client/src/app/pages/admin/admin-page.html index dc59547e4..c98f010a3 100644 --- a/apps/client/src/app/pages/admin/admin-page.html +++ b/apps/client/src/app/pages/admin/admin-page.html @@ -35,14 +35,15 @@
-
Last Data Gathering
+
Data Gathering
{{ lastDataGathering }} In ProgressIn Progress ({{ dataGatheringProgress | percent : '1.2-2' + }})
diff --git a/libs/common/src/lib/interfaces/admin-data.interface.ts b/libs/common/src/lib/interfaces/admin-data.interface.ts index e882d2d69..71df57229 100644 --- a/libs/common/src/lib/interfaces/admin-data.interface.ts +++ b/libs/common/src/lib/interfaces/admin-data.interface.ts @@ -1,6 +1,7 @@ export interface AdminData { + dataGatheringProgress?: number; exchangeRates: { label1: string; label2: string; value: number }[]; - lastDataGathering: Date | 'IN_PROGRESS'; + lastDataGathering?: Date | 'IN_PROGRESS'; transactionCount: number; userCount: number; users: {