Feature/improve data gathering (#276)

* Improve data gathering
  * Refactoring
  * On server restart, only reset if hanging in LOCKED_DATA_GATHERING state

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Improved the data gathering handling on server restart
+
 ## 1.35.0 - 08.08.2021
 
 ### Changed

apps/api/src/app/admin/admin.service.ts

@@ -1,3 +1,4 @@
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { AdminData } from '@ghostfolio/common/interfaces';
@@ -7,6 +8,7 @@ import { Currency } from '@prisma/client';
 @Injectable()
 export class AdminService {
   public constructor(
+    private readonly dataGatheringService: DataGatheringService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly prismaService: PrismaService
   ) {}
@@ -68,18 +70,15 @@ export class AdminService {
   }
 
   private async getLastDataGathering() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
+    const lastDataGathering =
-      where: { key: 'LAST_DATA_GATHERING' }
+      await this.dataGatheringService.getLastDataGathering();
-    });
 
-    if (lastDataGathering?.value) {
+    if (lastDataGathering) {
-      return new Date(lastDataGathering.value);
+      return lastDataGathering;
     }
 
     const dataGatheringInProgress =
-      await this.prismaService.property.findUnique({
+      await this.dataGatheringService.getIsInProgress();
-        where: { key: 'LOCKED_DATA_GATHERING' }
-      });
 
     if (dataGatheringInProgress) {
       return 'IN_PROGRESS';

apps/api/src/app/app.controller.ts

@@ -1,12 +1,12 @@
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Controller } from '@nestjs/common';
 
-import { PrismaService } from '../services/prisma.service';
 import { RedisCacheService } from './redis-cache/redis-cache.service';
 
 @Controller()
 export class AppController {
   public constructor(
-    private readonly prismaService: PrismaService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly redisCacheService: RedisCacheService
   ) {
     this.initialize();
@@ -15,17 +15,12 @@ export class AppController {
   private async initialize() {
     this.redisCacheService.reset();
 
-    const isDataGatheringLocked = await this.prismaService.property.findUnique({
+    const isDataGatheringInProgress =
-      where: { key: 'LOCKED_DATA_GATHERING' }
+      await this.dataGatheringService.getIsInProgress();
-    });
 
-    if (!isDataGatheringLocked) {
+    if (isDataGatheringInProgress) {
-      // Prepare for automatical data gather if not locked
+      // Prepare for automatical data gathering, if hung up in progress state
-      await this.prismaService.property.deleteMany({
+      await this.dataGatheringService.reset();
-        where: {
-          OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
-        }
-      });
     }
   }
 }

apps/api/src/app/cache/cache.module.ts

@@ -1,3 +1,10 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
+import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
 
@@ -8,6 +15,16 @@ import { CacheService } from './cache.service';
 @Module({
   imports: [RedisCacheModule],
   controllers: [CacheController],
-  providers: [CacheService, PrismaService]
+  providers: [
+    AlphaVantageService,
+    CacheService,
+    ConfigurationService,
+    DataGatheringService,
+    DataProviderService,
+    GhostfolioScraperApiService,
+    PrismaService,
+    RakutenRapidApiService,
+    YahooFinanceService
+  ]
 })
 export class CacheModule {}

apps/api/src/app/cache/cache.service.ts

@@ -1,16 +1,14 @@
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Injectable } from '@nestjs/common';
 
 @Injectable()
 export class CacheService {
-  public constructor(private readonly prismaService: PrismaService) {}
+  public constructor(
+    private readonly dataGaterhingService: DataGatheringService
+  ) {}
 
   public async flush(): Promise<void> {
-    await this.prismaService.property.deleteMany({
+    await this.dataGaterhingService.reset();
-      where: {
-        OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
-      }
-    });
 
     return;
   }

apps/api/src/app/info/info.module.ts

@@ -1,4 +1,10 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
+import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';
@@ -14,6 +20,16 @@ import { InfoService } from './info.service';
     })
   ],
   controllers: [InfoController],
-  providers: [ConfigurationService, InfoService, PrismaService]
+  providers: [
+    AlphaVantageService,
+    ConfigurationService,
+    DataGatheringService,
+    DataProviderService,
+    GhostfolioScraperApiService,
+    InfoService,
+    PrismaService,
+    RakutenRapidApiService,
+    YahooFinanceService
+  ]
 })
 export class InfoModule {}

apps/api/src/app/info/info.service.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { InfoItem } from '@ghostfolio/common/interfaces';
 import { Subscription } from '@ghostfolio/common/interfaces/subscription.interface';
@@ -15,6 +16,7 @@ export class InfoService {
 
   public constructor(
     private readonly configurationService: ConfigurationService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly jwtService: JwtService,
     private readonly prismaService: PrismaService
   ) {}
@@ -116,11 +118,10 @@ export class InfoService {
   }
 
   private async getLastDataGathering() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
+    const lastDataGathering =
-      where: { key: 'LAST_DATA_GATHERING' }
+      await this.dataGatheringService.getLastDataGathering();
-    });
 
-    return lastDataGathering?.value ? new Date(lastDataGathering.value) : null;
+    return lastDataGathering ?? null;
   }
 
   private async getStatistics() {

apps/api/src/services/data-gathering.service.ts

@@ -250,6 +250,34 @@ export class DataGatheringService {
     });
   }
 
+  public async getIsInProgress() {
+    return await this.prismaService.property.findUnique({
+      where: { key: 'LOCKED_DATA_GATHERING' }
+    });
+  }
+
+  public async getLastDataGathering() {
+    const lastDataGathering = await this.prismaService.property.findUnique({
+      where: { key: 'LAST_DATA_GATHERING' }
+    });
+
+    if (lastDataGathering?.value) {
+      return new Date(lastDataGathering.value);
+    }
+
+    return undefined;
+  }
+
+  public async reset() {
+    console.log('Data gathering has been reset.');
+
+    await this.prismaService.property.deleteMany({
+      where: {
+        OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
+      }
+    });
+  }
+
   private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
     const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
       return {
@@ -373,18 +401,13 @@ export class DataGatheringService {
   }
 
   private async isDataGatheringNeeded() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
+    const lastDataGathering = await this.getLastDataGathering();
-      where: { key: 'LAST_DATA_GATHERING' }
-    });
 
     const isDataGatheringLocked = await this.prismaService.property.findUnique({
       where: { key: 'LOCKED_DATA_GATHERING' }
     });
 
-    const diffInHours = differenceInHours(
+    const diffInHours = differenceInHours(new Date(), lastDataGathering);
-      new Date(),
-      new Date(lastDataGathering?.value)
-    );
 
     return (diffInHours >= 1 || !lastDataGathering) && !isDataGatheringLocked;
   }

apps/client/src/app/pages/admin/admin-page.component.ts

@@ -152,7 +152,7 @@ export class AdminPageComponent implements OnDestroy, OnInit {
           } else if (lastDataGathering === 'IN_PROGRESS') {
             this.dataGatheringInProgress = true;
           } else {
-            this.lastDataGathering = '-';
+            this.lastDataGathering = 'Starting soon...';
           }
 
           this.transactionCount = transactionCount;