Feature/adapt priorities of data gathering jobs (#3262)

* Adapt priorities of data gathering jobs

* Update changelog

CHANGELOG.md

@@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added support to immediately execute a queue job from the admin control panel
+- Added a priority column to the queue jobs view in the admin control panel
 
 ### Changed
 
+- Adapted the priorities of queue jobs
 - Upgraded `angular` from version `17.2.4` to `17.3.3`
 - Upgraded `Nx` from version `18.1.2` to `18.2.3`
 - Upgraded `prisma` from version `5.11.0` to `5.12.1`

apps/api/src/app/admin/admin.controller.ts

@@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/man
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
 import {
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+  DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
-import {
-  getAssetProfileIdentifier,
-  resetHours
-} from '@ghostfolio/common/helper';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import {
   AdminData,
   AdminMarketData,
@@ -94,7 +93,8 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: getAssetProfileIdentifier({ dataSource, symbol })
+            jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+            priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
           }
         };
       })
@@ -119,7 +119,8 @@ export class AdminController {
           name: GATHER_ASSET_PROFILE_PROCESS,
           opts: {
             ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: getAssetProfileIdentifier({ dataSource, symbol })
+            jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+            priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
           }
         };
       })
@@ -141,7 +142,8 @@ export class AdminController {
       name: GATHER_ASSET_PROFILE_PROCESS,
       opts: {
         ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: getAssetProfileIdentifier({ dataSource, symbol })
+        jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+        priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
       }
     });
   }

apps/api/src/app/admin/queue/queue.service.ts

@@ -58,6 +58,7 @@ export class QueueService {
             finishedOn: job.finishedOn,
             id: job.id,
             name: job.name,
+            opts: job.opts,
             stacktrace: job.stacktrace,
             state: await job.getState(),
             timestamp: job.timestamp

apps/api/src/app/import/import.service.ts

@@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/da
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import {
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+  DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+} from '@ghostfolio/common/config';
 import {
   DATE_FORMAT,
   getAssetProfileIdentifier,
@@ -448,15 +452,16 @@ export class ImportService {
         });
       });
 
-      this.dataGatheringService.gatherSymbols(
-        uniqueActivities.map(({ date, SymbolProfile }) => {
+      this.dataGatheringService.gatherSymbols({
+        dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => {
           return {
             date,
             dataSource: SymbolProfile.dataSource,
             symbol: SymbolProfile.symbol
           };
-        })
-      );
+        }),
+        priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+      });
     }
 
     return activities;

apps/api/src/app/order/order.controller.ts

@@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interc
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
-import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
+import {
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+  HEADER_KEY_IMPERSONATION
+} from '@ghostfolio/common/config';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { DateRange, RequestWithUser } from '@ghostfolio/common/types';
 
@@ -160,13 +163,16 @@ export class OrderController {
     if (data.dataSource && !order.isDraft) {
       // Gather symbol data in the background, if data source is set
       // (not MANUAL) and not draft
-      this.dataGatheringService.gatherSymbols([
-        {
-          dataSource: data.dataSource,
-          date: order.date,
-          symbol: data.symbol
-        }
-      ]);
+      this.dataGatheringService.gatherSymbols({
+        dataGatheringItems: [
+          {
+            dataSource: data.dataSource,
+            date: order.date,
+            symbol: data.symbol
+          }
+        ],
+        priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+      });
     }
 
     return order;

apps/api/src/app/order/order.service.ts

@@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
@@ -101,7 +102,8 @@ export class OrderService {
           jobId: getAssetProfileIdentifier({
             dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
             symbol: data.SymbolProfile.connectOrCreate.create.symbol
-          })
+          }),
+          priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
         }
       });
     }
@@ -427,13 +429,17 @@ export class OrderService {
 
       if (!isDraft) {
         // Gather symbol data of order in the background, if not draft
-        this.dataGatheringService.gatherSymbols([
-          {
-            dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource,
-            date: <Date>data.date,
-            symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
-          }
-        ]);
+        this.dataGatheringService.gatherSymbols({
+          dataGatheringItems: [
+            {
+              dataSource:
+                data.SymbolProfile.connect.dataSource_symbol.dataSource,
+              date: <Date>data.date,
+              symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
+            }
+          ],
+          priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+        });
       }
     }
 

apps/api/src/services/cron.service.ts

@@ -1,4 +1,5 @@
 import {
+  DATA_GATHERING_QUEUE_PRIORITY_LOW,
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
   PROPERTY_IS_DATA_GATHERING_ENABLED
@@ -56,7 +57,8 @@ export class CronService {
             name: GATHER_ASSET_PROFILE_PROCESS,
             opts: {
               ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-              jobId: getAssetProfileIdentifier({ dataSource, symbol })
+              jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+              priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
             }
           };
         })

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   DATA_GATHERING_QUEUE,
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+  DATA_GATHERING_QUEUE_PRIORITY_LOW,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
   PROPERTY_BENCHMARKS
@@ -61,24 +63,35 @@ export class DataGatheringService {
 
   public async gather7Days() {
     const dataGatheringItems = await this.getSymbols7D();
-    await this.gatherSymbols(dataGatheringItems);
+    await this.gatherSymbols({
+      dataGatheringItems,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+    });
   }
 
   public async gatherMax() {
     const dataGatheringItems = await this.getSymbolsMax();
-    await this.gatherSymbols(dataGatheringItems);
+    await this.gatherSymbols({
+      dataGatheringItems,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+    });
   }
 
   public async gatherSymbol({ dataSource, symbol }: UniqueAsset) {
     await this.marketDataService.deleteMany({ dataSource, symbol });
 
-    const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => {
-      return (
-        dataGatheringItem.dataSource === dataSource &&
-        dataGatheringItem.symbol === symbol
-      );
+    const dataGatheringItems = (await this.getSymbolsMax()).filter(
+      (dataGatheringItem) => {
+        return (
+          dataGatheringItem.dataSource === dataSource &&
+          dataGatheringItem.symbol === symbol
+        );
+      }
+    );
+    await this.gatherSymbols({
+      dataGatheringItems,
+      priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
     });
-    await this.gatherSymbols(symbols);
   }
 
   public async gatherSymbolForDate({
@@ -230,9 +243,15 @@ export class DataGatheringService {
     );
   }
 
-  public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
+  public async gatherSymbols({
+    dataGatheringItems,
+    priority
+  }: {
+    dataGatheringItems: IDataGatheringItem[];
+    priority: number;
+  }) {
     await this.addJobsToQueue(
-      aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
+      dataGatheringItems.map(({ dataSource, date, symbol }) => {
         return {
           data: {
             dataSource,
@@ -242,6 +261,7 @@ export class DataGatheringService {
           name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
           opts: {
             ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+            priority,
             jobId: `${getAssetProfileIdentifier({
               dataSource,
               symbol

apps/client/src/app/components/admin-jobs/admin-jobs.component.ts

@@ -1,6 +1,11 @@
 import { AdminService } from '@ghostfolio/client/services/admin.service';
 import { UserService } from '@ghostfolio/client/services/user/user.service';
-import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config';
+import {
+  DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+  DATA_GATHERING_QUEUE_PRIORITY_LOW,
+  DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
+  QUEUE_JOB_STATUS_LIST
+} from '@ghostfolio/common/config';
 import { getDateWithTimeFormatString } from '@ghostfolio/common/helper';
 import { AdminJobs, User } from '@ghostfolio/common/interfaces';
 
@@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators';
   templateUrl: './admin-jobs.html'
 })
 export class AdminJobsComponent implements OnDestroy, OnInit {
+  public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW;
+  public DATA_GATHERING_QUEUE_PRIORITY_HIGH =
+    DATA_GATHERING_QUEUE_PRIORITY_HIGH;
+  public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM =
+    DATA_GATHERING_QUEUE_PRIORITY_MEDIUM;
   public defaultDateTimeFormat: string;
   public filterForm: FormGroup;
   public dataSource: MatTableDataSource<AdminJobs['jobs'][0]> =
@@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit {
     'type',
     'symbol',
     'dataSource',
+    'priority',
     'attempts',
     'created',
     'finished',

apps/client/src/app/components/admin-jobs/admin-jobs.html

@@ -58,6 +58,25 @@
           </td>
         </ng-container>
 
+        <ng-container matColumnDef="priority">
+          <th *matHeaderCellDef class="px-1 py-2" mat-header-cell>
+            <ng-container i18n>Priority</ng-container>
+          </th>
+          <td *matCellDef="let element" class="px-1 py-2" mat-cell>
+            @if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) {
+              <ion-icon class="h6 mb-0" name="chevron-down-circle-outline" />
+            } @else if (
+              element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+            ) {
+              <ion-icon class="h6 mb-0" name="remove-circle-outline" />
+            } @else if (
+              element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH
+            ) {
+              <ion-icon class="h6 mb-0" name="chevron-up-circle-outline" />
+            }
+          </td>
+        </ng-container>
+
         <ng-container matColumnDef="attempts">
           <th *matHeaderCellDef class="px-1 py-2 text-right" mat-header-cell>
             <ng-container i18n>Attempts</ng-container>
@@ -90,24 +109,37 @@
             <ng-container i18n>Status</ng-container>
           </th>
           <td *matCellDef="let element" class="px-1 py-2" mat-cell>
-            <ion-icon *ngIf="element.state === 'active'" name="play-outline" />
+            <ion-icon
+              *ngIf="element.state === 'active'"
+              class="h6 mb-0"
+              name="play-outline"
+            />
             <ion-icon
               *ngIf="element.state === 'completed'"
-              class="text-success"
+              class="h6 mb-0 text-success"
               name="checkmark-circle-outline"
             />
             <ion-icon
               *ngIf="element.state === 'delayed'"
+              class="h6 mb-0"
               name="time-outline"
               [ngClass]="{ 'text-danger': element.stacktrace?.length > 0 }"
             />
             <ion-icon
               *ngIf="element.state === 'failed'"
-              class="text-danger"
+              class="h6 mb-0 text-danger"
               name="alert-circle-outline"
             />
-            <ion-icon *ngIf="element.state === 'paused'" name="pause-outline" />
-            <ion-icon *ngIf="element.state === 'waiting'" name="cafe-outline" />
+            <ion-icon
+              *ngIf="element.state === 'paused'"
+              class="h6 mb-0"
+              name="pause-outline"
+            />
+            <ion-icon
+              *ngIf="element.state === 'waiting'"
+              class="h6 mb-0"
+              name="cafe-outline"
+            />
           </td>
         </ng-container>
 

libs/common/src/lib/config.ts

@@ -32,8 +32,11 @@ export const warnColorRgb = {
 };
 
 export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE';
-export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
 export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1;
+export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
+export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round(
+  DATA_GATHERING_QUEUE_PRIORITY_LOW / 2
+);
 
 export const DEFAULT_CURRENCY = 'USD';
 export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
@@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
     delay: ms('1 minute'),
     type: 'exponential'
   },
-  priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,
   removeOnComplete: true
 };
 export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =
@@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
     delay: ms('1 minute'),
     type: 'exponential'
   },
-  priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,
   removeOnComplete: true
 };
 

libs/common/src/lib/interfaces/admin-jobs.interface.ts

@@ -8,6 +8,7 @@ export interface AdminJobs {
     | 'finishedOn'
     | 'id'
     | 'name'
+    | 'opts'
     | 'stacktrace'
     | 'timestamp'
   > & {