diff --git a/CHANGELOG.md b/CHANGELOG.md index 68a85fd95..73999efb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added support to immediately execute a queue job from the admin control panel +- Added a priority column to the queue jobs view in the admin control panel ### Changed +- Adapted the priorities of queue jobs - Upgraded `angular` from version `17.2.4` to `17.3.3` - Upgraded `Nx` from version `18.1.2` to `18.2.3` - Upgraded `prisma` from version `5.11.0` to `5.12.1` diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index dab8fb8b2..298a471c3 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/man import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PropertyDto } from '@ghostfolio/api/services/property/property.dto'; import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; -import { - getAssetProfileIdentifier, - resetHours -} from '@ghostfolio/common/helper'; +import { getAssetProfileIdentifier } from '@ghostfolio/common/helper'; import { AdminData, AdminMarketData, @@ -94,7 +93,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } }; }) @@ -119,7 +119,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM } }; }) @@ -141,7 +142,8 @@ export class AdminController { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH } }); } diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts index e4f29bc60..abae3cad1 100644 --- a/apps/api/src/app/admin/queue/queue.service.ts +++ b/apps/api/src/app/admin/queue/queue.service.ts @@ -58,6 +58,7 @@ export class QueueService { finishedOn: job.finishedOn, id: job.id, name: job.name, + opts: job.opts, stacktrace: job.stacktrace, state: await job.getState(), timestamp: job.timestamp diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index cbdff87c0..26df9d069 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/da import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM +} from '@ghostfolio/common/config'; import { DATE_FORMAT, getAssetProfileIdentifier, @@ -448,15 +452,16 @@ export class ImportService { }); }); - this.dataGatheringService.gatherSymbols( - uniqueActivities.map(({ date, SymbolProfile }) => { + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => { return { date, dataSource: SymbolProfile.dataSource, symbol: SymbolProfile.symbol }; - }) - ); + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } return activities; diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts index c7fec0dac..3dadedcaf 100644 --- a/apps/api/src/app/order/order.controller.ts +++ b/apps/api/src/app/order/order.controller.ts @@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interc import { ApiService } from '@ghostfolio/api/services/api/api.service'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service'; -import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + HEADER_KEY_IMPERSONATION +} from '@ghostfolio/common/config'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import type { DateRange, RequestWithUser } from '@ghostfolio/common/types'; @@ -160,13 +163,16 @@ export class OrderController { if (data.dataSource && !order.isDraft) { // Gather symbol data in the background, if data source is set // (not MANUAL) and not draft - this.dataGatheringService.gatherSymbols([ - { - dataSource: data.dataSource, - date: order.date, - symbol: data.symbol - } - ]); + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: [ + { + dataSource: data.dataSource, + date: order.date, + symbol: data.symbol + } + ], + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } return order; diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index 20b2d5f15..35bfa1bcf 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate- import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS } from '@ghostfolio/common/config'; @@ -101,7 +102,8 @@ export class OrderService { jobId: getAssetProfileIdentifier({ dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, symbol: data.SymbolProfile.connectOrCreate.create.symbol - }) + }), + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH } }); } @@ -427,13 +429,17 @@ export class OrderService { if (!isDraft) { // Gather symbol data of order in the background, if not draft - this.dataGatheringService.gatherSymbols([ - { - dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource, - date: data.date, - symbol: data.SymbolProfile.connect.dataSource_symbol.symbol - } - ]); + this.dataGatheringService.gatherSymbols({ + dataGatheringItems: [ + { + dataSource: + data.SymbolProfile.connect.dataSource_symbol.dataSource, + date: data.date, + symbol: data.SymbolProfile.connect.dataSource_symbol.symbol + } + ], + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH + }); } } diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts index d74ad6a94..fc5d613a2 100644 --- a/apps/api/src/services/cron.service.ts +++ b/apps/api/src/services/cron.service.ts @@ -1,4 +1,5 @@ import { + DATA_GATHERING_QUEUE_PRIORITY_LOW, GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS_OPTIONS, PROPERTY_IS_DATA_GATHERING_ENABLED @@ -56,7 +57,8 @@ export class CronService { name: GATHER_ASSET_PROFILE_PROCESS, opts: { ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, - jobId: getAssetProfileIdentifier({ dataSource, symbol }) + jobId: getAssetProfileIdentifier({ dataSource, symbol }), + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW } }; }) diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index 6dccd645e..b2b0c371c 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service'; import { DATA_GATHERING_QUEUE, + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_LOW, GATHER_HISTORICAL_MARKET_DATA_PROCESS, GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, PROPERTY_BENCHMARKS @@ -61,24 +63,35 @@ export class DataGatheringService { public async gather7Days() { const dataGatheringItems = await this.getSymbols7D(); - await this.gatherSymbols(dataGatheringItems); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); } public async gatherMax() { const dataGatheringItems = await this.getSymbolsMax(); - await this.gatherSymbols(dataGatheringItems); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_LOW + }); } public async gatherSymbol({ dataSource, symbol }: UniqueAsset) { await this.marketDataService.deleteMany({ dataSource, symbol }); - const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => { - return ( - dataGatheringItem.dataSource === dataSource && - dataGatheringItem.symbol === symbol - ); + const dataGatheringItems = (await this.getSymbolsMax()).filter( + (dataGatheringItem) => { + return ( + dataGatheringItem.dataSource === dataSource && + dataGatheringItem.symbol === symbol + ); + } + ); + await this.gatherSymbols({ + dataGatheringItems, + priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH }); - await this.gatherSymbols(symbols); } public async gatherSymbolForDate({ @@ -230,9 +243,15 @@ export class DataGatheringService { ); } - public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) { + public async gatherSymbols({ + dataGatheringItems, + priority + }: { + dataGatheringItems: IDataGatheringItem[]; + priority: number; + }) { await this.addJobsToQueue( - aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => { + dataGatheringItems.map(({ dataSource, date, symbol }) => { return { data: { dataSource, @@ -242,6 +261,7 @@ export class DataGatheringService { name: GATHER_HISTORICAL_MARKET_DATA_PROCESS, opts: { ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS, + priority, jobId: `${getAssetProfileIdentifier({ dataSource, symbol diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts index 99d67dea2..23730f3aa 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts @@ -1,6 +1,11 @@ import { AdminService } from '@ghostfolio/client/services/admin.service'; import { UserService } from '@ghostfolio/client/services/user/user.service'; -import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config'; +import { + DATA_GATHERING_QUEUE_PRIORITY_HIGH, + DATA_GATHERING_QUEUE_PRIORITY_LOW, + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM, + QUEUE_JOB_STATUS_LIST +} from '@ghostfolio/common/config'; import { getDateWithTimeFormatString } from '@ghostfolio/common/helper'; import { AdminJobs, User } from '@ghostfolio/common/interfaces'; @@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators'; templateUrl: './admin-jobs.html' }) export class AdminJobsComponent implements OnDestroy, OnInit { + public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW; + public DATA_GATHERING_QUEUE_PRIORITY_HIGH = + DATA_GATHERING_QUEUE_PRIORITY_HIGH; + public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = + DATA_GATHERING_QUEUE_PRIORITY_MEDIUM; public defaultDateTimeFormat: string; public filterForm: FormGroup; public dataSource: MatTableDataSource = @@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit { 'type', 'symbol', 'dataSource', + 'priority', 'attempts', 'created', 'finished', diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.html index 31dad0ca6..5da929fe1 100644 --- a/apps/client/src/app/components/admin-jobs/admin-jobs.html +++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html @@ -58,6 +58,25 @@ + + + Priority + + + @if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) { + + } @else if ( + element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM + ) { + + } @else if ( + element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH + ) { + + } + + + Attempts @@ -90,24 +109,37 @@ Status - + - - + + diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 293f77488..5e1366ce2 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -32,8 +32,11 @@ export const warnColorRgb = { }; export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE'; -export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1; +export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER; +export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round( + DATA_GATHERING_QUEUE_PRIORITY_LOW / 2 +); export const DEFAULT_CURRENCY = 'USD'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; @@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = { delay: ms('1 minute'), type: 'exponential' }, - priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH, removeOnComplete: true }; export const GATHER_HISTORICAL_MARKET_DATA_PROCESS = @@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = { delay: ms('1 minute'), type: 'exponential' }, - priority: DATA_GATHERING_QUEUE_PRIORITY_LOW, removeOnComplete: true }; diff --git a/libs/common/src/lib/interfaces/admin-jobs.interface.ts b/libs/common/src/lib/interfaces/admin-jobs.interface.ts index 25e937626..b4c91ebc0 100644 --- a/libs/common/src/lib/interfaces/admin-jobs.interface.ts +++ b/libs/common/src/lib/interfaces/admin-jobs.interface.ts @@ -8,6 +8,7 @@ export interface AdminJobs { | 'finishedOn' | 'id' | 'name' + | 'opts' | 'stacktrace' | 'timestamp' > & {