Feature/setup bull queue system (#886)

* Setup @nestjs/bull and asset profile data gathering job

* Update changelog

CHANGELOG.md

@@ -7,8 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Set up a queue for the data gathering jobs
+
 ### Changed
 
+- Migrated the asset profile data gathering to the queue design pattern
 - Harmonized the _No data available_ label in the portfolio proportion chart component
 
 ## 1.145.0 - 07.05.2022

apps/api/src/app/admin/admin.controller.ts

@@ -1,6 +1,10 @@
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
+import {
+  DATA_GATHERING_QUEUE,
+  GATHER_ASSET_PROFILE_PROCESS
+} from '@ghostfolio/common/config';
 import {
   AdminData,
   AdminMarketData,
@@ -8,6 +12,7 @@ import {
 } from '@ghostfolio/common/interfaces';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { RequestWithUser } from '@ghostfolio/common/types';
+import { InjectQueue } from '@nestjs/bull';
 import {
   Body,
   Controller,
@@ -23,6 +28,7 @@ import {
 import { REQUEST } from '@nestjs/core';
 import { AuthGuard } from '@nestjs/passport';
 import { DataSource, MarketData } from '@prisma/client';
+import { Queue } from 'bull';
 import { isDate } from 'date-fns';
 import { StatusCodes, getReasonPhrase } from 'http-status-codes';
 
@@ -33,6 +39,8 @@ import { UpdateMarketDataDto } from './update-market-data.dto';
 export class AdminController {
   public constructor(
     private readonly adminService: AdminService,
+    @InjectQueue(DATA_GATHERING_QUEUE)
+    private readonly dataGatheringQueue: Queue,
     private readonly dataGatheringService: DataGatheringService,
     private readonly marketDataService: MarketDataService,
     @Inject(REQUEST) private readonly request: RequestWithUser
@@ -71,10 +79,16 @@ export class AdminController {
       );
     }
 
-    await this.dataGatheringService.gatherProfileData();
-    this.dataGatheringService.gatherMax();
+    const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    return;
+    for (const { dataSource, symbol } of uniqueAssets) {
+      await this.dataGatheringQueue.add(GATHER_ASSET_PROFILE_PROCESS, {
+        dataSource,
+        symbol
+      });
+    }
+
+    this.dataGatheringService.gatherMax();
   }
 
   @Post('gather/profile-data')
@@ -92,9 +106,14 @@ export class AdminController {
       );
     }
 
-    this.dataGatheringService.gatherProfileData();
+    const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    return;
+    for (const { dataSource, symbol } of uniqueAssets) {
+      await this.dataGatheringQueue.add(GATHER_ASSET_PROFILE_PROCESS, {
+        dataSource,
+        symbol
+      });
+    }
   }
 
   @Post('gather/profile-data/:dataSource/:symbol')
@@ -115,9 +134,10 @@ export class AdminController {
       );
     }
 
-    this.dataGatheringService.gatherProfileData([{ dataSource, symbol }]);
-
-    return;
+    await this.dataGatheringQueue.add(GATHER_ASSET_PROFILE_PROCESS, {
+      dataSource,
+      symbol
+    });
   }
 
   @Post('gather/:dataSource/:symbol')

apps/api/src/app/admin/admin.service.ts

@@ -15,7 +15,7 @@ import {
   UniqueAsset
 } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
-import { DataSource, Property } from '@prisma/client';
+import { Property } from '@prisma/client';
 import { differenceInDays } from 'date-fns';
 
 @Injectable()

apps/api/src/app/app.module.ts

@@ -9,6 +9,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
 import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
 import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module';
+import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { ScheduleModule } from '@nestjs/schedule';
@@ -36,6 +37,12 @@ import { UserModule } from './user/user.module';
     AccountModule,
     AuthDeviceModule,
     AuthModule,
+    BullModule.forRoot({
+      redis: {
+        host: process.env.REDIS_HOST,
+        port: parseInt(process.env.REDIS_PORT, 10)
+      }
+    }),
     CacheModule,
     ConfigModule.forRoot(),
     ConfigurationModule,

apps/api/src/app/order/order.service.ts

@@ -4,8 +4,13 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import {
+  DATA_GATHERING_QUEUE,
+  GATHER_ASSET_PROFILE_PROCESS
+} from '@ghostfolio/common/config';
 import { Filter } from '@ghostfolio/common/interfaces';
 import { OrderWithAccount } from '@ghostfolio/common/types';
+import { InjectQueue } from '@nestjs/bull';
 import { Injectable } from '@nestjs/common';
 import {
   AssetClass,
@@ -16,6 +21,7 @@ import {
   Type as TypeOfOrder
 } from '@prisma/client';
 import Big from 'big.js';
+import { Queue } from 'bull';
 import { endOfToday, isAfter } from 'date-fns';
 import { groupBy } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
@@ -27,6 +33,8 @@ export class OrderService {
   public constructor(
     private readonly accountService: AccountService,
     private readonly cacheService: CacheService,
+    @InjectQueue(DATA_GATHERING_QUEUE)
+    private readonly dataGatheringQueue: Queue,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly dataGatheringService: DataGatheringService,
     private readonly prismaService: PrismaService,
@@ -112,12 +120,10 @@ export class OrderService {
         data.SymbolProfile.connectOrCreate.create.symbol.toUpperCase();
     }
 
-    await this.dataGatheringService.gatherProfileData([
-      {
-        dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
-        symbol: data.SymbolProfile.connectOrCreate.create.symbol
-      }
-    ]);
+    await this.dataGatheringQueue.add(GATHER_ASSET_PROFILE_PROCESS, {
+      dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
+      symbol: data.SymbolProfile.connectOrCreate.create.symbol
+    });
 
     const isDraft = isAfter(data.date as Date, endOfToday());
 

apps/api/src/services/cron.service.ts

@@ -1,5 +1,11 @@
+import {
+  DATA_GATHERING_QUEUE,
+  GATHER_ASSET_PROFILE_PROCESS
+} from '@ghostfolio/common/config';
+import { InjectQueue } from '@nestjs/bull';
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
+import { Queue } from 'bull';
 
 import { DataGatheringService } from './data-gathering.service';
 import { ExchangeRateDataService } from './exchange-rate-data.service';
@@ -8,6 +14,8 @@ import { TwitterBotService } from './twitter-bot/twitter-bot.service';
 @Injectable()
 export class CronService {
   public constructor(
+    @InjectQueue(DATA_GATHERING_QUEUE)
+    private readonly dataGatheringQueue: Queue,
     private readonly dataGatheringService: DataGatheringService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly twitterBotService: TwitterBotService
@@ -30,6 +38,13 @@ export class CronService {
 
   @Cron(CronExpression.EVERY_WEEKEND)
   public async runEveryWeekend() {
-    await this.dataGatheringService.gatherProfileData();
+    const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
+
+    for (const { dataSource, symbol } of uniqueAssets) {
+      await this.dataGatheringQueue.add(GATHER_ASSET_PROFILE_PROCESS, {
+        dataSource,
+        symbol
+      });
+    }
   }
 }

apps/api/src/services/data-gathering.module.ts

@@ -3,13 +3,19 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se
 import { DataEnhancerModule } from '@ghostfolio/api/services/data-provider/data-enhancer/data-enhancer.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
 import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { DATA_GATHERING_QUEUE } from '@ghostfolio/common/config';
+import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 
+import { DataGatheringProcessor } from './data-gathering.processor';
 import { ExchangeRateDataModule } from './exchange-rate-data.module';
 import { SymbolProfileModule } from './symbol-profile.module';
 
 @Module({
   imports: [
+    BullModule.registerQueue({
+      name: DATA_GATHERING_QUEUE
+    }),
     ConfigurationModule,
     DataEnhancerModule,
     DataProviderModule,
@@ -17,7 +23,7 @@ import { SymbolProfileModule } from './symbol-profile.module';
     PrismaModule,
     SymbolProfileModule
   ],
-  providers: [DataGatheringService],
-  exports: [DataEnhancerModule, DataGatheringService]
+  providers: [DataGatheringProcessor, DataGatheringService],
+  exports: [BullModule, DataEnhancerModule, DataGatheringService]
 })
 export class DataGatheringModule {}

apps/api/src/services/data-gathering.processor.ts

@@ -0,0 +1,27 @@
+import {
+  DATA_GATHERING_QUEUE,
+  GATHER_ASSET_PROFILE_PROCESS
+} from '@ghostfolio/common/config';
+import { UniqueAsset } from '@ghostfolio/common/interfaces';
+import { Process, Processor } from '@nestjs/bull';
+import { Injectable, Logger } from '@nestjs/common';
+import { Job } from 'bull';
+
+import { DataGatheringService } from './data-gathering.service';
+
+@Injectable()
+@Processor(DATA_GATHERING_QUEUE)
+export class DataGatheringProcessor {
+  public constructor(
+    private readonly dataGatheringService: DataGatheringService
+  ) {}
+
+  @Process(GATHER_ASSET_PROFILE_PROCESS)
+  public async gatherAssetProfile(job: Job<UniqueAsset>) {
+    try {
+      await this.dataGatheringService.gatherAssetProfiles([job.data]);
+    } catch (error) {
+      Logger.error(error, 'DataGatheringProcessor');
+    }
+  }
+}

apps/api/src/services/data-gathering.service.ts

@@ -226,28 +226,29 @@ export class DataGatheringService {
     }
   }
 
-  public async gatherProfileData(aDataGatheringItems?: IDataGatheringItem[]) {
+  public async gatherAssetProfiles(aUniqueAssets?: UniqueAsset[]) {
+    let uniqueAssets = aUniqueAssets?.filter((dataGatheringItem) => {
+      return dataGatheringItem.dataSource !== 'MANUAL';
+    });
+
+    if (!uniqueAssets) {
+      uniqueAssets = await this.getUniqueAssets();
+    }
+
     Logger.log(
-      'Profile data gathering has been started.',
+      `Asset profile data gathering has been started for ${uniqueAssets
+        .map(({ dataSource, symbol }) => {
+          return `${symbol} (${dataSource})`;
+        })
+        .join(',')}.`,
       'DataGatheringService'
     );
-    console.time('data-gathering-profile');
-
-    let dataGatheringItems = aDataGatheringItems?.filter(
-      (dataGatheringItem) => {
-        return dataGatheringItem.dataSource !== 'MANUAL';
-      }
-    );
-
-    if (!dataGatheringItems) {
-      dataGatheringItems = await this.getSymbolsProfileData();
-    }
 
     const assetProfiles = await this.dataProviderService.getAssetProfiles(
-      dataGatheringItems
+      uniqueAssets
     );
     const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
-      dataGatheringItems.map(({ symbol }) => {
+      uniqueAssets.map(({ symbol }) => {
         return symbol;
       })
     );
@@ -322,10 +323,13 @@ export class DataGatheringService {
     }
 
     Logger.log(
-      'Profile data gathering has been completed.',
+      `Asset profile data gathering has been completed for ${uniqueAssets
+        .map(({ dataSource, symbol }) => {
+          return `${symbol} (${dataSource})`;
+        })
+        .join(',')}.`,
       'DataGatheringService'
     );
-    console.timeEnd('data-gathering-profile');
   }
 
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
@@ -508,6 +512,27 @@ export class DataGatheringService {
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
+  public async getUniqueAssets(): Promise<UniqueAsset[]> {
+    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
+      orderBy: [{ symbol: 'asc' }]
+    });
+
+    return symbolProfiles
+      .filter(({ dataSource }) => {
+        return (
+          dataSource !== DataSource.GHOSTFOLIO &&
+          dataSource !== DataSource.MANUAL &&
+          dataSource !== DataSource.RAKUTEN
+        );
+      })
+      .map(({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol
+        };
+      });
+  }
+
   public async reset() {
     Logger.log('Data gathering has been reset.', 'DataGatheringService');
 
@@ -584,27 +609,6 @@ export class DataGatheringService {
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
-  private async getSymbolsProfileData(): Promise<IDataGatheringItem[]> {
-    const symbolProfiles = await this.prismaService.symbolProfile.findMany({
-      orderBy: [{ symbol: 'asc' }]
-    });
-
-    return symbolProfiles
-      .filter((symbolProfile) => {
-        return (
-          symbolProfile.dataSource !== DataSource.GHOSTFOLIO &&
-          symbolProfile.dataSource !== DataSource.MANUAL &&
-          symbolProfile.dataSource !== DataSource.RAKUTEN
-        );
-      })
-      .map((symbolProfile) => {
-        return {
-          dataSource: symbolProfile.dataSource,
-          symbol: symbolProfile.symbol
-        };
-      });
-  }
-
   private async isDataGatheringNeeded() {
     const lastDataGathering = await this.getLastDataGathering();
 

libs/common/src/lib/config.ts

@@ -44,8 +44,12 @@ export const warnColorRgb = {
 
 export const ASSET_SUB_CLASS_EMERGENCY_FUND = 'EMERGENCY_FUND';
 
+export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE';
+
 export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
 
+export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';
+
 export const PROPERTY_COUPONS = 'COUPONS';
 export const PROPERTY_CURRENCIES = 'CURRENCIES';
 export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE';

libs/common/src/lib/interfaces/info-item.interface.ts

@@ -1,4 +1,5 @@
 import { Tag } from '@prisma/client';
+
 import { Statistics } from './statistics.interface';
 import { Subscription } from './subscription.interface';
 

libs/ui/src/lib/activities-table/activities-table.component.ts

@@ -20,7 +20,7 @@ import Big from 'big.js';
 import { isUUID } from 'class-validator';
 import { endOfToday, format, isAfter } from 'date-fns';
 import { isNumber } from 'lodash';
-import { distinctUntilChanged, Subject, Subscription, takeUntil } from 'rxjs';
+import { Subject, Subscription, distinctUntilChanged, takeUntil } from 'rxjs';
 
 const SEARCH_PLACEHOLDER = 'Search for account, currency, symbol or type...';
 const SEARCH_STRING_SEPARATOR = ',';

package.json

@@ -62,6 +62,7 @@
     "@angular/router": "13.2.2",
     "@codewithdan/observable-store": "2.2.11",
     "@dinero.js/currencies": "2.0.0-alpha.8",
+    "@nestjs/bull": "0.5.5",
     "@nestjs/common": "8.2.3",
     "@nestjs/config": "1.1.3",
     "@nestjs/core": "8.2.3",
@@ -82,6 +83,7 @@
     "bent": "7.3.12",
     "big.js": "6.1.1",
     "bootstrap": "4.6.0",
+    "bull": "4.8.2",
     "cache-manager": "3.4.3",
     "cache-manager-redis-store": "2.0.0",
     "chart.js": "3.7.0",
@@ -146,6 +148,7 @@
     "@storybook/builder-webpack5": "6.4.18",
     "@storybook/manager-webpack5": "6.4.18",
     "@types/big.js": "6.1.2",
+    "@types/bull": "3.15.8",
     "@types/cache-manager": "3.4.2",
     "@types/color": "3.0.2",
     "@types/google-spreadsheet": "3.1.5",

yarn.lock

@@ -3012,6 +3012,21 @@
     call-me-maybe "^1.0.1"
     glob-to-regexp "^0.3.0"
 
+"@nestjs/bull-shared@^0.0.5":
+  version "0.0.5"
+  resolved "https://registry.yarnpkg.com/@nestjs/bull-shared/-/bull-shared-0.0.5.tgz#5cd9907fdc99cdd2a2088406bcae086e13918c4a"
+  integrity sha512-MPkl1q7N/ZlzLq4SC/NWdZ+7pIOF9x70+92xBcUPx7J6mewqVR1gzADHzEexErZEgg+K/n5nwJGe+BLlYFTUgg==
+  dependencies:
+    tslib "2.3.1"
+
+"@nestjs/bull@0.5.5":
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/@nestjs/bull/-/bull-0.5.5.tgz#1ee9b7d8b78a52af58c4654de8d765a2a2b16862"
+  integrity sha512-g42/KH8YJmJE60kQVzQucWm5xEfyOj8iwXp2s3Ox3Yu6B/mmzNmqmXLnACnaD15q3WtH84vOhME41CJANxbQeQ==
+  dependencies:
+    "@nestjs/bull-shared" "^0.0.5"
+    tslib "2.3.1"
+
 "@nestjs/common@8.2.3":
   version "8.2.3"
   resolved "https://registry.yarnpkg.com/@nestjs/common/-/common-8.2.3.tgz#77ac1ef59c36dd9ae131f5c5ddabfc6258802d1c"
@@ -4753,6 +4768,14 @@
   dependencies:
     "@types/node" "*"
 
+"@types/bull@3.15.8":
+  version "3.15.8"
+  resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.8.tgz#ae2139f94490d740b37c8da5d828ce75dd82ce7c"
+  integrity sha512-8DbSPMSsZH5PWPnGEkAZLYgJEH4ghHJNKF7LB6Wr5R0/v6g+Vs+JoaA7kcvLtHE936xg2WpFPkaoaJgExOmKDw==
+  dependencies:
+    "@types/ioredis" "*"
+    "@types/redis" "^2.8.0"
+
 "@types/cache-manager@3.4.2":
   version "3.4.2"
   resolved "https://registry.yarnpkg.com/@types/cache-manager/-/cache-manager-3.4.2.tgz#d57e7e5e6374d1037bdce753a05c9703e4483401"
@@ -4880,6 +4903,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/ioredis@*":
+  version "4.28.10"
+  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
+  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/is-function@^1.0.0":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@types/is-function/-/is-function-1.0.0.tgz#1b0b819b1636c7baf0d6785d030d12edf70c3e83"
@@ -5084,6 +5114,13 @@
     "@types/scheduler" "*"
     csstype "^3.0.2"
 
+"@types/redis@^2.8.0":
+  version "2.8.32"
+  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
+  integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
+  dependencies:
+    "@types/node" "*"
+
 "@types/retry@^0.12.0":
   version "0.12.1"
   resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.1.tgz#d8f1c0d0dc23afad6dc16a9e993a0865774b4065"
@@ -6945,6 +6982,21 @@ builtins@^1.0.3:
   resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88"
   integrity sha1-y5T662HIaWRR2zZTThQi+U8K7og=
 
+bull@4.8.2:
+  version "4.8.2"
+  resolved "https://registry.yarnpkg.com/bull/-/bull-4.8.2.tgz#0d02fe389777abe29d50fd46d123bc62e074cfcd"
+  integrity sha512-S7CNIL9+vsbLKwOGkUI6mawY5iABKQJLZn5a7KPnxAZrDhFXkrxsHHXLCKUR/+Oqys3Vk5ElWdj0SLtK84b1Nw==
+  dependencies:
+    cron-parser "^4.2.1"
+    debuglog "^1.0.0"
+    get-port "^5.1.1"
+    ioredis "^4.28.5"
+    lodash "^4.17.21"
+    msgpackr "^1.5.2"
+    p-timeout "^3.2.0"
+    semver "^7.3.2"
+    uuid "^8.3.0"
+
 busboy@^0.2.11:
   version "0.2.14"
   resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.2.14.tgz#6c2a622efcf47c57bbbe1e2a9c37ad36c7925453"
@@ -7500,6 +7552,11 @@ clsx@^1.1.1:
   resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188"
   integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==
 
+cluster-key-slot@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
+
 co@^4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
@@ -7978,6 +8035,13 @@ critters@0.0.16:
     postcss "^8.3.7"
     pretty-bytes "^5.3.0"
 
+cron-parser@^4.2.1:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.4.0.tgz#829d67f9e68eb52fa051e62de0418909f05db983"
+  integrity sha512-TrE5Un4rtJaKgmzPewh67yrER5uKM0qI9hGLDBfWb8GGRe9pn/SDkhVrdHa4z7h0SeyeNxnQnogws/H+AQANQA==
+  dependencies:
+    luxon "^1.28.0"
+
 cron@1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/cron/-/cron-1.7.2.tgz#2ea1f35c138a07edac2ac5af5084ed6fee5723db"
@@ -8303,6 +8367,11 @@ debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.6, debug@^3.2.7:
   dependencies:
     ms "^2.1.1"
 
+debuglog@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
+  integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=
+
 decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -8422,7 +8491,7 @@ delegates@^1.0.0:
   resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
   integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
 
-denque@^1.5.0:
+denque@^1.1.0, denque@^1.5.0:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
   integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
@@ -10284,6 +10353,11 @@ get-package-type@^0.1.0:
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
   integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==
 
+get-port@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193"
+  integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==
+
 get-stream@^4.0.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@@ -11325,6 +11399,23 @@ ionicons@5.5.1:
   dependencies:
     "@stencil/core" "^2.5.0"
 
+ioredis@^4.28.5:
+  version "4.28.5"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
+  integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    lodash.isarguments "^3.1.0"
+    p-map "^2.1.0"
+    redis-commands "1.7.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 ip@^1.1.0, ip@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
@@ -13041,6 +13132,16 @@ lodash.debounce@^4.0.8:
   resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
   integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168=
 
+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
+
+lodash.flatten@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
+
 lodash.get@4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
@@ -13056,6 +13157,11 @@ lodash.includes@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
   integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=
 
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
+
 lodash.isboolean@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
@@ -13171,6 +13277,11 @@ lru-cache@^7.3.1:
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.3.1.tgz#7702e80694ec2bf19865567a469f2b081fcf53f5"
   integrity sha512-nX1x4qUrKqwbIAhv4s9et4FIUVzNOpeY07bsjGUy8gwJrXH/wScImSQqXErmo/b2jZY2r0mohbLA9zVj7u1cNw==
 
+luxon@^1.28.0:
+  version "1.28.0"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.28.0.tgz#e7f96daad3938c06a62de0fb027115d251251fbf"
+  integrity sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==
+
 magic-string@0.25.7, magic-string@^0.25.0:
   version "0.25.7"
   resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051"
@@ -13681,6 +13792,57 @@ ms@^2.0.0, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
+msgpackr-extract-darwin-arm64@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-1.1.0.tgz#d590dffac6b90edc3ab53392f7ec5668ed94638c"
+  integrity sha512-s1kHoT12tS2cCQOv+Wl3I+/cYNJXBPtwQqGA+dPYoXmchhXiE0Nso+BIfvQ5PxbmAyjj54Q5o7PnLTqVquNfZA==
+
+msgpackr-extract-darwin-x64@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-1.1.0.tgz#568cbdf5e819ac120659c02b0dbaabf483523ee3"
+  integrity sha512-yx/H/i12IKg4eWGu/eKdKzJD4jaYvvujQSaVmeOMCesbSQnWo5X6YR9TFjoiNoU9Aexk1KufzL9gW+1DozG1yw==
+
+msgpackr-extract-linux-arm64@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-1.1.0.tgz#c0a30e6687cea4f79115f5762c5fdff90e4a20d4"
+  integrity sha512-AxFle3fHNwz2V4CYDIGFxI6o/ZuI0lBKg0uHI8EcCMUmDE5mVAUWYge5WXmORVvb8sVWyVgFlmi3MTu4Ve6tNQ==
+
+msgpackr-extract-linux-arm@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-1.1.0.tgz#38e8db873b6b3986558bde4d7bb15eacc8743a9e"
+  integrity sha512-0VvSCqi12xpavxl14gMrauwIzHqHbmSChUijy/uo3mpjB1Pk4vlisKpZsaOZvNJyNKj0ACi5jYtbWnnOd7hYGw==
+
+msgpackr-extract-linux-x64@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-1.1.0.tgz#8c44ca5211d9fa6af77be64a8e687c0be0491ce7"
+  integrity sha512-O+XoyNFWpdB8oQL6O/YyzffPpmG5rTNrr1nKLW70HD2ENJUhcITzbV7eZimHPzkn8LAGls1tBaMTHQezTBpFOw==
+
+msgpackr-extract-win32-x64@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-1.1.0.tgz#7bf9bd258e334668842c7532e5e40a60ca3325d7"
+  integrity sha512-6AJdM5rNsL4yrskRfhujVSPEd6IBpgvsnIT/TPowKNLQ62iIdryizPY2PJNFiW3AJcY249AHEiDBXS1cTDPxzA==
+
+msgpackr-extract@^1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.1.4.tgz#665037c1470f225d01d2d735dad0334fff5faae6"
+  integrity sha512-WQbHvsThprXh+EqZYy+SQFEs7z6bNM7a0vgirwUfwUcphWGT2mdPcpyLCNiRsN6w5q5VKJUMblHY+tNEyceb9Q==
+  dependencies:
+    node-gyp-build-optional-packages "^4.3.2"
+  optionalDependencies:
+    msgpackr-extract-darwin-arm64 "1.1.0"
+    msgpackr-extract-darwin-x64 "1.1.0"
+    msgpackr-extract-linux-arm "1.1.0"
+    msgpackr-extract-linux-arm64 "1.1.0"
+    msgpackr-extract-linux-x64 "1.1.0"
+    msgpackr-extract-win32-x64 "1.1.0"
+
+msgpackr@^1.5.2:
+  version "1.5.7"
+  resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.5.7.tgz#53b3fd0e7afdf4184a594881a18832df9422b660"
+  integrity sha512-Hsa80i8W4BiObSMHslfnwC+CC1CYHZzoXJZn0+3EvoCEOgt3c5QlXhdcjgFk2aZxMgpV8aUFZqJyQUCIp4UrzA==
+  optionalDependencies:
+    msgpackr-extract "^1.1.4"
+
 multer@1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.3.tgz#4db352d6992e028ac0eacf7be45c6efd0264297b"
@@ -13864,6 +14026,11 @@ node-forge@^1.2.0:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.2.1.tgz#82794919071ef2eb5c509293325cec8afd0fd53c"
   integrity sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==
 
+node-gyp-build-optional-packages@^4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz#82de9bdf9b1ad042457533afb2f67469dc2264bb"
+  integrity sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==
+
 node-gyp-build@^4.2.2:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739"
@@ -14445,7 +14612,7 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
-p-map@^2.0.0:
+p-map@^2.0.0, p-map@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
   integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
@@ -14472,7 +14639,7 @@ p-retry@^4.5.0:
     "@types/retry" "^0.12.0"
     retry "^0.13.1"
 
-p-timeout@^3.1.0:
+p-timeout@^3.1.0, p-timeout@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
   integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
@@ -15811,7 +15978,7 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
-redis-commands@^1.7.0:
+redis-commands@1.7.0, redis-commands@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
   integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
@@ -16948,6 +17115,11 @@ stack-utils@^2.0.3:
   dependencies:
     escape-string-regexp "^2.0.0"
 
+standard-as-callback@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+  integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
 state-toggle@^1.0.0:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe"
@@ -18166,7 +18338,7 @@ uuid-browser@^3.1.0:
   resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410"
   integrity sha1-DwWkCu90+eWVHiDvv0SxGHHlZBA=
 
-uuid@8.3.2, uuid@^8.3.2:
+uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==