From 7ea906185219f608473a1737894fbb0278a75c39 Mon Sep 17 00:00:00 2001
From: Shaunak Das <51281688+shaun-ak@users.noreply.github.com>
Date: Thu, 5 Sep 2024 00:47:38 +0530
Subject: [PATCH] Feature/execute scraper configuration instantly (#3723)

* Execute scraper configuration instantly

* Update changelog

---------

Co-authored-by: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
---
 CHANGELOG.md                                  |  1 +
 .../data-provider/manual/manual.service.ts    | 35 +++++++++++++++++--
 .../symbol-profile/symbol-profile.service.ts  |  2 ++
 .../scraper-configuration.interface.ts        |  1 +
 4 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d598db05c..4639d2485 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Set up a performance logging service
+- Added the attribute `mode` to the scraper configuration to get quotes instantly
 
 ### Changed
 
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 0655d2318..02e50a145 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -166,11 +166,42 @@ export class ManualService implements DataProviderInterface {
         }
       });
 
+      const symbolProfilesWithScraperConfigurationAndInstantMode =
+        symbolProfiles.filter(({ scraperConfiguration }) => {
+          return scraperConfiguration?.mode === 'instant';
+        });
+
+      const scraperResultPromises =
+        symbolProfilesWithScraperConfigurationAndInstantMode.map(
+          async ({ scraperConfiguration, symbol }) => {
+            try {
+              const marketPrice = await this.scrape(scraperConfiguration);
+              return { marketPrice, symbol };
+            } catch (error) {
+              Logger.error(
+                `Could not get quote for ${symbol} (${this.getName()}): [${error.name}] ${error.message}`,
+                'ManualService'
+              );
+              return { symbol, marketPrice: undefined };
+            }
+          }
+        );
+
+      // Wait for all scraping requests to complete concurrently
+      const scraperResults = await Promise.all(scraperResultPromises);
+
       for (const { currency, symbol } of symbolProfiles) {
-        let marketPrice =
+        let { marketPrice } =
+          scraperResults.find((result) => {
+            return result.symbol === symbol;
+          }) ?? {};
+
+        marketPrice =
+          marketPrice ??
           marketData.find((marketDataItem) => {
             return marketDataItem.symbol === symbol;
-          })?.marketPrice ?? 0;
+          })?.marketPrice ??
+          0;
 
         response[symbol] = {
           currency,
diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
index 50cb25000..283da7b52 100644
--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts
+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
@@ -275,6 +275,8 @@ export class SymbolProfileService {
         headers:
           scraperConfiguration.headers as ScraperConfiguration['headers'],
         locale: scraperConfiguration.locale as string,
+        mode:
+          (scraperConfiguration.mode as ScraperConfiguration['mode']) ?? 'lazy',
         selector: scraperConfiguration.selector as string,
         url: scraperConfiguration.url as string
       };
diff --git a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
index ef5506328..70fcd939d 100644
--- a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
+++ b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
@@ -2,6 +2,7 @@ export interface ScraperConfiguration {
   defaultMarketPrice?: number;
   headers?: { [key: string]: string };
   locale?: string;
+  mode?: 'instant' | 'lazy';
   selector: string;
   url: string;
 }