From 43b4f14acebb46cbc1e00c69676e844193fe092f Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:53:42 +0100 Subject: [PATCH] Feature/add button to test scraper configuration (#2808) * Add button to test scraper configuration * Update changelog --------- Co-authored-by: Manushreshta B L Co-authored-by: Hugo Persson --- CHANGELOG.md | 6 ++ apps/api/src/app/admin/admin.controller.ts | 29 +++++++++- .../data-provider/data-provider.module.ts | 2 +- .../data-provider/manual/manual.service.ts | 56 +++++++++++++------ .../asset-profile-dialog.component.ts | 28 ++++++++++ .../asset-profile-dialog.html | 24 ++++++-- apps/client/src/app/services/admin.service.ts | 13 +++++ 7 files changed, 134 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d9406281..c985bcaca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Added a button to test the scraper configuration in the asset profile details dialog of the admin control + ## 2.33.0 - 2023-12-31 ### Added diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index cc4da298b..634bbbfdc 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -3,6 +3,7 @@ import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard' import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor'; import { ApiService } from '@ghostfolio/api/services/api/api.service'; import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service'; +import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PropertyDto } from '@ghostfolio/api/services/property/property.dto'; import { @@ -31,6 +32,7 @@ import { Get, HttpException, Inject, + Logger, Param, Patch, Post, @@ -56,6 +58,7 @@ export class AdminController { private readonly adminService: AdminService, private readonly apiService: ApiService, private readonly dataGatheringService: DataGatheringService, + private readonly manualService: ManualService, private readonly marketDataService: MarketDataService, @Inject(REQUEST) private readonly request: RequestWithUser ) {} @@ -179,8 +182,8 @@ export class AdminController { } @Get('market-data') - @UseGuards(AuthGuard('jwt'), HasPermissionGuard) @HasPermission(permissions.accessAdminControl) + @UseGuards(AuthGuard('jwt'), HasPermissionGuard) public async getMarketData( @Query('assetSubClasses') filterByAssetSubClasses?: string, @Query('presetId') presetId?: MarketDataPreset, @@ -215,6 +218,30 @@ export class AdminController { return this.adminService.getMarketDataBySymbol({ dataSource, symbol }); } + @HasPermission(permissions.accessAdminControl) + @Post('market-data/:dataSource/:symbol/test') + @UseGuards(AuthGuard('jwt'), HasPermissionGuard) + public async testMarketData( + @Body() data: { scraperConfiguration: string }, + @Param('dataSource') dataSource: DataSource, + @Param('symbol') symbol: string + ): Promise<{ price: number }> { + try { + const { headers, selector, url } = JSON.parse(data.scraperConfiguration); + const price = await this.manualService.test({ headers, selector, url }); + + if (price) { + return { price }; + } + + throw new Error('Could not parse the current market price'); + } catch (error) { + Logger.error(error); + + throw new HttpException(error.message, StatusCodes.BAD_REQUEST); + } + } + @HasPermission(permissions.accessAdminControl) @Post('market-data/:dataSource/:symbol') @UseGuards(AuthGuard('jwt'), HasPermissionGuard) diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index b3a219a50..e63ec0807 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -74,6 +74,6 @@ import { DataProviderService } from './data-provider.service'; }, YahooFinanceDataEnhancerService ], - exports: [DataProviderService, YahooFinanceService] + exports: [DataProviderService, ManualService, YahooFinanceService] }) export class DataProviderModule {} diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 9a0ff82d9..d5a5c7eb3 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -18,7 +18,7 @@ import { DataSource, SymbolProfile } from '@prisma/client'; import * as cheerio from 'cheerio'; import { isUUID } from 'class-validator'; import { addDays, format, isBefore } from 'date-fns'; -import got from 'got'; +import got, { Headers } from 'got'; @Injectable() export class ManualService implements DataProviderInterface { @@ -97,21 +97,7 @@ export class ManualService implements DataProviderInterface { return {}; } - const abortController = new AbortController(); - - setTimeout(() => { - abortController.abort(); - }, this.configurationService.get('REQUEST_TIMEOUT')); - - const { body } = await got(url, { - headers, - // @ts-ignore - signal: abortController.signal - }); - - const $ = cheerio.load(body); - - const value = extractNumberFromString($(selector).text()); + const value = await this.scrape({ headers, selector, url }); return { [symbol]: { @@ -233,4 +219,42 @@ export class ManualService implements DataProviderInterface { return { items }; } + + public async test(params: any) { + return this.scrape({ + headers: params.headers, + selector: params.selector, + url: params.url + }); + } + + private async scrape({ + headers = {}, + selector, + url + }: { + headers?: Headers; + selector: string; + url: string; + }): Promise { + try { + const abortController = new AbortController(); + + setTimeout(() => { + abortController.abort(); + }, this.configurationService.get('REQUEST_TIMEOUT')); + + const { body } = await got(url, { + headers, + // @ts-ignore + signal: abortController.signal + }); + + const $ = cheerio.load(body); + + return extractNumberFromString($(selector).first().text()); + } catch (error) { + throw error; + } + } } diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts index 6421e888b..6a6e4e64a 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts @@ -277,6 +277,34 @@ export class AssetProfileDialog implements OnDestroy, OnInit { }); } + public onTestMarketData() { + this.adminService + .testMarketData({ + dataSource: this.data.dataSource, + scraperConfiguration: + this.assetProfileForm.controls['scraperConfiguration'].value, + symbol: this.data.symbol + }) + .pipe( + catchError(({ error }) => { + alert(`Error: ${error?.message}`); + return EMPTY; + }), + takeUntil(this.unsubscribeSubject) + ) + .subscribe(({ price }) => { + alert( + $localize`The current market price is` + + ' ' + + price + + ' ' + + (( + (this.assetProfileForm.controls['currency'].value) + ))?.value + ); + }); + } + public onUnsetBenchmark({ dataSource, symbol }: UniqueAsset) { this.dataService .deleteBenchmark({ dataSource, symbol }) diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html index 08051a0ef..3ff71154f 100644 --- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html +++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html @@ -243,12 +243,24 @@
Scraper Configuration - +
+ + +
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts index 9f8fc9d13..a1c7d626a 100644 --- a/apps/client/src/app/services/admin.service.ts +++ b/apps/client/src/app/services/admin.service.ts @@ -259,4 +259,17 @@ export class AdminService { public putTag(aTag: UpdateTagDto) { return this.http.put(`/api/v1/tag/${aTag.id}`, aTag); } + + public testMarketData({ + dataSource, + scraperConfiguration, + symbol + }: UniqueAsset & UpdateAssetProfileDto['scraperConfiguration']) { + return this.http.post( + `/api/v1/admin/market-data/${dataSource}/${symbol}/test`, + { + scraperConfiguration + } + ); + } }