From e62da06c5cfe0c311b676b5e13ddb42792f32cf5 Mon Sep 17 00:00:00 2001
From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
Date: Sat, 1 Jul 2023 11:08:10 +0200
Subject: [PATCH] Feature/extend scraper configuration support (#2110)
* Extend scraper configuration support
* Update changelog
---
CHANGELOG.md | 5 +++++
apps/api/src/app/admin/admin.service.ts | 2 ++
apps/api/src/app/admin/update-asset-profile.dto.ts | 5 +++++
.../services/data-provider/manual/manual.service.ts | 10 +++++++---
.../symbol-profile/symbol-profile.service.ts | 5 ++++-
.../asset-profile-dialog.component.ts | 13 +++++++++++++
.../asset-profile-dialog/asset-profile-dialog.html | 11 +++++++++++
apps/client/src/app/services/admin.service.ts | 3 ++-
.../interfaces/scraper-configuration.interface.ts | 1 +
9 files changed, 50 insertions(+), 5 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c7f58d03..72d092876 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added pagination to the historical market data table of the admin control panel
+- Added the attribute `headers` to the scraper configuration
+
+### Changed
+
+- Extended the asset profile details dialog in the admin control panel by the scraper configuration
## 1.284.0 - 2023-06-27
diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts
index 861cca019..4dce77982 100644
--- a/apps/api/src/app/admin/admin.service.ts
+++ b/apps/api/src/app/admin/admin.service.ts
@@ -249,12 +249,14 @@ export class AdminService {
public async patchAssetProfileData({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
await this.symbolProfileService.updateSymbolProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
});
diff --git a/apps/api/src/app/admin/update-asset-profile.dto.ts b/apps/api/src/app/admin/update-asset-profile.dto.ts
index 228aa6f2c..54f2d8f25 100644
--- a/apps/api/src/app/admin/update-asset-profile.dto.ts
+++ b/apps/api/src/app/admin/update-asset-profile.dto.ts
@@ -1,3 +1,4 @@
+import { Prisma } from '@prisma/client';
import { IsObject, IsOptional, IsString } from 'class-validator';
export class UpdateAssetProfileDto {
@@ -5,6 +6,10 @@ export class UpdateAssetProfileDto {
@IsOptional()
comment?: string;
+ @IsObject()
+ @IsOptional()
+ scraperConfiguration?: Prisma.InputJsonObject;
+
@IsObject()
@IsOptional()
symbolMapping?: {
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 14df4b7f4..7b3933532 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -67,8 +67,12 @@ export class ManualService implements DataProviderInterface {
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
[{ symbol, dataSource: this.getName() }]
);
- const { defaultMarketPrice, selector, url } =
- symbolProfile.scraperConfiguration ?? {};
+ const {
+ defaultMarketPrice,
+ headers = {},
+ selector,
+ url
+ } = symbolProfile.scraperConfiguration ?? {};
if (defaultMarketPrice) {
const historical: {
@@ -91,7 +95,7 @@ export class ManualService implements DataProviderInterface {
return {};
}
- const get = bent(url, 'GET', 'string', 200, {});
+ const get = bent(url, 'GET', 'string', 200, headers);
const html = await get();
const $ = cheerio.load(html);
diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
index 2f90771bd..99244c352 100644
--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts
+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
@@ -96,11 +96,12 @@ export class SymbolProfileService {
public updateSymbolProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
return this.prismaService.symbolProfile.update({
- data: { comment, symbolMapping },
+ data: { comment, scraperConfiguration, symbolMapping },
where: { dataSource_symbol: { dataSource, symbol } }
});
}
@@ -195,6 +196,8 @@ export class SymbolProfileService {
if (scraperConfiguration) {
return {
defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number,
+ headers:
+ scraperConfiguration.headers as ScraperConfiguration['headers'],
selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string
};
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
index fac98982b..be1892e91 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@@ -13,6 +13,7 @@ import { AdminService } from '@ghostfolio/client/services/admin.service';
import { DataService } from '@ghostfolio/client/services/data.service';
import {
AdminMarketDataDetails,
+ ScraperConfiguration,
UniqueAsset
} from '@ghostfolio/common/interfaces';
import { translate } from '@ghostfolio/ui/i18n';
@@ -34,6 +35,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
public assetProfile: AdminMarketDataDetails['assetProfile'];
public assetProfileForm = this.formBuilder.group({
comment: '',
+ scraperConfiguration: '',
symbolMapping: ''
});
public assetSubClass: string;
@@ -103,6 +105,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
this.assetProfileForm.setValue({
comment: this.assetProfile?.comment ?? '',
+ scraperConfiguration: JSON.stringify(
+ this.assetProfile?.scraperConfiguration ?? {}
+ ),
symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {})
});
@@ -148,8 +153,15 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
}
public onSubmit() {
+ let scraperConfiguration = {};
let symbolMapping = {};
+ try {
+ scraperConfiguration = JSON.parse(
+ this.assetProfileForm.controls['scraperConfiguration'].value
+ );
+ } catch {}
+
try {
symbolMapping = JSON.parse(
this.assetProfileForm.controls['symbolMapping'].value
@@ -157,6 +169,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
} catch {}
const assetProfileData: UpdateAssetProfileDto = {
+ scraperConfiguration,
symbolMapping,
comment: this.assetProfileForm.controls['comment'].value ?? null
};
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
index a6f782b50..be99df7cb 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@@ -162,6 +162,17 @@
>
+
+
+ Scraper Configuration
+
+
+
Note
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts
index 856fe9341..2ffb59b65 100644
--- a/apps/client/src/app/services/admin.service.ts
+++ b/apps/client/src/app/services/admin.service.ts
@@ -191,12 +191,13 @@ export class AdminService {
public patchAssetProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: UniqueAsset & UpdateAssetProfileDto) {
return this.http.patch(
`/api/v1/admin/profile-data/${dataSource}/${symbol}`,
- { comment, symbolMapping }
+ { comment, scraperConfiguration, symbolMapping }
);
}
diff --git a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
index 9bae19418..0446459ad 100644
--- a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
+++ b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
@@ -1,5 +1,6 @@
export interface ScraperConfiguration {
defaultMarketPrice?: number;
+ headers?: { [key: string]: string };
selector: string;
url: string;
}