diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7b07508..59d40cfa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Improved the validation of `json` files in the import functionality for transactions +- Moved the scraper configuration to the symbol profile model + +### Todo + +- Apply data migration (`yarn database:migrate`) ## 1.69.0 - 07.11.2021 diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 1e3ecb478..d5f5c7770 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -272,21 +272,6 @@ export class DataGatheringService { } } - public async getCustomSymbolsToGather( - startDate?: Date - ): Promise { - const scraperConfigurations = - await this.ghostfolioScraperApi.getScraperConfigurations(); - - return scraperConfigurations.map((scraperConfiguration) => { - return { - dataSource: DataSource.GHOSTFOLIO, - date: startDate, - symbol: scraperConfiguration.symbol - }; - }); - } - public async getIsInProgress() { return await this.prismaService.property.findUnique({ where: { key: 'LOCKED_DATA_GATHERING' } @@ -343,6 +328,7 @@ export class DataGatheringService { orderBy: [{ symbol: 'asc' }], select: { dataSource: true, + scraperConfiguration: true, symbol: true } }) @@ -363,12 +349,8 @@ export class DataGatheringService { }; }); - const customSymbolsToGather = - await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate); - return [ ...this.getBenchmarksToGather(startDate), - ...customSymbolsToGather, ...currencyPairsToGather, ...symbolProfilesToGather ]; @@ -382,9 +364,6 @@ export class DataGatheringService { }) )?.date ?? new Date(); - const customSymbolsToGather = - await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate); - const currencyPairsToGather = this.exchangeRateDataService .getCurrencyPairs() .map(({ dataSource, symbol }) => { @@ -405,20 +384,19 @@ export class DataGatheringService { select: { date: true }, take: 1 }, + scraperConfiguration: true, symbol: true } }) - ).map((item) => { + ).map((symbolProfile) => { return { - dataSource: item.dataSource, - date: item.Order?.[0]?.date ?? startDate, - symbol: item.symbol + ...symbolProfile, + date: symbolProfile.Order?.[0]?.date ?? startDate }; }); return [ ...this.getBenchmarksToGather(startDate), - ...customSymbolsToGather, ...currencyPairsToGather, ...symbolProfilesToGather ]; diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index cacf2bf80..37afb3abf 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -4,13 +4,19 @@ import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provi import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; +import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module'; import { Module } from '@nestjs/common'; import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service'; import { DataProviderService } from './data-provider.service'; @Module({ - imports: [ConfigurationModule, CryptocurrencyModule, PrismaModule], + imports: [ + ConfigurationModule, + CryptocurrencyModule, + PrismaModule, + SymbolProfileModule + ], providers: [ AlphaVantageService, DataProviderService, diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 6c1a313d6..7c30b3dd3 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -1,5 +1,6 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; +import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service'; import { DATE_FORMAT, getYesterday, @@ -13,19 +14,20 @@ import * as cheerio from 'cheerio'; import { format } from 'date-fns'; import { - IDataGatheringItem, IDataProviderHistoricalResponse, IDataProviderResponse, MarketState } from '../../interfaces/interfaces'; import { DataProviderInterface } from '../interfaces/data-provider.interface'; -import { ScraperConfig } from './interfaces/scraper-config.interface'; @Injectable() export class GhostfolioScraperApiService implements DataProviderInterface { private static NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g; - public constructor(private readonly prismaService: PrismaService) {} + public constructor( + private readonly prismaService: PrismaService, + private readonly symbolProfileService: SymbolProfileService + ) {} public canHandle(symbol: string) { return isGhostfolioScraperApiSymbol(symbol); @@ -39,9 +41,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface { } try { - const symbol = aSymbols[0]; - - const scraperConfig = await this.getScraperConfigurationBySymbol(symbol); + const [symbol] = aSymbols; + const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( + [symbol] + ); const { marketPrice } = await this.prismaService.marketData.findFirst({ orderBy: { @@ -55,7 +58,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return { [symbol]: { marketPrice, - currency: scraperConfig?.currency, + currency: symbolProfile?.currency, dataSource: DataSource.GHOSTFOLIO, marketState: MarketState.delayed } @@ -67,25 +70,6 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return {}; } - public async getCustomSymbolsToGather( - startDate?: Date - ): Promise { - const ghostfolioSymbolProfiles = - await this.prismaService.symbolProfile.findMany({ - where: { - dataSource: DataSource.GHOSTFOLIO - } - }); - - return ghostfolioSymbolProfiles.map(({ dataSource, symbol }) => { - return { - dataSource, - symbol, - date: startDate - }; - }); - } - public async getHistorical( aSymbols: string[], aGranularity: Granularity = 'day', @@ -99,11 +83,11 @@ export class GhostfolioScraperApiService implements DataProviderInterface { } try { - const symbol = aSymbols[0]; - - const scraperConfiguration = await this.getScraperConfigurationBySymbol( - symbol + const [symbol] = aSymbols; + const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( + [symbol] ); + const scraperConfiguration = symbolProfile?.scraperConfiguration; const get = bent(scraperConfiguration?.url, 'GET', 'string', 200, {}); @@ -128,22 +112,6 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return {}; } - public async getScraperConfigurations(): Promise { - try { - const { value: scraperConfigString } = - await this.prismaService.property.findFirst({ - select: { - value: true - }, - where: { key: 'SCRAPER_CONFIG' } - }); - - return JSON.parse(scraperConfigString); - } catch {} - - return []; - } - public getName(): DataSource { return DataSource.GHOSTFOLIO; } @@ -162,11 +130,4 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return undefined; } } - - private async getScraperConfigurationBySymbol(aSymbol: string) { - const scraperConfigurations = await this.getScraperConfigurations(); - return scraperConfigurations.find((scraperConfiguration) => { - return scraperConfiguration.symbol === aSymbol; - }); - } } diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts deleted file mode 100644 index a6229bb85..000000000 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface ScraperConfig { - currency: string; - selector: string; - symbol: string; - url: string; -} diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface.ts new file mode 100644 index 000000000..21380253e --- /dev/null +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface.ts @@ -0,0 +1,4 @@ +export interface ScraperConfiguration { + selector: string; + url: string; +} diff --git a/apps/api/src/services/interfaces/symbol-profile.interface.ts b/apps/api/src/services/interfaces/symbol-profile.interface.ts index 80e9da1e7..5c786729a 100644 --- a/apps/api/src/services/interfaces/symbol-profile.interface.ts +++ b/apps/api/src/services/interfaces/symbol-profile.interface.ts @@ -1,3 +1,4 @@ +import { ScraperConfiguration } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface'; import { Country } from '@ghostfolio/common/interfaces/country.interface'; import { Sector } from '@ghostfolio/common/interfaces/sector.interface'; import { AssetClass, AssetSubClass, DataSource } from '@prisma/client'; @@ -11,6 +12,7 @@ export interface EnhancedSymbolProfile { dataSource: DataSource; id: string; name: string | null; + scraperConfiguration?: ScraperConfiguration; sectors: Sector[]; symbol: string; symbolMapping?: { [key: string]: string }; diff --git a/apps/api/src/services/symbol-profile.service.ts b/apps/api/src/services/symbol-profile.service.ts index acf8118e2..09a59de07 100644 --- a/apps/api/src/services/symbol-profile.service.ts +++ b/apps/api/src/services/symbol-profile.service.ts @@ -7,6 +7,8 @@ import { Injectable } from '@nestjs/common'; import { Prisma, SymbolProfile } from '@prisma/client'; import { continents, countries } from 'countries-list'; +import { ScraperConfiguration } from './data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface'; + @Injectable() export class SymbolProfileService { constructor(private readonly prismaService: PrismaService) {} @@ -29,6 +31,7 @@ export class SymbolProfileService { return symbolProfiles.map((symbolProfile) => ({ ...symbolProfile, countries: this.getCountries(symbolProfile), + scraperConfiguration: this.getScraperConfiguration(symbolProfile), sectors: this.getSectors(symbolProfile), symbolMapping: this.getSymbolMapping(symbolProfile) })); @@ -50,6 +53,18 @@ export class SymbolProfileService { ); } + private getScraperConfiguration( + symbolProfile: SymbolProfile + ): ScraperConfiguration { + const scraperConfiguration = + symbolProfile.scraperConfiguration as Prisma.JsonObject; + + return { + selector: scraperConfiguration.selector as string, + url: scraperConfiguration.url as string + }; + } + private getSectors(symbolProfile: SymbolProfile): Sector[] { return ((symbolProfile?.sectors as Prisma.JsonArray) ?? []).map( (sector) => { diff --git a/prisma/migrations/20211107171624_added_scraper_configuration_to_symbol_profile/migration.sql b/prisma/migrations/20211107171624_added_scraper_configuration_to_symbol_profile/migration.sql new file mode 100644 index 000000000..84ca4bb96 --- /dev/null +++ b/prisma/migrations/20211107171624_added_scraper_configuration_to_symbol_profile/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "SymbolProfile" ADD COLUMN "scraperConfiguration" JSONB; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d087f5463..58c2f0dc9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -118,19 +118,20 @@ model Settings { } model SymbolProfile { - assetClass AssetClass? - assetSubClass AssetSubClass? - countries Json? - createdAt DateTime @default(now()) - currency String? - dataSource DataSource - id String @id @default(uuid()) - name String? - Order Order[] - updatedAt DateTime @updatedAt - sectors Json? - symbol String - symbolMapping Json? + assetClass AssetClass? + assetSubClass AssetSubClass? + countries Json? + createdAt DateTime @default(now()) + currency String? + dataSource DataSource + id String @id @default(uuid()) + name String? + Order Order[] + updatedAt DateTime @updatedAt + scraperConfiguration Json? + sectors Json? + symbol String + symbolMapping Json? @@unique([dataSource, symbol]) }