From c8ca82b8037d0d045f81fe3617c5d7875c9dc9ad Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 18 Mar 2023 10:09:11 +0100 Subject: [PATCH] Feature/extend data source eod historical data by asset class and isin (#1791) * Extend EodHistoricalDataService * asset and asset sub class * isin * Update changelog --- CHANGELOG.md | 5 + apps/api/src/app/import/import.service.ts | 1 + apps/api/src/app/symbol/symbol.controller.ts | 13 ++- apps/api/src/app/symbol/symbol.service.ts | 20 +++- .../src/services/data-gathering.service.ts | 5 +- .../data-provider/data-provider.service.ts | 33 +++++-- .../eod-historical-data.service.ts | 94 +++++++++++++++++-- .../subscription-interstitial-dialog.html | 8 +- .../src/app/pages/pricing/pricing-page.html | 7 ++ .../migration.sql | 2 + prisma/schema.prisma | 1 + 11 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index d39c31058..a5c8890c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added support for asset and asset sub class to the `EOD_HISTORICAL_DATA` data source type +- Added `isin` to the asset profile model + ### Changed - Improved the language localization for _Gather Data_ diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts index 37693f160..5b10e381d 100644 --- a/apps/api/src/app/import/import.service.ts +++ b/apps/api/src/app/import/import.service.ts @@ -254,6 +254,7 @@ export class ImportService { countries: null, createdAt: undefined, id: undefined, + isin: null, name: null, scraperConfiguration: null, sectors: null, diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts index 249284ea8..6003c50d3 100644 --- a/apps/api/src/app/symbol/symbol.controller.ts +++ b/apps/api/src/app/symbol/symbol.controller.ts @@ -1,15 +1,18 @@ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor'; import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor'; import { IDataProviderHistoricalResponse } from '@ghostfolio/api/services/interfaces/interfaces'; +import type { RequestWithUser } from '@ghostfolio/common/types'; import { Controller, Get, HttpException, + Inject, Param, Query, UseGuards, UseInterceptors } from '@nestjs/common'; +import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; import { DataSource } from '@prisma/client'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; @@ -21,7 +24,10 @@ import { SymbolService } from './symbol.service'; @Controller('symbol') export class SymbolController { - public constructor(private readonly symbolService: SymbolService) {} + public constructor( + @Inject(REQUEST) private readonly request: RequestWithUser, + private readonly symbolService: SymbolService + ) {} /** * Must be before /:symbol @@ -33,7 +39,10 @@ export class SymbolController { @Query() { query = '' } ): Promise<{ items: LookupItem[] }> { try { - return this.symbolService.lookup(query.toLowerCase()); + return this.symbolService.lookup({ + query: query.toLowerCase(), + user: this.request.user + }); } catch { throw new HttpException( getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR), diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 7a5f5586d..718387b3b 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -5,7 +5,10 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { MarketDataService } from '@ghostfolio/api/services/market-data.service'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; -import { HistoricalDataItem } from '@ghostfolio/common/interfaces'; +import { + HistoricalDataItem, + UserWithSettings +} from '@ghostfolio/common/interfaces'; import { Injectable, Logger } from '@nestjs/common'; import { format, subDays } from 'date-fns'; @@ -79,15 +82,24 @@ export class SymbolService { }; } - public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> { + public async lookup({ + query, + user + }: { + query: string; + user: UserWithSettings; + }): Promise<{ items: LookupItem[] }> { const results: { items: LookupItem[] } = { items: [] }; - if (!aQuery) { + if (!query) { return results; } try { - const { items } = await this.dataProviderService.search(aQuery); + const { items } = await this.dataProviderService.search({ + query, + user + }); results.items = items; return results; } catch (error) { diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index d3843456d..aef198e5b 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -152,10 +152,11 @@ export class DataGatheringService { countries, currency, dataSource, + isin, name, sectors, url - } = assetProfiles[symbol]; + } = assetProfile; try { await this.prismaService.symbolProfile.upsert({ @@ -165,6 +166,7 @@ export class DataGatheringService { countries, currency, dataSource, + isin, name, sectors, symbol, @@ -175,6 +177,7 @@ export class DataGatheringService { assetSubClass, countries, currency, + isin, name, sectors, url diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 9edeedc1c..ad8929eee 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -8,6 +8,7 @@ import { } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { UserWithSettings } from '@ghostfolio/common/interfaces'; import { Granularity } from '@ghostfolio/common/types'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; @@ -260,18 +261,33 @@ export class DataProviderService { return response; } - public async search(aQuery: string): Promise<{ items: LookupItem[] }> { + public async search({ + query, + user + }: { + query: string; + user: UserWithSettings; + }): Promise<{ items: LookupItem[] }> { const promises: Promise<{ items: LookupItem[] }>[] = []; let lookupItems: LookupItem[] = []; - if (aQuery?.length < 2) { + if (query?.length < 2) { return { items: lookupItems }; } - for (const dataSource of this.configurationService.get('DATA_SOURCES')) { - promises.push( - this.getDataProvider(DataSource[dataSource]).search(aQuery) - ); + let dataSources = this.configurationService.get('DATA_SOURCES'); + + if ( + this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') && + user.subscription.type === 'Basic' + ) { + dataSources = dataSources.filter((dataSource) => { + return !this.isPremiumDataSource(DataSource[dataSource]); + }); + } + + for (const dataSource of dataSources) { + promises.push(this.getDataProvider(DataSource[dataSource]).search(query)); } const searchResults = await Promise.all(promises); @@ -305,4 +321,9 @@ export class DataProviderService { throw new Error('No data provider has been found.'); } + + private isPremiumDataSource(aDataSource: DataSource) { + const premiumDataSources: DataSource[] = [DataSource.EOD_HISTORICAL_DATA]; + return premiumDataSources.includes(aDataSource); + } } diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index a39100d1f..e2d1efd95 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -8,7 +8,12 @@ import { import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; -import { DataSource, SymbolProfile } from '@prisma/client'; +import { + AssetClass, + AssetSubClass, + DataSource, + SymbolProfile +} from '@prisma/client'; import bent from 'bent'; import { format, isToday } from 'date-fns'; @@ -30,12 +35,15 @@ export class EodHistoricalDataService implements DataProviderInterface { public async getAssetProfile( aSymbol: string ): Promise> { - const { items } = await this.search(aSymbol); + const [searchResult] = await this.getSearchResult(aSymbol); return { - currency: items[0]?.currency, + assetClass: searchResult?.assetClass, + assetSubClass: searchResult?.assetSubClass, + currency: searchResult?.currency, dataSource: this.getName(), - name: items[0]?.name + isin: searchResult?.isin, + name: searchResult?.name }; } @@ -156,7 +164,27 @@ export class EodHistoricalDataService implements DataProviderInterface { } public async search(aQuery: string): Promise<{ items: LookupItem[] }> { - let items: LookupItem[] = []; + const searchResult = await this.getSearchResult(aQuery); + + return { + items: searchResult + .filter(({ symbol }) => { + return !symbol.toLowerCase().endsWith('forex'); + }) + .map(({ currency, dataSource, name, symbol }) => { + return { currency, dataSource, name, symbol }; + }) + }; + } + + private async getSearchResult(aQuery: string): Promise< + (LookupItem & { + assetClass: AssetClass; + assetSubClass: AssetSubClass; + isin: string; + })[] + > { + let searchResult = []; try { const get = bent( @@ -167,10 +195,25 @@ export class EodHistoricalDataService implements DataProviderInterface { ); const response = await get(); - items = response.map( - ({ Code, Currency: currency, Exchange, Name: name }) => { + searchResult = response.map( + ({ + Code, + Currency: currency, + Exchange, + ISIN: isin, + Name: name, + Type + }) => { + const { assetClass, assetSubClass } = this.parseAssetClass({ + Exchange, + Type + }); + return { + assetClass, + assetSubClass, currency, + isin, name, dataSource: this.getName(), symbol: `${Code}.${Exchange}` @@ -181,6 +224,41 @@ export class EodHistoricalDataService implements DataProviderInterface { Logger.error(error, 'EodHistoricalDataService'); } - return { items }; + return searchResult; + } + + private parseAssetClass({ + Exchange, + Type + }: { + Exchange: string; + Type: string; + }): { + assetClass: AssetClass; + assetSubClass: AssetSubClass; + } { + let assetClass: AssetClass; + let assetSubClass: AssetSubClass; + + switch (Type?.toLowerCase()) { + case 'common stock': + assetClass = AssetClass.EQUITY; + assetSubClass = AssetSubClass.STOCK; + break; + case 'currency': + assetClass = AssetClass.CASH; + + if (Exchange?.toLowerCase() === 'cc') { + assetSubClass = AssetSubClass.CRYPTOCURRENCY; + } + + break; + case 'etf': + assetClass = AssetClass.EQUITY; + assetSubClass = AssetSubClass.ETF; + break; + } + + return { assetClass, assetSubClass }; } } diff --git a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html index 061b43e3d..321ab1e13 100644 --- a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html +++ b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html @@ -19,16 +19,20 @@
  • - Performance Benchmarks + Portfolio Allocations
  • - Allocations + Performance Benchmarks
  • FIRE Calculator
    • +
  • + + Professional Data Provider +
  • and more Features... diff --git a/apps/client/src/app/pages/pricing/pricing-page.html b/apps/client/src/app/pages/pricing/pricing-page.html index 8140822a6..1284cda00 100644 --- a/apps/client/src/app/pages/pricing/pricing-page.html +++ b/apps/client/src/app/pages/pricing/pricing-page.html @@ -280,6 +280,13 @@
    • +
  • + + Professional Data Provider +