From 5ba5b86d5f28e4a6831f0649523bac946c79e6e5 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 21 Jan 2024 11:12:48 +0100 Subject: [PATCH] Feature/improve handling of derived currencies (#2891) * Improve handling of derived currencies * Update changelog --- CHANGELOG.md | 1 + .../data-provider/data-provider.service.ts | 198 ++++++++++++++++-- .../eod-historical-data.service.ts | 69 +----- .../yahoo-finance/yahoo-finance.service.ts | 83 +------- .../exchange-rate-data.service.ts | 43 ++-- libs/common/src/lib/config.ts | 19 ++ 6 files changed, 217 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a3e0392..4d9b1660c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Improved the handling of derived currencies - Improved the labels in the portfolio evolution chart and investment timeline on the analysis page - Improved the language localization for German (`de`) - Upgraded `prisma` from version `5.7.1` to `5.8.1` diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index aee33d729..b8ea11f25 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -9,14 +9,19 @@ import { import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; -import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config'; +import { + DEFAULT_CURRENCY, + DERIVED_CURRENCIES, + PROPERTY_DATA_SOURCE_MAPPING +} from '@ghostfolio/common/config'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import type { Granularity, UserWithSettings } from '@ghostfolio/common/types'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; -import { format, isValid } from 'date-fns'; -import { groupBy, isEmpty, isNumber } from 'lodash'; +import Big from 'big.js'; +import { eachDayOfInterval, format, isValid } from 'date-fns'; +import { groupBy, isEmpty, isNumber, uniqWith } from 'lodash'; import ms from 'ms'; @Injectable() @@ -205,6 +210,31 @@ export class DataProviderService { ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { + let dataGatheringItems = aDataGatheringItems; + + for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { + if ( + this.hasCurrency({ + dataGatheringItems, + currency: `${DEFAULT_CURRENCY}${currency}` + }) + ) { + // Skip derived currency + dataGatheringItems = dataGatheringItems.filter(({ symbol }) => { + return symbol !== `${DEFAULT_CURRENCY}${currency}`; + }); + // Add root currency + dataGatheringItems.push({ + dataSource: this.getDataSourceForExchangeRates(), + symbol: `${DEFAULT_CURRENCY}${rootCurrency}` + }); + } + } + + dataGatheringItems = uniqWith(dataGatheringItems, (obj1, obj2) => { + return obj1.dataSource === obj2.dataSource && obj1.symbol === obj2.symbol; + }); + const result: { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; } = {}; @@ -213,25 +243,59 @@ export class DataProviderService { data: { [date: string]: IDataProviderHistoricalResponse }; symbol: string; }>[] = []; - for (const { dataSource, symbol } of aDataGatheringItems) { + for (const { dataSource, symbol } of dataGatheringItems) { const dataProvider = this.getDataProvider(dataSource); if (dataProvider.canHandle(symbol)) { - promises.push( - dataProvider - .getHistorical({ - from, - symbol, - to, - requestTimeout: ms('30 seconds') + if (symbol === `${DEFAULT_CURRENCY}USX`) { + const data: { + [date: string]: IDataProviderHistoricalResponse; + } = {}; + + for (const date of eachDayOfInterval({ end: to, start: from })) { + data[format(date, DATE_FORMAT)] = { marketPrice: 100 }; + } + + promises.push( + Promise.resolve({ + data, + symbol }) - .then((data) => ({ data: data?.[symbol], symbol })) - ); + ); + } else { + promises.push( + dataProvider + .getHistorical({ + from, + symbol, + to, + requestTimeout: ms('30 seconds') + }) + .then((data) => { + return { symbol, data: data?.[symbol] }; + }) + ); + } } } try { const allData = await Promise.all(promises); + for (const { data, symbol } of allData) { + const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => { + return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol; + }); + + if (currency) { + // Add derived currency + result[`${DEFAULT_CURRENCY}${currency.currency}`] = + this.transformHistoricalData({ + allData, + currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`, + factor: currency.factor + }); + } + result[symbol] = data; } } catch (error) { @@ -257,6 +321,19 @@ export class DataProviderService { } = {}; const startTimeTotal = performance.now(); + if ( + items.some(({ symbol }) => { + return symbol === `${DEFAULT_CURRENCY}USX`; + }) + ) { + response[`${DEFAULT_CURRENCY}USX`] = { + currency: 'USX', + dataSource: this.getDataSourceForExchangeRates(), + marketPrice: 100, + marketState: 'open' + }; + } + // Get items from cache const itemsToFetch: UniqueAsset[] = []; @@ -326,19 +403,56 @@ export class DataProviderService { promises.push( promise.then(async (result) => { - for (const [symbol, dataProviderResponse] of Object.entries( - result - )) { + for (let [symbol, dataProviderResponse] of Object.entries(result)) { + if ( + [ + ...DERIVED_CURRENCIES.map(({ currency }) => { + return `${DEFAULT_CURRENCY}${currency}`; + }), + `${DEFAULT_CURRENCY}USX` + ].includes(symbol) + ) { + continue; + } + response[symbol] = dataProviderResponse; this.redisCacheService.set( this.redisCacheService.getQuoteKey({ - dataSource: DataSource[dataSource], - symbol + symbol, + dataSource: DataSource[dataSource] }), - JSON.stringify(dataProviderResponse), + JSON.stringify(response[symbol]), this.configurationService.get('CACHE_QUOTES_TTL') ); + + for (const { + currency, + factor, + rootCurrency + } of DERIVED_CURRENCIES) { + if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) { + response[`${DEFAULT_CURRENCY}${currency}`] = { + ...dataProviderResponse, + currency, + marketPrice: new Big( + result[`${DEFAULT_CURRENCY}${rootCurrency}`].marketPrice + ) + .mul(factor) + .toNumber(), + marketState: 'open' + }; + + this.redisCacheService.set( + this.redisCacheService.getQuoteKey({ + dataSource: DataSource[dataSource], + symbol: `${DEFAULT_CURRENCY}${currency}` + }), + JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]), + this.configurationService.get('CACHE_QUOTES_TTL') + ); + } + } } Logger.debug( @@ -472,6 +586,21 @@ export class DataProviderService { throw new Error('No data provider has been found.'); } + private hasCurrency({ + currency, + dataGatheringItems + }: { + currency: string; + dataGatheringItems: UniqueAsset[]; + }) { + return dataGatheringItems.some(({ dataSource, symbol }) => { + return ( + dataSource === this.getDataSourceForExchangeRates() && + symbol === currency + ); + }); + } + private isPremiumDataSource(aDataSource: DataSource) { const premiumDataSources: DataSource[] = [ DataSource.EOD_HISTORICAL_DATA, @@ -479,4 +608,35 @@ export class DataProviderService { ]; return premiumDataSources.includes(aDataSource); } + + private transformHistoricalData({ + allData, + currency, + factor + }: { + allData: { + data: { + [date: string]: IDataProviderHistoricalResponse; + }; + symbol: string; + }[]; + currency: string; + factor: number; + }) { + const rootData = allData.find(({ symbol }) => { + return symbol === currency; + })?.data; + + const data: { + [date: string]: IDataProviderHistoricalResponse; + } = {}; + + for (const date in rootData) { + data[date] = { + marketPrice: new Big(factor).mul(rootData[date].marketPrice).toNumber() + }; + } + + return data; + } } diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts index 173994df7..37f6ee9ea 100644 --- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -20,7 +20,6 @@ import { DataSource, SymbolProfile } from '@prisma/client'; -import Big from 'big.js'; import { format, isToday } from 'date-fns'; import got from 'got'; @@ -93,10 +92,7 @@ export class EodHistoricalDataService implements DataProviderInterface { return response.reduce( (result, historicalItem, index, array) => { result[this.convertFromEodSymbol(symbol)][historicalItem.date] = { - marketPrice: this.getConvertedValue({ - symbol: symbol, - value: historicalItem.close - }) + marketPrice: historicalItem.close }; return result; @@ -196,48 +192,6 @@ export class EodHistoricalDataService implements DataProviderInterface { {} ); - if (response[`${DEFAULT_CURRENCY}GBP`]) { - response[`${DEFAULT_CURRENCY}GBp`] = { - ...response[`${DEFAULT_CURRENCY}GBP`], - currency: 'GBp', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}GBp`, - value: response[`${DEFAULT_CURRENCY}GBP`].marketPrice - }) - }; - } - - if (response[`${DEFAULT_CURRENCY}ILS`]) { - response[`${DEFAULT_CURRENCY}ILA`] = { - ...response[`${DEFAULT_CURRENCY}ILS`], - currency: 'ILA', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}ILA`, - value: response[`${DEFAULT_CURRENCY}ILS`].marketPrice - }) - }; - } - - if (response[`${DEFAULT_CURRENCY}USX`]) { - response[`${DEFAULT_CURRENCY}USX`] = { - currency: 'USX', - dataSource: this.getName(), - marketPrice: new Big(1).mul(100).toNumber(), - marketState: 'open' - }; - } - - if (response[`${DEFAULT_CURRENCY}ZAR`]) { - response[`${DEFAULT_CURRENCY}ZAc`] = { - ...response[`${DEFAULT_CURRENCY}ZAR`], - currency: 'ZAc', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}ZAc`, - value: response[`${DEFAULT_CURRENCY}ZAR`].marketPrice - }) - }; - } - return response; } catch (error) { let message = error; @@ -337,27 +291,6 @@ export class EodHistoricalDataService implements DataProviderInterface { return aSymbol; } - private getConvertedValue({ - symbol, - value - }: { - symbol: string; - value: number; - }) { - if (symbol === `${DEFAULT_CURRENCY}GBp`) { - // Convert GPB to GBp (pence) - return new Big(value).mul(100).toNumber(); - } else if (symbol === `${DEFAULT_CURRENCY}ILA`) { - // Convert ILS to ILA - return new Big(value).mul(100).toNumber(); - } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) { - // Convert ZAR to ZAc - return new Big(value).mul(100).toNumber(); - } - - return value; - } - private async getSearchResult(aQuery: string): Promise< (LookupItem & { assetClass: AssetClass; diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index e027c227f..47869d3e8 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -16,7 +16,6 @@ import { DEFAULT_CURRENCY } from '@ghostfolio/common/config'; import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import Big from 'big.js'; import { addDays, format, isSameDay } from 'date-fns'; import yahooFinance from 'yahoo-finance2'; import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote'; @@ -77,10 +76,7 @@ export class YahooFinanceService implements DataProviderInterface { for (const historicalItem of historicalResult) { response[format(historicalItem.date, DATE_FORMAT)] = { - marketPrice: this.getConvertedValue({ - symbol, - value: historicalItem.dividends - }) + marketPrice: historicalItem.dividends }; } @@ -129,10 +125,7 @@ export class YahooFinanceService implements DataProviderInterface { for (const historicalItem of historicalResult) { response[symbol][format(historicalItem.date, DATE_FORMAT)] = { - marketPrice: this.getConvertedValue({ - symbol: symbol, - value: historicalItem.close - }) + marketPrice: historicalItem.close }; } @@ -204,57 +197,6 @@ export class YahooFinanceService implements DataProviderInterface { : 'closed', marketPrice: quote.regularMarketPrice || 0 }; - - if ( - symbol === `${DEFAULT_CURRENCY}GBP` && - yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}GBp=X`) - ) { - // Convert GPB to GBp (pence) - response[`${DEFAULT_CURRENCY}GBp`] = { - ...response[symbol], - currency: 'GBp', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}GBp`, - value: response[symbol].marketPrice - }) - }; - } else if ( - symbol === `${DEFAULT_CURRENCY}ILS` && - yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ILA=X`) - ) { - // Convert ILS to ILA - response[`${DEFAULT_CURRENCY}ILA`] = { - ...response[symbol], - currency: 'ILA', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}ILA`, - value: response[symbol].marketPrice - }) - }; - } else if ( - symbol === `${DEFAULT_CURRENCY}ZAR` && - yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ZAc=X`) - ) { - // Convert ZAR to ZAc (cents) - response[`${DEFAULT_CURRENCY}ZAc`] = { - ...response[symbol], - currency: 'ZAc', - marketPrice: this.getConvertedValue({ - symbol: `${DEFAULT_CURRENCY}ZAc`, - value: response[symbol].marketPrice - }) - }; - } - } - - if (yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}USX=X`)) { - // Convert USD to USX (cent) - response[`${DEFAULT_CURRENCY}USX`] = { - currency: 'USX', - dataSource: this.getName(), - marketPrice: new Big(1).mul(100).toNumber(), - marketState: 'open' - }; } return response; @@ -357,27 +299,6 @@ export class YahooFinanceService implements DataProviderInterface { return { items }; } - private getConvertedValue({ - symbol, - value - }: { - symbol: string; - value: number; - }) { - if (symbol === `${DEFAULT_CURRENCY}GBp`) { - // Convert GPB to GBp (pence) - return new Big(value).mul(100).toNumber(); - } else if (symbol === `${DEFAULT_CURRENCY}ILA`) { - // Convert ILS to ILA - return new Big(value).mul(100).toNumber(); - } else if (symbol === `${DEFAULT_CURRENCY}ZAc`) { - // Convert ZAR to ZAc (cents) - return new Big(value).mul(100).toNumber(); - } - - return value; - } - private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) { const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => { return yahooFinance.quoteSummary(symbol).catch(() => { diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts index 36dd5c0a2..3c3fc5573 100644 --- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts @@ -5,6 +5,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DEFAULT_CURRENCY, + DERIVED_CURRENCIES, PROPERTY_CURRENCIES } from '@ghostfolio/common/config'; import { @@ -168,30 +169,6 @@ export class ExchangeRateDataService { const [currency1, currency2] = symbol.match(/.{1,3}/g); const [date] = Object.keys(result[symbol]); - // Add derived currencies - if (currency2 === 'GBP') { - resultExtended[`${currency1}GBp`] = { - [date]: { - marketPrice: - result[`${currency1}${currency2}`][date].marketPrice * 100 - } - }; - } else if (currency2 === 'ILS') { - resultExtended[`${currency1}ILA`] = { - [date]: { - marketPrice: - result[`${currency1}${currency2}`][date].marketPrice * 100 - } - }; - } else if (currency2 === 'ZAR') { - resultExtended[`${currency1}ZAc`] = { - [date]: { - marketPrice: - result[`${currency1}${currency2}`][date].marketPrice * 100 - } - }; - } - // Calculate the opposite direction resultExtended[`${currency2}${currency1}`] = { [date]: { @@ -486,8 +463,8 @@ export class ExchangeRateDataService { } } }) - ).forEach((account) => { - currencies.push(account.currency); + ).forEach(({ currency }) => { + currencies.push(currency); }); ( @@ -496,8 +473,8 @@ export class ExchangeRateDataService { orderBy: [{ currency: 'asc' }], select: { currency: true } }) - ).forEach((symbolProfile) => { - currencies.push(symbolProfile.currency); + ).forEach(({ currency }) => { + currencies.push(currency); }); const customCurrencies = (await this.propertyService.getByKey( @@ -508,6 +485,16 @@ export class ExchangeRateDataService { currencies = currencies.concat(customCurrencies); } + // Add derived currencies + currencies.push('USX'); + + for (const { currency, rootCurrency } of DERIVED_CURRENCIES) { + if (currencies.includes(currency) || currencies.includes(rootCurrency)) { + currencies.push(currency); + currencies.push(rootCurrency); + } + } + return uniq(currencies).filter(Boolean).sort(); } diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts index 18926f0b8..df89ae9e9 100644 --- a/libs/common/src/lib/config.ts +++ b/libs/common/src/lib/config.ts @@ -41,6 +41,25 @@ export const DEFAULT_LANGUAGE_CODE = 'en'; export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_ROOT_URL = 'http://localhost:4200'; +// USX is handled separately +export const DERIVED_CURRENCIES = [ + { + currency: 'GBp', + factor: 100, + rootCurrency: 'GBP' + }, + { + currency: 'ILA', + factor: 100, + rootCurrency: 'ILS' + }, + { + currency: 'ZAc', + factor: 100, + rootCurrency: 'ZAR' + } +]; + export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180'; export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';