open-source-mirrors
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Feature/improve handling of derived currencies (#2891)

Browse Source 
* Improve handling of derived currencies

* Update changelog
pull/2899/head
Thomas Kaul 1 year ago committed by GitHub
parent 6167f105fe
commit 5ba5b86d5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 217 additions and 196 deletions
Show Stats Download Patch File Download Diff File

1
CHANGELOG.md
Unescape View File

@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Improved the handling of derived currencies
- Improved the labels in the portfolio evolution chart and investment timeline on the analysis page
- Improved the language localization for German (`de`)
- Upgraded `prisma` from version `5.7.1` to `5.8.1`

180
apps/api/src/services/data-provider/data-provider.service.ts
Unescape View File

@ -9,14 +9,19 @@ import {
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
import {
DEFAULT_CURRENCY,
DERIVED_CURRENCIES,
PROPERTY_DATA_SOURCE_MAPPING
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
import { format, isValid } from 'date-fns';
import { groupBy, isEmpty, isNumber } from 'lodash';
import Big from 'big.js';
import { eachDayOfInterval, format, isValid } from 'date-fns';
import { groupBy, isEmpty, isNumber, uniqWith } from 'lodash';
import ms from 'ms';
@Injectable()
@ -205,6 +210,31 @@ export class DataProviderService {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
let dataGatheringItems = aDataGatheringItems;
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if (
this.hasCurrency({
dataGatheringItems,
currency: `${DEFAULT_CURRENCY}${currency}`
})
) {
// Skip derived currency
dataGatheringItems = dataGatheringItems.filter(({ symbol }) => {
return symbol !== `${DEFAULT_CURRENCY}${currency}`;
});
// Add root currency
dataGatheringItems.push({
dataSource: this.getDataSourceForExchangeRates(),
symbol: `${DEFAULT_CURRENCY}${rootCurrency}`
});
}
}
dataGatheringItems = uniqWith(dataGatheringItems, (obj1, obj2) => {
return obj1.dataSource === obj2.dataSource && obj1.symbol === obj2.symbol;
});
const result: {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
} = {};
@ -213,9 +243,25 @@ export class DataProviderService {
data: { [date: string]: IDataProviderHistoricalResponse };
symbol: string;
}>[] = [];
for (const { dataSource, symbol } of aDataGatheringItems) {
for (const { dataSource, symbol } of dataGatheringItems) {
const dataProvider = this.getDataProvider(dataSource);
if (dataProvider.canHandle(symbol)) {
if (symbol === `${DEFAULT_CURRENCY}USX`) {
const data: {
[date: string]: IDataProviderHistoricalResponse;
} = {};
for (const date of eachDayOfInterval({ end: to, start: from })) {
data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
}
promises.push(
Promise.resolve({
data,
symbol
})
);
} else {
promises.push(
dataProvider
.getHistorical({
@ -224,14 +270,32 @@ export class DataProviderService {
to,
requestTimeout: ms('30 seconds')
})
.then((data) => ({ data: data?.[symbol], symbol }))
.then((data) => {
return { symbol, data: data?.[symbol] };
})
);
}
}
}
try {
const allData = await Promise.all(promises);
for (const { data, symbol } of allData) {
const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
});
if (currency) {
// Add derived currency
result[`${DEFAULT_CURRENCY}${currency.currency}`] =
this.transformHistoricalData({
allData,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
factor: currency.factor
});
}
result[symbol] = data;
}
} catch (error) {
@ -257,6 +321,19 @@ export class DataProviderService {
} = {};
const startTimeTotal = performance.now();
if (
items.some(({ symbol }) => {
return symbol === `${DEFAULT_CURRENCY}USX`;
})
) {
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getDataSourceForExchangeRates(),
marketPrice: 100,
marketState: 'open'
};
}
// Get items from cache
const itemsToFetch: UniqueAsset[] = [];
@ -326,20 +403,57 @@ export class DataProviderService {
promises.push(
promise.then(async (result) => {
for (const [symbol, dataProviderResponse] of Object.entries(
result
)) {
for (let [symbol, dataProviderResponse] of Object.entries(result)) {
if (
[
...DERIVED_CURRENCIES.map(({ currency }) => {
return `${DEFAULT_CURRENCY}${currency}`;
}),
`${DEFAULT_CURRENCY}USX`
].includes(symbol)
) {
continue;
}
response[symbol] = dataProviderResponse;
this.redisCacheService.set(
this.redisCacheService.getQuoteKey({
symbol,
dataSource: DataSource[dataSource]
}),
JSON.stringify(response[symbol]),
this.configurationService.get('CACHE_QUOTES_TTL')
);
for (const {
currency,
factor,
rootCurrency
} of DERIVED_CURRENCIES) {
if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) {
response[`${DEFAULT_CURRENCY}${currency}`] = {
...dataProviderResponse,
currency,
marketPrice: new Big(
result[`${DEFAULT_CURRENCY}${rootCurrency}`].marketPrice
)
.mul(factor)
.toNumber(),
marketState: 'open'
};
this.redisCacheService.set(
this.redisCacheService.getQuoteKey({
dataSource: DataSource[dataSource],
symbol
symbol: `${DEFAULT_CURRENCY}${currency}`
}),
JSON.stringify(dataProviderResponse),
JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]),
this.configurationService.get('CACHE_QUOTES_TTL')
);
}
}
}
Logger.debug(
`Fetched ${symbolsChunk.length} quote${
@ -472,6 +586,21 @@ export class DataProviderService {
throw new Error('No data provider has been found.');
}
private hasCurrency({
currency,
dataGatheringItems
}: {
currency: string;
dataGatheringItems: UniqueAsset[];
}) {
return dataGatheringItems.some(({ dataSource, symbol }) => {
return (
dataSource === this.getDataSourceForExchangeRates() &&
symbol === currency
);
});
}
private isPremiumDataSource(aDataSource: DataSource) {
const premiumDataSources: DataSource[] = [
DataSource.EOD_HISTORICAL_DATA,
@ -479,4 +608,35 @@ export class DataProviderService {
];
return premiumDataSources.includes(aDataSource);
}
private transformHistoricalData({
allData,
currency,
factor
}: {
allData: {
data: {
[date: string]: IDataProviderHistoricalResponse;
};
symbol: string;
}[];
currency: string;
factor: number;
}) {
const rootData = allData.find(({ symbol }) => {
return symbol === currency;
})?.data;
const data: {
[date: string]: IDataProviderHistoricalResponse;
} = {};
for (const date in rootData) {
data[date] = {
marketPrice: new Big(factor).mul(rootData[date].marketPrice).toNumber()
};
}
return data;
}
}

69
apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
Unescape View File

@ -20,7 +20,6 @@ import {
DataSource,
SymbolProfile
} from '@prisma/client';
import Big from 'big.js';
import { format, isToday } from 'date-fns';
import got from 'got';
@ -93,10 +92,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
return response.reduce(
(result, historicalItem, index, array) => {
result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
marketPrice: this.getConvertedValue({
symbol: symbol,
value: historicalItem.close
})
marketPrice: historicalItem.close
};
return result;
@ -196,48 +192,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
{}
);
if (response[`${DEFAULT_CURRENCY}GBP`]) {
response[`${DEFAULT_CURRENCY}GBp`] = {
...response[`${DEFAULT_CURRENCY}GBP`],
currency: 'GBp',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}GBp`,
value: response[`${DEFAULT_CURRENCY}GBP`].marketPrice
})
};
}
if (response[`${DEFAULT_CURRENCY}ILS`]) {
response[`${DEFAULT_CURRENCY}ILA`] = {
...response[`${DEFAULT_CURRENCY}ILS`],
currency: 'ILA',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ILA`,
value: response[`${DEFAULT_CURRENCY}ILS`].marketPrice
})
};
}
if (response[`${DEFAULT_CURRENCY}USX`]) {
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getName(),
marketPrice: new Big(1).mul(100).toNumber(),
marketState: 'open'
};
}
if (response[`${DEFAULT_CURRENCY}ZAR`]) {
response[`${DEFAULT_CURRENCY}ZAc`] = {
...response[`${DEFAULT_CURRENCY}ZAR`],
currency: 'ZAc',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ZAc`,
value: response[`${DEFAULT_CURRENCY}ZAR`].marketPrice
})
};
}
return response;
} catch (error) {
let message = error;
@ -337,27 +291,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
return aSymbol;
}
private getConvertedValue({
symbol,
value
}: {
symbol: string;
value: number;
}) {
if (symbol === `${DEFAULT_CURRENCY}GBp`) {
// Convert GPB to GBp (pence)
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
// Convert ILS to ILA
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
// Convert ZAR to ZAc
return new Big(value).mul(100).toNumber();
}
return value;
}
private async getSearchResult(aQuery: string): Promise<
(LookupItem & {
assetClass: AssetClass;

83
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
Unescape View File

@ -16,7 +16,6 @@ import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import Big from 'big.js';
import { addDays, format, isSameDay } from 'date-fns';
import yahooFinance from 'yahoo-finance2';
import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote';
@ -77,10 +76,7 @@ export class YahooFinanceService implements DataProviderInterface {
for (const historicalItem of historicalResult) {
response[format(historicalItem.date, DATE_FORMAT)] = {
marketPrice: this.getConvertedValue({
symbol,
value: historicalItem.dividends
})
marketPrice: historicalItem.dividends
};
}
@ -129,10 +125,7 @@ export class YahooFinanceService implements DataProviderInterface {
for (const historicalItem of historicalResult) {
response[symbol][format(historicalItem.date, DATE_FORMAT)] = {
marketPrice: this.getConvertedValue({
symbol: symbol,
value: historicalItem.close
})
marketPrice: historicalItem.close
};
}
@ -204,57 +197,6 @@ export class YahooFinanceService implements DataProviderInterface {
: 'closed',
marketPrice: quote.regularMarketPrice || 0
};
if (
symbol === `${DEFAULT_CURRENCY}GBP` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}GBp=X`)
) {
// Convert GPB to GBp (pence)
response[`${DEFAULT_CURRENCY}GBp`] = {
...response[symbol],
currency: 'GBp',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}GBp`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${DEFAULT_CURRENCY}ILS` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ILA=X`)
) {
// Convert ILS to ILA
response[`${DEFAULT_CURRENCY}ILA`] = {
...response[symbol],
currency: 'ILA',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ILA`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${DEFAULT_CURRENCY}ZAR` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ZAc=X`)
) {
// Convert ZAR to ZAc (cents)
response[`${DEFAULT_CURRENCY}ZAc`] = {
...response[symbol],
currency: 'ZAc',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ZAc`,
value: response[symbol].marketPrice
})
};
}
}
if (yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}USX=X`)) {
// Convert USD to USX (cent)
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getName(),
marketPrice: new Big(1).mul(100).toNumber(),
marketState: 'open'
};
}
return response;
@ -357,27 +299,6 @@ export class YahooFinanceService implements DataProviderInterface {
return { items };
}
private getConvertedValue({
symbol,
value
}: {
symbol: string;
value: number;
}) {
if (symbol === `${DEFAULT_CURRENCY}GBp`) {
// Convert GPB to GBp (pence)
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
// Convert ILS to ILA
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
// Convert ZAR to ZAc (cents)
return new Big(value).mul(100).toNumber();
}
return value;
}
private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) {
const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => {
return yahooFinance.quoteSummary(symbol).catch(() => {

43
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
Unescape View File

@ -5,6 +5,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import {
DEFAULT_CURRENCY,
DERIVED_CURRENCIES,
PROPERTY_CURRENCIES
} from '@ghostfolio/common/config';
import {
@ -168,30 +169,6 @@ export class ExchangeRateDataService {
const [currency1, currency2] = symbol.match(/.{1,3}/g);
const [date] = Object.keys(result[symbol]);
// Add derived currencies
if (currency2 === 'GBP') {
resultExtended[`${currency1}GBp`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
} else if (currency2 === 'ILS') {
resultExtended[`${currency1}ILA`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
} else if (currency2 === 'ZAR') {
resultExtended[`${currency1}ZAc`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
}
// Calculate the opposite direction
resultExtended[`${currency2}${currency1}`] = {
[date]: {
@ -486,8 +463,8 @@ export class ExchangeRateDataService {
}
}
})
).forEach((account) => {
currencies.push(account.currency);
).forEach(({ currency }) => {
currencies.push(currency);
});
(
@ -496,8 +473,8 @@ export class ExchangeRateDataService {
orderBy: [{ currency: 'asc' }],
select: { currency: true }
})
).forEach((symbolProfile) => {
currencies.push(symbolProfile.currency);
).forEach(({ currency }) => {
currencies.push(currency);
});
const customCurrencies = (await this.propertyService.getByKey(
@ -508,6 +485,16 @@ export class ExchangeRateDataService {
currencies = currencies.concat(customCurrencies);
}
// Add derived currencies
currencies.push('USX');
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if (currencies.includes(currency) || currencies.includes(rootCurrency)) {
currencies.push(currency);
currencies.push(rootCurrency);
}
}
return uniq(currencies).filter(Boolean).sort();
}

19
libs/common/src/lib/config.ts
Unescape View File

@ -41,6 +41,25 @@ export const DEFAULT_LANGUAGE_CODE = 'en';
export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_ROOT_URL = 'http://localhost:4200';
// USX is handled separately
export const DERIVED_CURRENCIES = [
{
currency: 'GBp',
factor: 100,
rootCurrency: 'GBP'
},
{
currency: 'ILA',
factor: 100,
rootCurrency: 'ILS'
},
{
currency: 'ZAc',
factor: 100,
rootCurrency: 'ZAR'
}
];
export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';
export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';

Write Preview
Loading…
Cancel
Save