diff --git a/CHANGELOG.md b/CHANGELOG.md index 526e0d194..0256f3a56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added a fallback to historical market data if a data provider does not provide live data + +### Changed + +- Persisted today's market data continuously + ### Fixed - Fixed the alignment of the performance column header in the holdings table diff --git a/apps/api/src/app/portfolio/current-rate.service.mock.ts b/apps/api/src/app/portfolio/current-rate.service.mock.ts index cac110f05..9fe6b2ec2 100644 --- a/apps/api/src/app/portfolio/current-rate.service.mock.ts +++ b/apps/api/src/app/portfolio/current-rate.service.mock.ts @@ -1,9 +1,9 @@ import { parseDate, resetHours } from '@ghostfolio/common/helper'; -import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns'; import { GetValueObject } from './interfaces/get-value-object.interface'; import { GetValuesParams } from './interfaces/get-values-params.interface'; +import { GetValuesObject } from './interfaces/get-values-object.interface'; function mockGetValue(symbol: string, date: Date) { switch (symbol) { @@ -49,11 +49,9 @@ export const CurrentRateServiceMock = { getValues: ({ dataGatheringItems, dateQuery - }: GetValuesParams): Promise<{ - dataProviderInfos: DataProviderInfo[]; - values: GetValueObject[]; - }> => { + }: GetValuesParams): Promise => { const values: GetValueObject[] = []; + if (dateQuery.lt) { for ( let date = resetHours(dateQuery.gte); @@ -85,6 +83,7 @@ export const CurrentRateServiceMock = { } } } - return Promise.resolve({ values, dataProviderInfos: [] }); + + return Promise.resolve({ values, dataProviderInfos: [], errors: [] }); } }; diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 226d9e7db..a4eb10156 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -1,12 +1,11 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data.service'; -import { DataProviderInfo } from '@ghostfolio/common/interfaces'; +import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { DataSource, MarketData } from '@prisma/client'; import { CurrentRateService } from './current-rate.service'; -import { GetValueObject } from './interfaces/get-value-object.interface'; -import { PropertyService } from '@ghostfolio/api/services/property/property.service'; +import { GetValuesObject } from './interfaces/get-values-object.interface'; jest.mock('@ghostfolio/api/services/market-data.service', () => { return { @@ -123,21 +122,14 @@ describe('CurrentRateService', () => { }, userCurrency: 'CHF' }) - ).toMatchObject<{ - dataProviderInfos: DataProviderInfo[]; - values: GetValueObject[]; - }>({ + ).toMatchObject({ dataProviderInfos: [], + errors: [], values: [ { date: undefined, marketPriceInBaseCurrency: 1841.823902, symbol: 'AMZN' - }, - { - date: undefined, - marketPriceInBaseCurrency: 1847.839966, - symbol: 'AMZN' } ] }); diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index b3a432521..9ca95d779 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -2,13 +2,14 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data.service'; import { resetHours } from '@ghostfolio/common/helper'; -import { DataProviderInfo } from '@ghostfolio/common/interfaces'; +import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; import { isBefore, isToday } from 'date-fns'; -import { flatten } from 'lodash'; +import { flatten, isEmpty, uniqBy } from 'lodash'; import { GetValueObject } from './interfaces/get-value-object.interface'; import { GetValuesParams } from './interfaces/get-values-params.interface'; +import { GetValuesObject } from './interfaces/get-values-object.interface'; @Injectable() export class CurrentRateService { @@ -23,10 +24,7 @@ export class CurrentRateService { dataGatheringItems, dateQuery, userCurrency - }: GetValuesParams): Promise<{ - dataProviderInfos: DataProviderInfo[]; - values: GetValueObject[]; - }> { + }: GetValuesParams): Promise { const dataProviderInfos: DataProviderInfo[] = []; const includeToday = (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) && @@ -34,9 +32,10 @@ export class CurrentRateService { (!dateQuery.in || this.containsToday(dateQuery.in)); const promises: Promise[] = []; + const quoteErrors: ResponseError['errors'] = []; + const today = resetHours(new Date()); if (includeToday) { - const today = resetHours(new Date()); promises.push( this.dataProviderService .getQuotes(dataGatheringItems) @@ -51,18 +50,26 @@ export class CurrentRateService { ); } - result.push({ - date: today, - marketPriceInBaseCurrency: - this.exchangeRateDataService.toCurrency( - dataResultProvider?.[dataGatheringItem.symbol] - ?.marketPrice ?? 0, - dataResultProvider?.[dataGatheringItem.symbol]?.currency, - userCurrency - ), - symbol: dataGatheringItem.symbol - }); + if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) { + result.push({ + date: today, + marketPriceInBaseCurrency: + this.exchangeRateDataService.toCurrency( + dataResultProvider?.[dataGatheringItem.symbol] + ?.marketPrice, + dataResultProvider?.[dataGatheringItem.symbol]?.currency, + userCurrency + ), + symbol: dataGatheringItem.symbol + }); + } else { + quoteErrors.push({ + dataSource: dataGatheringItem.dataSource, + symbol: dataGatheringItem.symbol + }); + } } + return result; }) ); @@ -94,10 +101,60 @@ export class CurrentRateService { }) ); - return { + const values = flatten(await Promise.all(promises)); + + const response: GetValuesObject = { dataProviderInfos, - values: flatten(await Promise.all(promises)) + errors: quoteErrors.map(({ dataSource, symbol }) => { + return { dataSource, symbol }; + }), + values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`) }; + + if (!isEmpty(quoteErrors)) { + for (const { symbol } of quoteErrors) { + try { + // If missing quote, fallback to the latest available historical market price + let value: GetValueObject = response.values.find((currentValue) => { + return currentValue.symbol === symbol && isToday(currentValue.date); + }); + + if (!value) { + value = { + symbol, + date: today, + marketPriceInBaseCurrency: 0 + }; + + response.values.push(value); + } + + const [latestValue] = response.values + .filter((currentValue) => { + return ( + currentValue.symbol === symbol && + currentValue.marketPriceInBaseCurrency + ); + }) + .sort((a, b) => { + if (a.date < b.date) { + return 1; + } + + if (a.date > b.date) { + return -1; + } + + return 0; + }); + + value.marketPriceInBaseCurrency = + latestValue.marketPriceInBaseCurrency; + } catch {} + } + } + + return response; } private containsToday(dates: Date[]): boolean { diff --git a/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts new file mode 100644 index 000000000..ef6cb8f96 --- /dev/null +++ b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts @@ -0,0 +1,9 @@ +import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces'; + +import { GetValueObject } from './get-value-object.interface'; + +export interface GetValuesObject { + dataProviderInfos: DataProviderInfo[]; + errors: ResponseError['errors']; + values: GetValueObject[]; +} diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts index 952b81677..f71597e14 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.ts @@ -24,9 +24,10 @@ import { isSameYear, max, min, - set + set, + subDays } from 'date-fns'; -import { first, flatten, isNumber, last, sortBy } from 'lodash'; +import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash'; import { CurrentRateService } from './current-rate.service'; import { CurrentPositions } from './interfaces/current-positions.interface'; @@ -360,7 +361,7 @@ export class PortfolioCalculator { let firstTransactionPoint: TransactionPoint = null; let firstIndex = transactionPointsBeforeEndDate.length; - const dates = []; + let dates = []; const dataGatheringItems: IDataGatheringItem[] = []; const currencies: { [symbol: string]: string } = {}; @@ -389,15 +390,37 @@ export class PortfolioCalculator { dates.push(resetHours(end)); - const { dataProviderInfos, values: marketSymbols } = - await this.currentRateService.getValues({ - currencies, - dataGatheringItems, - dateQuery: { - in: dates - }, - userCurrency: this.currency - }); + // Add dates of last week for fallback + dates.push(subDays(resetHours(new Date()), 7)); + dates.push(subDays(resetHours(new Date()), 6)); + dates.push(subDays(resetHours(new Date()), 5)); + dates.push(subDays(resetHours(new Date()), 4)); + dates.push(subDays(resetHours(new Date()), 3)); + dates.push(subDays(resetHours(new Date()), 2)); + dates.push(subDays(resetHours(new Date()), 1)); + dates.push(resetHours(new Date())); + + dates = uniq( + dates.map((date) => { + return date.getTime(); + }) + ).map((timestamp) => { + return new Date(timestamp); + }); + dates.sort((a, b) => a.getTime() - b.getTime()); + + const { + dataProviderInfos, + errors: currentRateErrors, + values: marketSymbols + } = await this.currentRateService.getValues({ + currencies, + dataGatheringItems, + dateQuery: { + in: dates + }, + userCurrency: this.currency + }); this.dataProviderInfos = dataProviderInfos; @@ -472,7 +495,13 @@ export class PortfolioCalculator { transactionCount: item.transactionCount }); - if (hasErrors && item.investment.gt(0)) { + if ( + (hasErrors || + currentRateErrors.find(({ dataSource, symbol }) => { + return dataSource === item.dataSource && symbol === item.symbol; + })) && + item.investment.gt(0) + ) { errors.push({ dataSource: item.dataSource, symbol: item.symbol }); } } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index d35e01b6e..0fd7cceba 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -7,13 +7,13 @@ import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; -import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { UserWithSettings } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; import { format, isValid } from 'date-fns'; -import { groupBy, isEmpty } from 'lodash'; +import { groupBy, isEmpty, isNumber } from 'lodash'; import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config'; @@ -241,7 +241,7 @@ export class DataProviderService { const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk)); promises.push( - promise.then((result) => { + promise.then(async (result) => { for (const [symbol, dataProviderResponse] of Object.entries( result )) { @@ -256,6 +256,38 @@ export class DataProviderService { 1000 ).toFixed(3)} seconds` ); + + try { + const date = getStartOfUtcDate(new Date()); + + // Upsert quotes by imitating missing upsertMany functionality + // with $transaction + const upsertPromises = Object.keys(response) + .filter((symbol) => { + return ( + isNumber(response[symbol].marketPrice) && + response[symbol].marketPrice > 0 + ); + }) + .map((symbol) => + this.prismaService.marketData.upsert({ + create: { + date, + symbol, + dataSource: response[symbol].dataSource, + marketPrice: response[symbol].marketPrice + }, + update: { + marketPrice: response[symbol].marketPrice + }, + where: { + date_symbol: { date, symbol } + } + }) + ); + + await this.prismaService.$transaction(upsertPromises); + } catch {} }) ); } diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts index 0bfefb729..d6da4ed94 100644 --- a/libs/common/src/lib/helper.ts +++ b/libs/common/src/lib/helper.ts @@ -152,6 +152,13 @@ export function getNumberFormatGroup(aLocale?: string) { }).value; } +export function getStartOfUtcDate(aDate: Date) { + const date = new Date(aDate); + date.setUTCHours(0, 0, 0, 0); + + return date; +} + export function getSum(aArray: Big[]) { if (aArray?.length > 0) { return aArray.reduce((a, b) => a.plus(b), new Big(0));