diff --git a/CHANGELOG.md b/CHANGELOG.md index 442ba1512..c648ceac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added the data source attribute to the symbol profile model + +### Changed + +- Respected the data source attribute in the data provider service +- Respected the data source attribute in the symbol data endpoint +- Improved the search functionality of the data management (multiple data sources) + ### Fixed - Hid the net performance in the _Presenter View_ (portfolio holdings and summary tab on the home page) - Hid the sign if the performance is zero in the value component +### Todo + +- Apply data migration (`yarn database:push`) + ## 1.53.0 - 13.09.2021 ### Changed diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts index ef9f9278c..e5bfaa1a5 100644 --- a/apps/api/src/app/order/order.service.ts +++ b/apps/api/src/app/order/order.service.ts @@ -56,7 +56,9 @@ export class OrderService { ]); } - this.dataGatheringService.gatherProfileData([data.symbol]); + this.dataGatheringService.gatherProfileData([ + { dataSource: data.dataSource, symbol: data.symbol } + ]); await this.cacheService.flush(); diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 5df9fc1c9..7054633a0 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -1,6 +1,6 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; -import { Currency, MarketData } from '@prisma/client'; +import { Currency, DataSource, MarketData } from '@prisma/client'; import { CurrentRateService } from './current-rate.service'; import { MarketDataService } from './market-data.service'; @@ -14,6 +14,7 @@ jest.mock('./market-data.service', () => { date, symbol, createdAt: date, + dataSource: DataSource.YAHOO, id: 'aefcbe3a-ee10-4c4f-9f2d-8ffad7b05584', marketPrice: 1847.839966 }); @@ -30,6 +31,7 @@ jest.mock('./market-data.service', () => { return Promise.resolve([ { createdAt: dateRangeStart, + dataSource: DataSource.YAHOO, date: dateRangeStart, id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d', marketPrice: 1841.823902, @@ -37,6 +39,7 @@ jest.mock('./market-data.service', () => { }, { createdAt: dateRangeEnd, + dataSource: DataSource.YAHOO, date: dateRangeEnd, id: '082d6893-df27-4c91-8a5d-092e84315b56', marketPrice: 1847.839966, @@ -106,11 +109,11 @@ describe('CurrentRateService', () => { expect( await currentRateService.getValues({ currencies: { AMZN: Currency.USD }, + dataGatheringItems: [{ dataSource: DataSource.YAHOO, symbol: 'AMZN' }], dateQuery: { lt: new Date(Date.UTC(2020, 0, 2, 0, 0, 0)), gte: new Date(Date.UTC(2020, 0, 1, 0, 0, 0)) }, - symbols: ['AMZN'], userCurrency: Currency.CHF }) ).toMatchObject([ diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 09c62a656..f52189a6d 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -2,6 +2,7 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service'; import { resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; import { isBefore, isToday } from 'date-fns'; import { flatten } from 'lodash'; @@ -25,7 +26,9 @@ export class CurrentRateService { userCurrency }: GetValueParams): Promise { if (isToday(date)) { - const dataProviderResult = await this.dataProviderService.get([symbol]); + const dataProviderResult = await this.dataProviderService.get([ + { symbol, dataSource: DataSource.YAHOO } + ]); return { date: resetHours(date), marketPrice: dataProviderResult?.[symbol]?.marketPrice ?? 0, @@ -55,8 +58,8 @@ export class CurrentRateService { public async getValues({ currencies, + dataGatheringItems, dateQuery, - symbols, userCurrency }: GetValuesParams): Promise { const includeToday = @@ -75,24 +78,31 @@ export class CurrentRateService { if (includeToday) { const today = resetHours(new Date()); promises.push( - this.dataProviderService.get(symbols).then((dataResultProvider) => { - const result = []; - for (const symbol of symbols) { - result.push({ - symbol, - date: today, - marketPrice: this.exchangeRateDataService.toCurrency( - dataResultProvider?.[symbol]?.marketPrice ?? 0, - dataResultProvider?.[symbol]?.currency, - userCurrency - ) - }); - } - return result; - }) + this.dataProviderService + .get(dataGatheringItems) + .then((dataResultProvider) => { + const result = []; + for (const dataGatheringItem of dataGatheringItems) { + result.push({ + date: today, + marketPrice: this.exchangeRateDataService.toCurrency( + dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice ?? + 0, + dataResultProvider?.[dataGatheringItem.symbol]?.currency, + userCurrency + ), + symbol: dataGatheringItem.symbol + }); + } + return result; + }) ); } + const symbols = dataGatheringItems.map((dataGatheringItem) => { + return dataGatheringItem.symbol; + }); + promises.push( this.marketDataService .getRange({ diff --git a/apps/api/src/app/portfolio/interfaces/get-values-params.interface.ts b/apps/api/src/app/portfolio/interfaces/get-values-params.interface.ts index df5261bcc..c736eebf2 100644 --- a/apps/api/src/app/portfolio/interfaces/get-values-params.interface.ts +++ b/apps/api/src/app/portfolio/interfaces/get-values-params.interface.ts @@ -1,10 +1,11 @@ +import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { Currency } from '@prisma/client'; import { DateQuery } from './date-query.interface'; export interface GetValuesParams { currencies: { [symbol: string]: Currency }; + dataGatheringItems: IDataGatheringItem[]; dateQuery: DateQuery; - symbols: string[]; userCurrency: Currency; } diff --git a/apps/api/src/app/portfolio/interfaces/portfolio-order.interface.ts b/apps/api/src/app/portfolio/interfaces/portfolio-order.interface.ts index 2b443236e..71e6cb99e 100644 --- a/apps/api/src/app/portfolio/interfaces/portfolio-order.interface.ts +++ b/apps/api/src/app/portfolio/interfaces/portfolio-order.interface.ts @@ -1,10 +1,11 @@ import { OrderType } from '@ghostfolio/api/models/order-type'; -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import Big from 'big.js'; export interface PortfolioOrder { currency: Currency; date: string; + dataSource: DataSource; fee: Big; name: string; quantity: Big; diff --git a/apps/api/src/app/portfolio/interfaces/transaction-point-symbol.interface.ts b/apps/api/src/app/portfolio/interfaces/transaction-point-symbol.interface.ts index 91dcdd63b..51465f8fd 100644 --- a/apps/api/src/app/portfolio/interfaces/transaction-point-symbol.interface.ts +++ b/apps/api/src/app/portfolio/interfaces/transaction-point-symbol.interface.ts @@ -1,8 +1,9 @@ -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import Big from 'big.js'; export interface TransactionPointSymbol { currency: Currency; + dataSource: DataSource; fee: Big; firstBuyDate: string; investment: Big; diff --git a/apps/api/src/app/portfolio/portfolio-calculator.spec.ts b/apps/api/src/app/portfolio/portfolio-calculator.spec.ts index d1f967fb2..9dda2e8f0 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.spec.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.spec.ts @@ -1,7 +1,7 @@ import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service'; import { OrderType } from '@ghostfolio/api/models/order-type'; import { parseDate, resetHours } from '@ghostfolio/common/helper'; -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import Big from 'big.js'; import { addDays, @@ -85,7 +85,7 @@ jest.mock('./current-rate.service', () => { getValues: ({ currencies, dateQuery, - symbols, + dataGatheringItems, userCurrency }: GetValuesParams) => { const result = []; @@ -95,21 +95,23 @@ jest.mock('./current-rate.service', () => { isBefore(date, endOfDay(dateQuery.lt)); date = addDays(date, 1) ) { - for (const symbol of symbols) { + for (const dataGatheringItem of dataGatheringItems) { result.push({ date, - symbol, - marketPrice: mockGetValue(symbol, date).marketPrice + marketPrice: mockGetValue(dataGatheringItem.symbol, date) + .marketPrice, + symbol: dataGatheringItem.symbol }); } } } else { for (const date of dateQuery.in) { - for (const symbol of symbols) { + for (const dataGatheringItem of dataGatheringItems) { result.push({ date, - symbol, - marketPrice: mockGetValue(symbol, date).marketPrice + marketPrice: mockGetValue(dataGatheringItem.symbol, date) + .marketPrice, + symbol: dataGatheringItem.symbol }); } } @@ -148,7 +150,7 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const orders = [ + const orders: PortfolioOrder[] = [ { date: '2019-02-01', name: 'Vanguard Total Stock Market Index Fund ETF Shares', @@ -157,6 +159,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Buy, unitPrice: new Big('144.38'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('5') }, { @@ -167,6 +170,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Buy, unitPrice: new Big('147.99'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('10') }, { @@ -177,6 +181,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Sell, unitPrice: new Big('151.41'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('5') } ]; @@ -189,6 +194,7 @@ describe('PortfolioCalculator', () => { date: '2019-02-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTI', investment: new Big('1443.8'), @@ -203,6 +209,7 @@ describe('PortfolioCalculator', () => { date: '2019-08-03', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('20'), symbol: 'VTI', investment: new Big('2923.7'), @@ -217,6 +224,7 @@ describe('PortfolioCalculator', () => { date: '2020-02-02', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'VTI', investment: new Big('652.55'), @@ -235,7 +243,7 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const orders = [ + const orders: PortfolioOrder[] = [ { date: '2019-02-01', name: 'Vanguard Total Stock Market Index Fund ETF Shares', @@ -244,6 +252,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Buy, unitPrice: new Big('144.38'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('5') }, { @@ -254,6 +263,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Buy, unitPrice: new Big('147.99'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('10') }, { @@ -264,6 +274,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Sell, unitPrice: new Big('151.41'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big('5') } ]; @@ -276,6 +287,7 @@ describe('PortfolioCalculator', () => { date: '2019-02-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTI', investment: new Big('1443.8'), @@ -290,6 +302,7 @@ describe('PortfolioCalculator', () => { date: '2019-08-03', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTI', investment: new Big('1443.8'), @@ -299,6 +312,7 @@ describe('PortfolioCalculator', () => { fee: new Big('5') }, { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTX', investment: new Big('1479.9'), @@ -313,6 +327,7 @@ describe('PortfolioCalculator', () => { date: '2020-02-02', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'VTI', investment: new Big('686.75'), @@ -322,6 +337,7 @@ describe('PortfolioCalculator', () => { fee: new Big('10') }, { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTX', investment: new Big('1479.9'), @@ -336,10 +352,11 @@ describe('PortfolioCalculator', () => { }); it('with two orders at the same day of the same type', () => { - const orders = [ + const orders: PortfolioOrder[] = [ ...ordersVTI, { currency: Currency.USD, + dataSource: DataSource.YAHOO, date: '2021-02-01', name: 'Vanguard Total Stock Market Index Fund ETF Shares', quantity: new Big('20'), @@ -363,6 +380,7 @@ describe('PortfolioCalculator', () => { items: [ { currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', investment: new Big('1443.8'), quantity: new Big('10'), @@ -377,6 +395,7 @@ describe('PortfolioCalculator', () => { items: [ { currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', investment: new Big('2923.7'), quantity: new Big('20'), @@ -391,6 +410,7 @@ describe('PortfolioCalculator', () => { items: [ { currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', investment: new Big('652.55'), quantity: new Big('5'), @@ -405,6 +425,7 @@ describe('PortfolioCalculator', () => { items: [ { currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', investment: new Big('6627.05'), quantity: new Big('35'), @@ -419,6 +440,7 @@ describe('PortfolioCalculator', () => { items: [ { currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', investment: new Big('8403.95'), quantity: new Big('45'), @@ -432,10 +454,11 @@ describe('PortfolioCalculator', () => { }); it('with additional order', () => { - const orders = [ + const orders: PortfolioOrder[] = [ ...ordersVTI, { currency: Currency.USD, + dataSource: DataSource.YAHOO, date: '2019-09-01', name: 'Amazon.com, Inc.', quantity: new Big('5'), @@ -458,6 +481,7 @@ describe('PortfolioCalculator', () => { date: '2019-02-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('10'), symbol: 'VTI', investment: new Big('1443.8'), @@ -472,6 +496,7 @@ describe('PortfolioCalculator', () => { date: '2019-08-03', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('20'), symbol: 'VTI', investment: new Big('2923.7'), @@ -486,6 +511,7 @@ describe('PortfolioCalculator', () => { date: '2019-09-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'AMZN', investment: new Big('10109.95'), @@ -495,6 +521,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('20'), symbol: 'VTI', investment: new Big('2923.7'), @@ -509,6 +536,7 @@ describe('PortfolioCalculator', () => { date: '2020-02-02', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'AMZN', investment: new Big('10109.95'), @@ -518,6 +546,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'VTI', investment: new Big('652.55'), @@ -532,6 +561,7 @@ describe('PortfolioCalculator', () => { date: '2021-02-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'AMZN', investment: new Big('10109.95'), @@ -541,6 +571,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('15'), symbol: 'VTI', investment: new Big('2684.05'), @@ -555,6 +586,7 @@ describe('PortfolioCalculator', () => { date: '2021-08-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'AMZN', investment: new Big('10109.95'), @@ -564,6 +596,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('25'), symbol: 'VTI', investment: new Big('4460.95'), @@ -578,7 +611,7 @@ describe('PortfolioCalculator', () => { }); it('with additional buy & sell', () => { - const orders = [ + const orders: PortfolioOrder[] = [ ...ordersVTI, { date: '2019-09-01', @@ -588,6 +621,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Buy, unitPrice: new Big('2021.99'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -598,6 +632,7 @@ describe('PortfolioCalculator', () => { type: OrderType.Sell, unitPrice: new Big('2412.23'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) } ]; @@ -628,6 +663,7 @@ describe('PortfolioCalculator', () => { date: '2017-01-03', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('50'), symbol: 'TSLA', investment: new Big('2148.5'), @@ -642,6 +678,7 @@ describe('PortfolioCalculator', () => { date: '2017-07-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('0.5614682'), symbol: 'BTCUSD', investment: new Big('1999.9999999999998659756'), @@ -651,6 +688,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('50'), symbol: 'TSLA', investment: new Big('2148.5'), @@ -665,6 +703,7 @@ describe('PortfolioCalculator', () => { date: '2018-09-01', items: [ { + dataSource: DataSource.YAHOO, quantity: new Big('5'), symbol: 'AMZN', investment: new Big('10109.95'), @@ -674,6 +713,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('0.5614682'), symbol: 'BTCUSD', investment: new Big('1999.9999999999998659756'), @@ -683,6 +723,7 @@ describe('PortfolioCalculator', () => { transactionCount: 1 }, { + dataSource: DataSource.YAHOO, quantity: new Big('50'), symbol: 'TSLA', investment: new Big('2148.5'), @@ -929,6 +970,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('805.9'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 1 @@ -943,6 +985,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('0'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 2 @@ -957,6 +1000,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('1013.9'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 3 @@ -1005,16 +1049,16 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const transactionPoints = [ + const transactionPoints: TransactionPoint[] = [ { date: '2019-02-01', items: [ { quantity: new Big('10'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 1 @@ -1026,10 +1070,10 @@ describe('PortfolioCalculator', () => { items: [ { quantity: new Big('20'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 2 @@ -1088,16 +1132,16 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const transactionPoints = [ + const transactionPoints: TransactionPoint[] = [ { date: '2019-02-01', items: [ { quantity: new Big('10'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(50), transactionCount: 1 @@ -1109,10 +1153,10 @@ describe('PortfolioCalculator', () => { items: [ { quantity: new Big('20'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(100), transactionCount: 2 @@ -1155,6 +1199,7 @@ describe('PortfolioCalculator', () => { positions: [ { averagePrice: new Big('146.185'), + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', quantity: new Big('20'), symbol: 'VTI', @@ -1180,16 +1225,16 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const transactionPoints = [ + const transactionPoints: TransactionPoint[] = [ { date: '2019-02-01', items: [ { quantity: new Big('10'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(50), transactionCount: 1 @@ -1201,10 +1246,10 @@ describe('PortfolioCalculator', () => { items: [ { quantity: new Big('20'), - name: 'Vanguard Total Stock Market Index Fund ETF Shares', symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(100), transactionCount: 2 @@ -1277,6 +1322,7 @@ describe('PortfolioCalculator', () => { symbol: 'MFA', // Mutual Fund A investment: new Big('1000000'), // 1 million currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2010-12-31', fee: new Big(0), transactionCount: 1 @@ -1291,6 +1337,7 @@ describe('PortfolioCalculator', () => { symbol: 'MFA', // Mutual Fund A investment: new Big('1100000'), // 1,000,000 + 100,000 currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2010-12-31', fee: new Big(0), transactionCount: 2 @@ -1352,6 +1399,7 @@ describe('PortfolioCalculator', () => { symbol: 'SPA', // Sub Portfolio A investment: new Big('200'), currency: Currency.CHF, + dataSource: DataSource.YAHOO, firstBuyDate: '2012-12-31', fee: new Big(0), transactionCount: 1 @@ -1361,6 +1409,7 @@ describe('PortfolioCalculator', () => { symbol: 'SPB', // Sub Portfolio B investment: new Big('300'), currency: Currency.CHF, + dataSource: DataSource.YAHOO, firstBuyDate: '2012-12-31', fee: new Big(0), transactionCount: 1 @@ -1375,6 +1424,7 @@ describe('PortfolioCalculator', () => { symbol: 'SPA', // Sub Portfolio A investment: new Big('200'), currency: Currency.CHF, + dataSource: DataSource.YAHOO, firstBuyDate: '2012-12-31', fee: new Big(0), transactionCount: 1 @@ -1384,6 +1434,7 @@ describe('PortfolioCalculator', () => { symbol: 'SPB', // Sub Portfolio B investment: new Big('300'), currency: Currency.CHF, + dataSource: DataSource.YAHOO, firstBuyDate: '2012-12-31', fee: new Big(0), transactionCount: 1 @@ -1488,7 +1539,7 @@ describe('PortfolioCalculator', () => { currentRateService, Currency.USD ); - const transactionPoints = [ + const transactionPoints: TransactionPoint[] = [ { date: '2019-02-01', items: [ @@ -1497,6 +1548,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(50), transactionCount: 1 @@ -1511,6 +1563,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(100), transactionCount: 2 @@ -1525,6 +1578,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('652.55'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(150), transactionCount: 3 @@ -1539,6 +1593,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('2684.05'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(200), transactionCount: 4 @@ -1553,6 +1608,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('4460.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(250), transactionCount: 5 @@ -2217,6 +2273,7 @@ describe('PortfolioCalculator', () => { symbol: 'AMZN', investment: new Big('10109.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 1 @@ -2226,6 +2283,7 @@ describe('PortfolioCalculator', () => { symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 1 @@ -2334,6 +2392,7 @@ const ordersMixedSymbols: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('42.97'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2344,6 +2403,7 @@ const ordersMixedSymbols: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('3562.089535970158'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2354,6 +2414,7 @@ const ordersMixedSymbols: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('2021.99'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) } ]; @@ -2367,6 +2428,7 @@ const ordersVTI: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('144.38'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2377,6 +2439,7 @@ const ordersVTI: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('147.99'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2387,6 +2450,7 @@ const ordersVTI: PortfolioOrder[] = [ type: OrderType.Sell, unitPrice: new Big('151.41'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2397,6 +2461,7 @@ const ordersVTI: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('177.69'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) }, { @@ -2407,6 +2472,7 @@ const ordersVTI: PortfolioOrder[] = [ type: OrderType.Buy, unitPrice: new Big('203.15'), currency: Currency.USD, + dataSource: DataSource.YAHOO, fee: new Big(0) } ]; @@ -2420,6 +2486,7 @@ const orderTslaTransactionPoint: TransactionPoint[] = [ symbol: 'TSLA', investment: new Big('719.46'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2021-01-01', fee: new Big(0), transactionCount: 1 @@ -2437,6 +2504,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 1 @@ -2451,6 +2519,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 2 @@ -2465,6 +2534,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ symbol: 'VTI', investment: new Big('652.55'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 3 @@ -2479,6 +2549,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ symbol: 'VTI', investment: new Big('2684.05'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 4 @@ -2493,6 +2564,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ symbol: 'VTI', investment: new Big('4460.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 5 @@ -2501,7 +2573,7 @@ const ordersVTITransactionPoints: TransactionPoint[] = [ } ]; -const transactionPointsBuyAndSell = [ +const transactionPointsBuyAndSell: TransactionPoint[] = [ { date: '2019-02-01', items: [ @@ -2510,6 +2582,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('1443.8'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 1 @@ -2524,6 +2597,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 2 @@ -2538,6 +2612,7 @@ const transactionPointsBuyAndSell = [ symbol: 'AMZN', investment: new Big('10109.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 1 @@ -2547,6 +2622,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('2923.7'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 2 @@ -2561,6 +2637,7 @@ const transactionPointsBuyAndSell = [ symbol: 'AMZN', investment: new Big('10109.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 1 @@ -2570,6 +2647,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('652.55'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 3 @@ -2584,6 +2662,7 @@ const transactionPointsBuyAndSell = [ symbol: 'AMZN', investment: new Big('0'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 2 @@ -2593,6 +2672,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('652.55'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 3 @@ -2607,6 +2687,7 @@ const transactionPointsBuyAndSell = [ symbol: 'AMZN', investment: new Big('0'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 2 @@ -2616,6 +2697,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('2684.05'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 4 @@ -2630,6 +2712,7 @@ const transactionPointsBuyAndSell = [ symbol: 'AMZN', investment: new Big('0'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-09-01', fee: new Big(0), transactionCount: 2 @@ -2639,6 +2722,7 @@ const transactionPointsBuyAndSell = [ symbol: 'VTI', investment: new Big('4460.95'), currency: Currency.USD, + dataSource: DataSource.YAHOO, firstBuyDate: '2019-02-01', fee: new Big(0), transactionCount: 5 diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts index 1f2f57a7d..a1be2a0b3 100644 --- a/apps/api/src/app/portfolio/portfolio-calculator.ts +++ b/apps/api/src/app/portfolio/portfolio-calculator.ts @@ -1,7 +1,8 @@ import { OrderType } from '@ghostfolio/api/models/order-type'; +import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { DATE_FORMAT, parseDate, resetHours } from '@ghostfolio/common/helper'; import { TimelinePosition } from '@ghostfolio/common/interfaces'; -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import Big from 'big.js'; import { addDays, @@ -59,6 +60,7 @@ export class PortfolioCalculator { .plus(oldAccumulatedSymbol.quantity); currentTransactionPointItem = { currency: order.currency, + dataSource: order.dataSource, fee: order.fee.plus(oldAccumulatedSymbol.fee), firstBuyDate: oldAccumulatedSymbol.firstBuyDate, investment: newQuantity.eq(0) @@ -74,6 +76,7 @@ export class PortfolioCalculator { } else { currentTransactionPointItem = { currency: order.currency, + dataSource: order.dataSource, fee: order.fee, firstBuyDate: order.date, investment: unitPrice.mul(order.quantity).mul(factor), @@ -153,12 +156,15 @@ export class PortfolioCalculator { let firstTransactionPoint: TransactionPoint = null; let firstIndex = this.transactionPoints.length; const dates = []; - const symbols = new Set(); + const dataGatheringItems: IDataGatheringItem[] = []; const currencies: { [symbol: string]: Currency } = {}; dates.push(resetHours(start)); for (const item of this.transactionPoints[firstIndex - 1].items) { - symbols.add(item.symbol); + dataGatheringItems.push({ + dataSource: item.dataSource, + symbol: item.symbol + }); currencies[item.symbol] = item.currency; } for (let i = 0; i < this.transactionPoints.length; i++) { @@ -178,10 +184,10 @@ export class PortfolioCalculator { const marketSymbols = await this.currentRateService.getValues({ currencies, + dataGatheringItems, dateQuery: { in: dates }, - symbols: Array.from(symbols), userCurrency: this.currency }); @@ -309,6 +315,7 @@ export class PortfolioCalculator { ? new Big(0) : item.investment.div(item.quantity), currency: item.currency, + dataSource: item.dataSource, firstBuyDate: item.firstBuyDate, grossPerformance: isValid ? grossPerformance[item.symbol] ?? null @@ -515,25 +522,28 @@ export class PortfolioCalculator { } = {}; if (j >= 0) { const currencies: { [name: string]: Currency } = {}; - const symbols: string[] = []; + const dataGatheringItems: IDataGatheringItem[] = []; for (const item of this.transactionPoints[j].items) { currencies[item.symbol] = item.currency; - symbols.push(item.symbol); + dataGatheringItems.push({ + dataSource: item.dataSource, + symbol: item.symbol + }); investment = investment.add(item.investment); fees = fees.add(item.fee); } let marketSymbols: GetValueObject[] = []; - if (symbols.length > 0) { + if (dataGatheringItems.length > 0) { try { marketSymbols = await this.currentRateService.getValues({ + currencies, + dataGatheringItems, dateQuery: { gte: startDate, lt: endOfDay(endDate) }, - symbols, - currencies, userCurrency: this.currency }); } catch (error) { diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts index 0375c6b31..88a8ffeeb 100644 --- a/apps/api/src/app/portfolio/portfolio.service.ts +++ b/apps/api/src/app/portfolio/portfolio.service.ts @@ -191,12 +191,18 @@ export class PortfolioService { ); const totalValue = currentPositions.currentValue.plus(cashDetails.balance); + const dataGatheringItems = currentPositions.positions.map((position) => { + return { + dataSource: position.dataSource, + symbol: position.symbol + }; + }); const symbols = currentPositions.positions.map( (position) => position.symbol ); const [dataProviderResponses, symbolProfiles] = await Promise.all([ - this.dataProviderService.get(symbols), + this.dataProviderService.get(dataGatheringItems), this.symbolProfileService.getSymbolProfiles(symbols) ]); @@ -297,6 +303,7 @@ export class PortfolioService { const portfolioOrders: PortfolioOrder[] = orders.map((order) => ({ currency: order.currency, + dataSource: order.dataSource, date: format(order.date, DATE_FORMAT), fee: new Big(order.fee), name: order.SymbolProfile?.name, @@ -326,6 +333,7 @@ export class PortfolioService { const { averagePrice, currency, + dataSource, firstBuyDate, marketPrice, quantity, @@ -351,7 +359,7 @@ export class PortfolioService { ); const historicalData = await this.dataProviderService.getHistorical( - [aSymbol], + [{ dataSource, symbol: aSymbol }], 'day', parseISO(firstBuyDate), new Date() @@ -421,11 +429,13 @@ export class PortfolioService { symbol: aSymbol }; } else { - const currentData = await this.dataProviderService.get([aSymbol]); + const currentData = await this.dataProviderService.get([ + { dataSource: DataSource.YAHOO, symbol: aSymbol } + ]); const marketPrice = currentData[aSymbol]?.marketPrice; let historicalData = await this.dataProviderService.getHistorical( - [aSymbol], + [{ dataSource: DataSource.YAHOO, symbol: aSymbol }], 'day', portfolioStart, new Date() @@ -507,10 +517,16 @@ export class PortfolioService { const positions = currentPositions.positions.filter( (item) => !item.quantity.eq(0) ); + const dataGatheringItem = positions.map((position) => { + return { + dataSource: position.dataSource, + symbol: position.symbol + }; + }); const symbols = positions.map((position) => position.symbol); const [dataProviderResponses, symbolProfiles] = await Promise.all([ - this.dataProviderService.get(symbols), + this.dataProviderService.get(dataGatheringItem), this.symbolProfileService.getSymbolProfiles(symbols) ]); @@ -813,6 +829,7 @@ export class PortfolioService { const userCurrency = this.request.user.Settings.currency; const portfolioOrders: PortfolioOrder[] = orders.map((order) => ({ currency: order.currency, + dataSource: order.dataSource, date: format(order.date, DATE_FORMAT), fee: new Big( this.exchangeRateDataService.toCurrency( diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts index 4cb52c6a9..845645574 100644 --- a/apps/api/src/app/symbol/symbol.controller.ts +++ b/apps/api/src/app/symbol/symbol.controller.ts @@ -10,6 +10,7 @@ import { } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; +import { DataSource } from '@prisma/client'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; import { isEmpty } from 'lodash'; @@ -46,10 +47,20 @@ export class SymbolController { /** * Must be after /lookup */ - @Get(':symbol') + @Get(':dataSource/:symbol') @UseGuards(AuthGuard('jwt')) - public async getPosition(@Param('symbol') symbol): Promise { - const result = await this.symbolService.get(symbol); + public async getSymbolData( + @Param('dataSource') dataSource: DataSource, + @Param('symbol') symbol: string + ): Promise { + if (!DataSource[dataSource]) { + throw new HttpException( + getReasonPhrase(StatusCodes.NOT_FOUND), + StatusCodes.NOT_FOUND + ); + } + + const result = await this.symbolService.get({ dataSource, symbol }); if (!result || isEmpty(result)) { throw new HttpException( diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 3fdb2f753..33307d6db 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -1,4 +1,5 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service'; +import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { Injectable } from '@nestjs/common'; import { Currency, DataSource } from '@prisma/client'; @@ -13,15 +14,15 @@ export class SymbolService { private readonly prismaService: PrismaService ) {} - public async get(aSymbol: string): Promise { - const response = await this.dataProviderService.get([aSymbol]); - const { currency, dataSource, marketPrice } = response[aSymbol] ?? {}; + public async get(dataGatheringItem: IDataGatheringItem): Promise { + const response = await this.dataProviderService.get([dataGatheringItem]); + const { currency, marketPrice } = response[dataGatheringItem.symbol] ?? {}; - if (dataSource && marketPrice) { + if (dataGatheringItem.dataSource && marketPrice) { return { - dataSource, marketPrice, - currency: (currency) + currency: (currency), + dataSource: dataGatheringItem.dataSource }; } diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts index 9464fe51b..2809d17e6 100644 --- a/apps/api/src/services/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering.service.ts @@ -3,17 +3,11 @@ import { currencyPairs, ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config'; -import { - DATE_FORMAT, - getUtc, - isGhostfolioScraperApiSymbol, - resetHours -} from '@ghostfolio/common/helper'; +import { DATE_FORMAT, getUtc, resetHours } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; import { differenceInHours, - endOfToday, format, getDate, getMonth, @@ -123,20 +117,17 @@ export class DataGatheringService { } } - public async gatherProfileData(aSymbols?: string[]) { + public async gatherProfileData(aDataGatheringItems?: IDataGatheringItem[]) { console.log('Profile data gathering has been started.'); console.time('data-gathering-profile'); - let symbols = aSymbols; + let dataGatheringItems = aDataGatheringItems; - if (!symbols) { - const dataGatheringItems = await this.getSymbolsProfileData(); - symbols = dataGatheringItems.map((dataGatheringItem) => { - return dataGatheringItem.symbol; - }); + if (!dataGatheringItems) { + dataGatheringItems = await this.getSymbolsProfileData(); } - const currentData = await this.dataProviderService.get(symbols); + const currentData = await this.dataProviderService.get(dataGatheringItems); for (const [ symbol, @@ -215,6 +206,7 @@ export class DataGatheringService { try { await this.prismaService.marketData.create({ data: { + dataSource, symbol, date: currentDate, marketPrice: lastMarketPrice diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 9e57129f3..0fe506a92 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -6,11 +6,7 @@ import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; -import { - DATE_FORMAT, - isGhostfolioScraperApiSymbol, - isRakutenRapidApiSymbol -} from '@ghostfolio/common/helper'; +import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource, MarketData } from '@prisma/client'; @@ -20,8 +16,8 @@ import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service'; import { GhostfolioScraperApiService } from './ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { RakutenRapidApiService } from './rakuten-rapid-api/rakuten-rapid-api.service'; import { - convertToYahooFinanceSymbol, - YahooFinanceService + YahooFinanceService, + convertToYahooFinanceSymbol } from './yahoo-finance/yahoo-finance.service'; @Injectable() @@ -37,53 +33,32 @@ export class DataProviderService { this.rakutenRapidApiService?.setPrisma(this.prismaService); } - public async get( - aSymbols: string[] - ): Promise<{ [symbol: string]: IDataProviderResponse }> { - if (aSymbols.length === 1) { - const symbol = aSymbols[0]; - - if (isGhostfolioScraperApiSymbol(symbol)) { - return this.ghostfolioScraperApiService.get(aSymbols); - } else if (isRakutenRapidApiSymbol(symbol)) { - return this.rakutenRapidApiService.get(aSymbols); - } - } - - const yahooFinanceSymbols = aSymbols - .filter((symbol) => { - return ( - !isGhostfolioScraperApiSymbol(symbol) && - !isRakutenRapidApiSymbol(symbol) - ); - }) - .map((symbol) => { - return convertToYahooFinanceSymbol(symbol); - }); - - const response = await this.yahooFinanceService.get(yahooFinanceSymbols); - - const ghostfolioScraperApiSymbols = aSymbols.filter((symbol) => { - return isGhostfolioScraperApiSymbol(symbol); - }); - - for (const symbol of ghostfolioScraperApiSymbols) { - if (symbol) { - const ghostfolioScraperApiResult = - await this.ghostfolioScraperApiService.get([symbol]); - response[symbol] = ghostfolioScraperApiResult[symbol]; - } - } - - const rakutenRapidApiSymbols = aSymbols.filter((symbol) => { - return isRakutenRapidApiSymbol(symbol); - }); + public async get(items: IDataGatheringItem[]): Promise<{ + [symbol: string]: IDataProviderResponse; + }> { + const response: { + [symbol: string]: IDataProviderResponse; + } = {}; - for (const symbol of rakutenRapidApiSymbols) { - if (symbol) { - const rakutenRapidApiResult = - await this.ghostfolioScraperApiService.get([symbol]); - response[symbol] = rakutenRapidApiResult[symbol]; + for (const item of items) { + if (item.dataSource === DataSource.ALPHA_VANTAGE) { + response[item.symbol] = ( + await this.alphaVantageService.get([item.symbol]) + )[item.symbol]; + } else if (item.dataSource === DataSource.GHOSTFOLIO) { + response[item.symbol] = ( + await this.ghostfolioScraperApiService.get([item.symbol]) + )[item.symbol]; + } else if (item.dataSource === DataSource.RAKUTEN) { + response[item.symbol] = ( + await this.rakutenRapidApiService.get([item.symbol]) + )[item.symbol]; + } else if (item.dataSource === DataSource.YAHOO) { + response[item.symbol] = ( + await this.yahooFinanceService.get([ + convertToYahooFinanceSymbol(item.symbol) + ]) + )[item.symbol]; } } @@ -91,7 +66,7 @@ export class DataProviderService { } public async getHistorical( - aSymbols: string[], + aItems: IDataGatheringItem[], aGranularity: Granularity = 'month', from: Date, to: Date @@ -115,8 +90,17 @@ export class DataProviderService { )}'` : ''; + const dataSources = aItems.map((item) => { + return item.dataSource; + }); + const symbols = aItems.map((item) => { + return item.symbol; + }); + try { - const queryRaw = `SELECT * FROM "MarketData" WHERE "symbol" IN ('${aSymbols.join( + const queryRaw = `SELECT * FROM "MarketData" WHERE "dataSource" IN ('${dataSources.join( + `','` + )}') AND "symbol" IN ('${symbols.join( `','` )}') ${granularityQuery} ${rangeQuery} ORDER BY date;`; @@ -175,13 +159,24 @@ export class DataProviderService { } public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { - const { items } = await this.getDataProvider( - this.configurationService.get('DATA_SOURCES')[0] - ).search(aSymbol); + const promises: Promise<{ items: LookupItem[] }>[] = []; + let lookupItems: LookupItem[] = []; + + for (const dataSource of this.configurationService.get('DATA_SOURCES')) { + promises.push( + this.getDataProvider(DataSource[dataSource]).search(aSymbol) + ); + } + + const searchResults = await Promise.all(promises); + + searchResults.forEach((searchResult) => { + lookupItems = lookupItems.concat(searchResult.items); + }); - const filteredItems = items.filter((item) => { + const filteredItems = lookupItems.filter((lookupItem) => { // Only allow symbols with supported currency - return item.currency ? true : false; + return lookupItem.currency ? true : false; }); return { diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index a230f88bd..aad7dbd0f 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -94,6 +94,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, + dataSource: DataSource.RAKUTEN, date: subWeeks(getToday(), 1), marketPrice: fgi.oneWeekAgo.value } @@ -102,6 +103,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, + dataSource: DataSource.RAKUTEN, date: subMonths(getToday(), 1), marketPrice: fgi.oneMonthAgo.value } @@ -110,6 +112,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, + dataSource: DataSource.RAKUTEN, date: subYears(getToday(), 1), marketPrice: fgi.oneYearAgo.value } diff --git a/apps/api/src/services/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data.service.ts index a2617647f..a9a4e2d6e 100644 --- a/apps/api/src/services/exchange-rate-data.service.ts +++ b/apps/api/src/services/exchange-rate-data.service.ts @@ -1,15 +1,16 @@ import { currencyPairs } from '@ghostfolio/common/config'; import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; import { Injectable } from '@nestjs/common'; -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import { format } from 'date-fns'; import { isEmpty, isNumber } from 'lodash'; import { DataProviderService } from './data-provider/data-provider.service'; +import { IDataGatheringItem } from './interfaces/interfaces'; @Injectable() export class ExchangeRateDataService { - private currencyPairs: string[] = []; + private currencyPairs: IDataGatheringItem[] = []; private exchangeRates: { [currencyPair: string]: number } = {}; public constructor(private dataProviderService: DataProviderService) { @@ -20,8 +21,8 @@ export class ExchangeRateDataService { this.currencyPairs = []; this.exchangeRates = {}; - for (const { currency1, currency2 } of currencyPairs) { - this.addCurrencyPairs(currency1, currency2); + for (const { currency1, currency2, dataSource } of currencyPairs) { + this.addCurrencyPairs({ currency1, currency2, dataSource }); } await this.loadCurrencies(); @@ -39,8 +40,8 @@ export class ExchangeRateDataService { // Load currencies directly from data provider as a fallback // if historical data is not yet available const historicalData = await this.dataProviderService.get( - this.currencyPairs.map((currencyPair) => { - return currencyPair; + this.currencyPairs.map(({ dataSource, symbol }) => { + return { dataSource, symbol }; }) ); @@ -67,21 +68,21 @@ export class ExchangeRateDataService { }; }); - this.currencyPairs.forEach((pair) => { - const [currency1, currency2] = pair.match(/.{1,3}/g); + this.currencyPairs.forEach(({ symbol }) => { + const [currency1, currency2] = symbol.match(/.{1,3}/g); const date = format(getYesterday(), DATE_FORMAT); - this.exchangeRates[pair] = resultExtended[pair]?.[date]?.marketPrice; + this.exchangeRates[symbol] = resultExtended[symbol]?.[date]?.marketPrice; - if (!this.exchangeRates[pair]) { + if (!this.exchangeRates[symbol]) { // Not found, calculate indirectly via USD - this.exchangeRates[pair] = + this.exchangeRates[symbol] = resultExtended[`${currency1}${Currency.USD}`]?.[date]?.marketPrice * resultExtended[`${Currency.USD}${currency2}`]?.[date]?.marketPrice; // Calculate the opposite direction this.exchangeRates[`${currency2}${currency1}`] = - 1 / this.exchangeRates[pair]; + 1 / this.exchangeRates[symbol]; } }); } @@ -123,8 +124,22 @@ export class ExchangeRateDataService { return aValue; } - private addCurrencyPairs(aCurrency1: Currency, aCurrency2: Currency) { - this.currencyPairs.push(`${aCurrency1}${aCurrency2}`); - this.currencyPairs.push(`${aCurrency2}${aCurrency1}`); + private addCurrencyPairs({ + currency1, + currency2, + dataSource + }: { + currency1: Currency; + currency2: Currency; + dataSource: DataSource; + }) { + this.currencyPairs.push({ + dataSource, + symbol: `${currency1}${currency2}` + }); + this.currencyPairs.push({ + dataSource, + symbol: `${currency2}${currency1}` + }); } } diff --git a/apps/client/src/app/pages/home/home-page.component.ts b/apps/client/src/app/pages/home/home-page.component.ts index ce9b96956..ce053661b 100644 --- a/apps/client/src/app/pages/home/home-page.component.ts +++ b/apps/client/src/app/pages/home/home-page.component.ts @@ -29,6 +29,7 @@ import { } from '@ghostfolio/common/interfaces'; import { hasPermission, permissions } from '@ghostfolio/common/permissions'; import { DateRange } from '@ghostfolio/common/types'; +import { DataSource } from '@prisma/client'; import { DeviceDetectorService } from 'ngx-device-detector'; import { Subject, Subscription } from 'rxjs'; import { takeUntil } from 'rxjs/operators'; @@ -112,7 +113,10 @@ export class HomePageComponent implements OnDestroy, OnInit { if (this.hasPermissionToAccessFearAndGreedIndex) { this.dataService - .fetchSymbolItem(ghostfolioFearAndGreedIndexSymbol) + .fetchSymbolItem({ + dataSource: DataSource.RAKUTEN, + symbol: ghostfolioFearAndGreedIndexSymbol + }) .pipe(takeUntil(this.unsubscribeSubject)) .subscribe(({ marketPrice }) => { this.fearAndGreedIndex = marketPrice; diff --git a/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.component.ts b/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.component.ts index a8473ee64..52c92449b 100644 --- a/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.component.ts +++ b/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.component.ts @@ -3,7 +3,8 @@ import { ChangeDetectorRef, Component, Inject, - OnDestroy + OnDestroy, + ViewChild } from '@angular/core'; import { FormControl, Validators } from '@angular/forms'; import { MatAutocompleteSelectedEvent } from '@angular/material/autocomplete'; @@ -11,6 +12,7 @@ import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { DataService } from '@ghostfolio/client/services/data.service'; import { Currency } from '@prisma/client'; +import { isString } from 'lodash'; import { EMPTY, Observable, Subject } from 'rxjs'; import { catchError, @@ -31,13 +33,18 @@ import { CreateOrUpdateTransactionDialogParams } from './interfaces/interfaces'; templateUrl: 'create-or-update-transaction-dialog.html' }) export class CreateOrUpdateTransactionDialog implements OnDestroy { + @ViewChild('autocomplete') autocomplete; + public currencies: Currency[] = []; public currentMarketPrice = null; public filteredLookupItems: Observable; public isLoading = false; public platforms: { id: string; name: string }[]; public searchSymbolCtrl = new FormControl( - this.data.transaction.symbol, + { + dataSource: this.data.transaction.dataSource, + name: this.data.transaction.symbol + }, Validators.required ); @@ -60,9 +67,9 @@ export class CreateOrUpdateTransactionDialog implements OnDestroy { startWith(''), debounceTime(400), distinctUntilChanged(), - switchMap((aQuery: string) => { - if (aQuery) { - return this.dataService.fetchSymbols(aQuery); + switchMap((query: string) => { + if (isString(query)) { + return this.dataService.fetchSymbols(query); } return []; @@ -71,7 +78,10 @@ export class CreateOrUpdateTransactionDialog implements OnDestroy { if (this.data.transaction.symbol) { this.dataService - .fetchSymbolItem(this.data.transaction.symbol) + .fetchSymbolItem({ + dataSource: this.data.transaction.dataSource, + symbol: this.data.transaction.symbol + }) .pipe(takeUntil(this.unsubscribeSubject)) .subscribe(({ marketPrice }) => { this.currentMarketPrice = marketPrice; @@ -85,9 +95,21 @@ export class CreateOrUpdateTransactionDialog implements OnDestroy { this.data.transaction.unitPrice = this.currentMarketPrice; } + public displayFn(aLookupItem: LookupItem) { + return aLookupItem?.name ?? ''; + } + public onBlurSymbol() { - const symbol = this.searchSymbolCtrl.value; - this.updateSymbol(symbol); + this.data.transaction.currency = null; + this.data.transaction.dataSource = null; + + if (this.autocomplete.isOpen) { + this.searchSymbolCtrl.setErrors({ incorrect: true }); + } else { + this.data.transaction.unitPrice = null; + } + + this.changeDetectorRef.markForCheck(); } public onCancel(): void { @@ -95,7 +117,8 @@ export class CreateOrUpdateTransactionDialog implements OnDestroy { } public onUpdateSymbol(event: MatAutocompleteSelectedEvent) { - this.updateSymbol(event.option.value); + this.data.transaction.dataSource = event.option.value.dataSource; + this.updateSymbol(event.option.value.symbol); } public ngOnDestroy() { @@ -106,10 +129,15 @@ export class CreateOrUpdateTransactionDialog implements OnDestroy { private updateSymbol(symbol: string) { this.isLoading = true; + this.searchSymbolCtrl.setErrors(null); + this.data.transaction.symbol = symbol; this.dataService - .fetchSymbolItem(this.data.transaction.symbol) + .fetchSymbolItem({ + dataSource: this.data.transaction.dataSource, + symbol: this.data.transaction.symbol + }) .pipe( catchError(() => { this.data.transaction.currency = null; diff --git a/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html b/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html index f7a18abca..8af61bc35 100644 --- a/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html +++ b/apps/client/src/app/pages/portfolio/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html @@ -28,18 +28,19 @@ matInput required [formControl]="searchSymbolCtrl" - [matAutocomplete]="auto" + [matAutocomplete]="autocomplete" (blur)="onBlurSymbol()" /> {{ lookupItem.symbol | gfSymbol }}{{ lookupItem.name }} diff --git a/apps/client/src/app/services/data.service.ts b/apps/client/src/app/services/data.service.ts index 1d3b56361..936efc888 100644 --- a/apps/client/src/app/services/data.service.ts +++ b/apps/client/src/app/services/data.service.ts @@ -29,8 +29,11 @@ import { import { InvestmentItem } from '@ghostfolio/common/interfaces/investment-item.interface'; import { permissions } from '@ghostfolio/common/permissions'; import { DateRange } from '@ghostfolio/common/types'; -import { Order as OrderModel } from '@prisma/client'; -import { Account as AccountModel } from '@prisma/client'; +import { + Account as AccountModel, + DataSource, + Order as OrderModel +} from '@prisma/client'; import { parseISO } from 'date-fns'; import { cloneDeep } from 'lodash'; import { Observable } from 'rxjs'; @@ -108,8 +111,14 @@ export class DataService { return info; } - public fetchSymbolItem(aSymbol: string) { - return this.http.get(`/api/symbol/${aSymbol}`); + public fetchSymbolItem({ + dataSource, + symbol + }: { + dataSource: DataSource; + symbol: string; + }) { + return this.http.get(`/api/symbol/${dataSource}/${symbol}`); } public fetchPositions({ diff --git a/libs/common/src/lib/interfaces/timeline-position.interface.ts b/libs/common/src/lib/interfaces/timeline-position.interface.ts index 76604955a..f7f30f107 100644 --- a/libs/common/src/lib/interfaces/timeline-position.interface.ts +++ b/libs/common/src/lib/interfaces/timeline-position.interface.ts @@ -1,9 +1,10 @@ -import { Currency } from '@prisma/client'; +import { Currency, DataSource } from '@prisma/client'; import Big from 'big.js'; export interface TimelinePosition { averagePrice: Big; currency: Currency; + dataSource: DataSource; firstBuyDate: string; grossPerformance: Big; grossPerformancePercentage: Big; diff --git a/prisma/migrations/20210916182355_added_data_source_to_market_data/migration.sql b/prisma/migrations/20210916182355_added_data_source_to_market_data/migration.sql new file mode 100644 index 000000000..83911236a --- /dev/null +++ b/prisma/migrations/20210916182355_added_data_source_to_market_data/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "MarketData" ADD COLUMN "dataSource" "DataSource" NOT NULL DEFAULT E'YAHOO'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 15bf573c1..8cff7862c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -61,9 +61,10 @@ model AuthDevice { } model MarketData { - createdAt DateTime @default(now()) + createdAt DateTime @default(now()) + dataSource DataSource @default(YAHOO) date DateTime - id String @default(uuid()) + id String @default(uuid()) symbol String marketPrice Float