Feature/respect data source in data gathering (#107)

* Respect data source in data gathering

* Update changelog

* optimize fetching from multiple data sources (#123)

* optimize fetching from multiple data sources

* improve performance by executing data gathering promises in parallel

* removed unused imports

* rename hasHistoricalData to canHandle

* Sort imports

* Clean up

Co-authored-by: Valentin Zickner <3200232+vzickner@users.noreply.github.com>
pull/126/head
Thomas 3 years ago committed by GitHub
parent c0657a2e9e
commit 11b2379d98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Respected the data source attribute of the transactions model in the data management for historical data
## 1.8.0 - 24.05.2021
### Added

@ -37,7 +37,9 @@ export class ExperimentalController {
);
}
return benchmarks;
return benchmarks.map(({ symbol }) => {
return symbol;
});
}
@Get('benchmarks/:symbol')

@ -2,7 +2,7 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.se
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { OrderWithAccount } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { Order, Prisma } from '@prisma/client';
import { DataSource, Order, Prisma } from '@prisma/client';
import { CacheService } from '../cache/cache.service';
import { RedisCacheService } from '../redis-cache/redis-cache.service';
@ -53,6 +53,7 @@ export class OrderService {
// Gather symbol data of order in the background
this.dataGatheringService.gatherSymbols([
{
dataSource: data.dataSource,
date: <Date>data.date,
symbol: data.symbol
}
@ -90,6 +91,7 @@ export class OrderService {
// Gather symbol data of order in the background
this.dataGatheringService.gatherSymbols([
{
dataSource: <DataSource>data.dataSource,
date: <Date>data.date,
symbol: <string>data.symbol
}

@ -11,6 +11,7 @@ import {
import { DateRange, RequestWithUser } from '@ghostfolio/common/types';
import { Inject, Injectable } from '@nestjs/common';
import { REQUEST } from '@nestjs/core';
import { DataSource } from '@prisma/client';
import {
add,
format,
@ -289,7 +290,7 @@ export class PortfolioService {
if (isEmpty(historicalData)) {
historicalData = await this.dataProviderService.getHistoricalRaw(
[aSymbol],
[{ dataSource: DataSource.YAHOO, symbol: aSymbol }],
portfolio.getMinDate(),
new Date()
);

@ -5,6 +5,7 @@ import {
resetHours
} from '@ghostfolio/common/helper';
import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import {
differenceInHours,
format,
@ -18,6 +19,7 @@ import {
import { ConfigurationService } from './configuration.service';
import { DataProviderService } from './data-provider.service';
import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { IDataGatheringItem } from './interfaces/interfaces';
import { PrismaService } from './prisma.service';
@Injectable()
@ -115,15 +117,13 @@ export class DataGatheringService {
}
}
public async gatherSymbols(
aSymbolsWithStartDate: { date: Date; symbol: string }[]
) {
public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
let hasError = false;
for (const { date, symbol } of aSymbolsWithStartDate) {
for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
try {
const historicalData = await this.dataProviderService.getHistoricalRaw(
[symbol],
[{ dataSource, symbol }],
date,
new Date()
);
@ -184,20 +184,24 @@ export class DataGatheringService {
}
}
public async getCustomSymbolsToGather(startDate?: Date) {
public async getCustomSymbolsToGather(
startDate?: Date
): Promise<IDataGatheringItem[]> {
const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations();
return scraperConfigurations.map((scraperConfiguration) => {
return {
dataSource: DataSource.GHOSTFOLIO,
date: startDate,
symbol: scraperConfiguration.symbol
};
});
}
private getBenchmarksToGather(startDate: Date) {
const benchmarksToGather = benchmarks.map((symbol) => {
private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
return {
dataSource,
symbol,
date: startDate
};
@ -205,6 +209,7 @@ export class DataGatheringService {
if (this.configurationService.get('ENABLE_FEATURE_FEAR_AND_GREED_INDEX')) {
benchmarksToGather.push({
dataSource: DataSource.RAKUTEN,
date: startDate,
symbol: 'GF.FEAR_AND_GREED_INDEX'
});
@ -213,16 +218,16 @@ export class DataGatheringService {
return benchmarksToGather;
}
private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
private async getSymbols7D(): Promise<IDataGatheringItem[]> {
const startDate = subDays(resetHours(new Date()), 7);
const distinctOrders = await this.prisma.order.findMany({
distinct: ['symbol'],
orderBy: [{ symbol: 'asc' }],
select: { symbol: true }
select: { dataSource: true, symbol: true }
});
const distinctOrdersWithDate = distinctOrders
const distinctOrdersWithDate: IDataGatheringItem[] = distinctOrders
.filter((distinctOrder) => {
return !isGhostfolioScraperApiSymbol(distinctOrder.symbol);
})
@ -233,12 +238,15 @@ export class DataGatheringService {
};
});
const currencyPairsToGather = currencyPairs.map((symbol) => {
return {
symbol,
date: startDate
};
});
const currencyPairsToGather = currencyPairs.map(
({ dataSource, symbol }) => {
return {
dataSource,
symbol,
date: startDate
};
}
);
const customSymbolsToGather = await this.getCustomSymbolsToGather(
startDate
@ -252,24 +260,27 @@ export class DataGatheringService {
];
}
private async getSymbolsMax() {
private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
const startDate = new Date(getUtc('2015-01-01'));
const customSymbolsToGather = await this.getCustomSymbolsToGather(
startDate
);
const currencyPairsToGather = currencyPairs.map((symbol) => {
return {
symbol,
date: startDate
};
});
const currencyPairsToGather = currencyPairs.map(
({ dataSource, symbol }) => {
return {
dataSource,
symbol,
date: startDate
};
}
);
const distinctOrders = await this.prisma.order.findMany({
distinct: ['symbol'],
orderBy: [{ date: 'asc' }],
select: { date: true, symbol: true }
select: { dataSource: true, date: true, symbol: true }
});
return [

@ -1,6 +1,4 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import {
isCrypto,
isGhostfolioScraperApiSymbol,
isRakutenRapidApiSymbol
} from '@ghostfolio/common/helper';
@ -16,6 +14,7 @@ import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakute
import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
import { DataProviderInterface } from './interfaces/data-provider.interface';
import {
IDataGatheringItem,
IDataProviderHistoricalResponse,
IDataProviderResponse
} from './interfaces/interfaces';
@ -121,79 +120,53 @@ export class DataProviderService implements DataProviderInterface {
}
public async getHistoricalRaw(
aSymbols: string[],
aDataGatheringItems: IDataGatheringItem[],
from: Date,
to: Date
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
const filteredSymbols = aSymbols.filter((symbol) => {
return !isGhostfolioScraperApiSymbol(symbol);
});
const dataOfYahoo = await this.yahooFinanceService.getHistorical(
filteredSymbols,
undefined,
from,
to
);
if (aSymbols.length === 1) {
const symbol = aSymbols[0];
if (
isCrypto(symbol) &&
this.configurationService.get('ALPHA_VANTAGE_API_KEY')
) {
// Merge data from Yahoo with data from Alpha Vantage
const dataOfAlphaVantage = await this.alphaVantageService.getHistorical(
[symbol],
undefined,
from,
to
);
return {
[symbol]: {
...dataOfYahoo[symbol],
...dataOfAlphaVantage[symbol]
}
};
} else if (isGhostfolioScraperApiSymbol(symbol)) {
const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
[symbol],
undefined,
from,
to
);
const result: {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
} = {};
return dataOfGhostfolioScraperApi;
} else if (
isRakutenRapidApiSymbol(symbol) &&
this.configurationService.get('RAKUTEN_RAPID_API_KEY')
) {
const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical(
[symbol],
undefined,
from,
to
const promises: Promise<{
data: { [date: string]: IDataProviderHistoricalResponse };
symbol: string;
}>[] = [];
for (const { dataSource, symbol } of aDataGatheringItems) {
const dataProvider = this.getDataProvider(dataSource);
if (dataProvider.canHandle(symbol)) {
promises.push(
dataProvider
.getHistorical([symbol], undefined, from, to)
.then((data) => ({ data: data?.[symbol], symbol }))
);
return dataOfRakutenRapidApi;
}
}
return dataOfYahoo;
const allData = await Promise.all(promises);
for (const { data, symbol } of allData) {
result[symbol] = data;
}
return result;
}
public async search(aSymbol: string) {
return this.getDataProvider().search(aSymbol);
return this.getDataProvider(
this.configurationService.get('DATA_SOURCES')[0]
).search(aSymbol);
}
private getDataProvider() {
switch (this.configurationService.get('DATA_SOURCES')[0]) {
private getDataProvider(providerName: DataSource) {
switch (providerName) {
case DataSource.ALPHA_VANTAGE:
return this.alphaVantageService;
case DataSource.GHOSTFOLIO:
return this.ghostfolioScraperApiService;
case DataSource.RAKUTEN:
return this.rakutenRapidApiService;
case DataSource.YAHOO:
return this.yahooFinanceService;
default:

@ -24,6 +24,10 @@ export class AlphaVantageService implements DataProviderInterface {
});
}
public canHandle(symbol: string) {
return this.configurationService.get('ALPHA_VANTAGE_API_KEY');
}
public async get(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {

@ -1,4 +1,7 @@
import { getYesterday } from '@ghostfolio/common/helper';
import {
getYesterday,
isGhostfolioScraperApiSymbol
} from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
@ -21,6 +24,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
public constructor(private prisma: PrismaService) {}
public canHandle(symbol: string) {
return isGhostfolioScraperApiSymbol(symbol);
}
public async get(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {

@ -1,4 +1,8 @@
import { getToday, getYesterday } from '@ghostfolio/common/helper';
import {
getToday,
getYesterday,
isRakutenRapidApiSymbol
} from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
@ -24,6 +28,13 @@ export class RakutenRapidApiService implements DataProviderInterface {
private readonly configurationService: ConfigurationService
) {}
public canHandle(symbol: string) {
return (
isRakutenRapidApiSymbol(symbol) &&
this.configurationService.get('RAKUTEN_RAPID_API_KEY')
);
}
public async get(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {

@ -28,6 +28,10 @@ export class YahooFinanceService implements DataProviderInterface {
public constructor() {}
public canHandle(symbol: string) {
return true;
}
public async get(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {

@ -7,6 +7,8 @@ import {
} from './interfaces';
export interface DataProviderInterface {
canHandle(symbol: string): boolean;
get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>;
getHistorical(

@ -65,6 +65,12 @@ export interface IDataProviderResponse {
url?: string;
}
export interface IDataGatheringItem {
dataSource: DataSource;
date?: Date;
symbol: string;
}
export type Industry = typeof Industry[keyof typeof Industry];
export type MarketState = typeof MarketState[keyof typeof MarketState];

@ -1,13 +1,17 @@
import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
import { Currency } from '@prisma/client';
import { DataSource } from '@prisma/client';
export const baseCurrency = Currency.CHF;
export const benchmarks = ['VOO'];
export const benchmarks: Partial<IDataGatheringItem>[] = [
{ dataSource: DataSource.YAHOO, symbol: 'VOO' }
];
export const currencyPairs = [
`${Currency.USD}${Currency.EUR}`,
`${Currency.USD}${Currency.GBP}`,
`${Currency.USD}${Currency.CHF}`
export const currencyPairs: Partial<IDataGatheringItem>[] = [
{ dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.EUR}` },
{ dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.GBP}` },
{ dataSource: DataSource.YAHOO, symbol: `${Currency.USD}${Currency.CHF}` }
];
export const ghostfolioScraperApiSymbolPrefix = '_GF_';

Loading…
Cancel
Save