add sectors and countries for ETFs (#410)

* Update changelog

Co-Authored-By: Valentin Zickner <valentin@coderworks.de>

Co-authored-by: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
pull/413/head
Valentin Zickner 3 years ago committed by GitHub
parent a7a6b0608b
commit f308ae7a13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Added
- Added a data enhancer for symbol profile data (countries and sectors) via _Trackinsight_
### Changed
- Changed the values of the global heat map to fixed-point notation

@ -2,11 +2,7 @@ import { CacheService } from '@ghostfolio/api/app/cache/cache.service';
import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { Module } from '@nestjs/common';
@ -14,18 +10,13 @@ import { Module } from '@nestjs/common';
import { CacheController } from './cache.controller';
@Module({
imports: [ExchangeRateDataModule, RedisCacheModule],
imports: [DataProviderModule, ExchangeRateDataModule, RedisCacheModule],
controllers: [CacheController],
providers: [
AlphaVantageService,
CacheService,
ConfigurationService,
DataGatheringService,
DataProviderService,
GhostfolioScraperApiService,
PrismaService,
RakutenRapidApiService,
YahooFinanceService
PrismaService
]
})
export class CacheModule {}

@ -1,10 +1,6 @@
import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { Module } from '@nestjs/common';
@ -15,6 +11,7 @@ import { InfoService } from './info.service';
@Module({
imports: [
DataProviderModule,
ExchangeRateDataModule,
JwtModule.register({
secret: process.env.JWT_SECRET_KEY,
@ -23,15 +20,10 @@ import { InfoService } from './info.service';
],
controllers: [InfoController],
providers: [
AlphaVantageService,
ConfigurationService,
DataGatheringService,
DataProviderService,
GhostfolioScraperApiService,
InfoService,
PrismaService,
RakutenRapidApiService,
YahooFinanceService
PrismaService
]
})
export class InfoModule {}

@ -75,6 +75,7 @@ describe('CurrentRateService', () => {
dataProviderService = new DataProviderService(
null,
null,
[],
null,
null,
null,

@ -132,7 +132,15 @@ export class DataGatheringService {
for (const [
symbol,
{ assetClass, assetSubClass, countries, currency, dataSource, name }
{
assetClass,
assetSubClass,
countries,
currency,
dataSource,
name,
sectors
}
] of Object.entries(currentData)) {
try {
await this.prismaService.symbolProfile.upsert({
@ -143,6 +151,7 @@ export class DataGatheringService {
currency,
dataSource,
name,
sectors,
symbol
},
update: {
@ -150,7 +159,8 @@ export class DataGatheringService {
assetSubClass,
countries,
currency,
name
name,
sectors
},
where: {
dataSource_symbol: {

@ -6,11 +6,11 @@ import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import { isAfter, isBefore, parse } from 'date-fns';
import { DataProviderInterface } from '../../interfaces/data-provider.interface';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface';
import { IAlphaVantageHistoricalResponse } from './interfaces/interfaces';
@Injectable()

@ -0,0 +1,73 @@
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
import bent from 'bent';
const getJSON = bent('json');
export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
private static baseUrl = 'https://data.trackinsight.com/holdings';
private static countries = require('countries-list/dist/countries.json');
private static sectorsMapping = {
'Consumer Discretionary': 'Consumer Cyclical',
'Consumer Defensive': 'Consumer Staples',
'Health Care': 'Healthcare',
'Information Technology': 'Technology'
};
public async enhance({
response,
symbol
}: {
response: IDataProviderResponse;
symbol: string;
}): Promise<IDataProviderResponse> {
if (
!(response.assetClass === 'EQUITY' && response.assetSubClass === 'ETF')
) {
return response;
}
const holdings = await getJSON(
`${TrackinsightDataEnhancerService.baseUrl}/${symbol}.json`
).catch(() => {
return getJSON(
`${TrackinsightDataEnhancerService.baseUrl}/${
symbol.split('.')[0]
}.json`
);
});
if (!response.countries || response.countries.length === 0) {
response.countries = [];
for (const [name, value] of Object.entries<any>(holdings.countries)) {
let countryCode: string;
for (const [key, country] of Object.entries<any>(
TrackinsightDataEnhancerService.countries
)) {
if (country.name === name) {
countryCode = key;
break;
}
}
response.countries.push({
code: countryCode,
weight: value.weight
});
}
}
if (!response.sectors || response.sectors.length === 0) {
response.sectors = [];
for (const [name, value] of Object.entries<any>(holdings.sectors)) {
response.sectors.push({
name: TrackinsightDataEnhancerService.sectorsMapping[name] ?? name,
weight: value.weight
});
}
}
return Promise.resolve(response);
}
}

@ -1,4 +1,5 @@
import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
@ -15,7 +16,13 @@ import { DataProviderService } from './data-provider.service';
DataProviderService,
GhostfolioScraperApiService,
RakutenRapidApiService,
YahooFinanceService
TrackinsightDataEnhancerService,
YahooFinanceService,
{
inject: [TrackinsightDataEnhancerService],
provide: 'DataEnhancers',
useFactory: (trackinsight) => [trackinsight]
}
],
exports: [DataProviderService, GhostfolioScraperApiService]
})

@ -1,5 +1,6 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
import {
IDataGatheringItem,
IDataProviderHistoricalResponse,
@ -8,7 +9,7 @@ import {
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { DataSource, MarketData } from '@prisma/client';
import { format } from 'date-fns';
import { isEmpty } from 'lodash';
@ -16,16 +17,15 @@ import { isEmpty } from 'lodash';
import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service';
import { GhostfolioScraperApiService } from './ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { RakutenRapidApiService } from './rakuten-rapid-api/rakuten-rapid-api.service';
import {
YahooFinanceService,
convertToYahooFinanceSymbol
} from './yahoo-finance/yahoo-finance.service';
import { YahooFinanceService } from './yahoo-finance/yahoo-finance.service';
@Injectable()
export class DataProviderService {
public constructor(
private readonly alphaVantageService: AlphaVantageService,
private readonly configurationService: ConfigurationService,
@Inject('DataEnhancers')
private readonly dataEnhancers: DataEnhancerInterface[],
private readonly ghostfolioScraperApiService: GhostfolioScraperApiService,
private readonly prismaService: PrismaService,
private readonly rakutenRapidApiService: RakutenRapidApiService,
@ -42,27 +42,35 @@ export class DataProviderService {
} = {};
for (const item of items) {
if (item.dataSource === DataSource.ALPHA_VANTAGE) {
response[item.symbol] = (
await this.alphaVantageService.get([item.symbol])
)[item.symbol];
} else if (item.dataSource === DataSource.GHOSTFOLIO) {
response[item.symbol] = (
await this.ghostfolioScraperApiService.get([item.symbol])
)[item.symbol];
} else if (item.dataSource === DataSource.RAKUTEN) {
response[item.symbol] = (
await this.rakutenRapidApiService.get([item.symbol])
)[item.symbol];
} else if (item.dataSource === DataSource.YAHOO) {
response[item.symbol] = (
await this.yahooFinanceService.get([
convertToYahooFinanceSymbol(item.symbol)
])
)[item.symbol];
const dataProvider = this.getDataProvider(item.dataSource);
response[item.symbol] = (await dataProvider.get([item.symbol]))[
item.symbol
];
}
const promises = [];
for (const symbol of Object.keys(response)) {
let promise = Promise.resolve(response[symbol]);
for (const dataEnhancer of this.dataEnhancers) {
promise = promise.then((currentResponse) =>
dataEnhancer
.enhance({ symbol, response: currentResponse })
.catch((error) => {
console.error(
`Failed to enhance data for symbol ${symbol}`,
error
);
return currentResponse;
})
);
}
promises.push(
promise.then((currentResponse) => (response[symbol] = currentResponse))
);
}
await Promise.all(promises);
return response;
}
@ -103,11 +111,13 @@ export class DataProviderService {
});
try {
const queryRaw = `SELECT * FROM "MarketData" WHERE "dataSource" IN ('${dataSources.join(
`','`
)}') AND "symbol" IN ('${symbols.join(
const queryRaw = `SELECT *
FROM "MarketData"
WHERE "dataSource" IN ('${dataSources.join(`','`)}')
AND "symbol" IN ('${symbols.join(
`','`
)}') ${granularityQuery} ${rangeQuery} ORDER BY date;`;
)}') ${granularityQuery} ${rangeQuery}
ORDER BY date;`;
const marketDataByGranularity: MarketData[] =
await this.prismaService.$queryRaw(queryRaw);

@ -12,13 +12,13 @@ import * as bent from 'bent';
import * as cheerio from 'cheerio';
import { format } from 'date-fns';
import { DataProviderInterface } from '../../interfaces/data-provider.interface';
import {
IDataGatheringItem,
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface';
import { ScraperConfig } from './interfaces/scraper-config.interface';
@Injectable()

@ -0,0 +1,11 @@
import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
export interface DataEnhancerInterface {
enhance({
response,
symbol
}: {
response: IDataProviderResponse;
symbol: string;
}): Promise<IDataProviderResponse>;
}

@ -4,7 +4,7 @@ import { Granularity } from '@ghostfolio/common/types';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from './interfaces';
} from '../../interfaces/interfaces';
export interface DataProviderInterface {
canHandle(symbol: string): boolean;

@ -14,12 +14,12 @@ import { DataSource } from '@prisma/client';
import * as bent from 'bent';
import { format, subMonths, subWeeks, subYears } from 'date-fns';
import { DataProviderInterface } from '../../interfaces/data-provider.interface';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface';
@Injectable()
export class RakutenRapidApiService implements DataProviderInterface {

@ -10,12 +10,12 @@ import { countries } from 'countries-list';
import { format } from 'date-fns';
import * as yahooFinance from 'yahoo-finance';
import { DataProviderInterface } from '../../interfaces/data-provider.interface';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface';
import {
IYahooFinanceHistoricalResponse,
IYahooFinancePrice,
@ -33,11 +33,14 @@ export class YahooFinanceService implements DataProviderInterface {
}
public async get(
aYahooFinanceSymbols: string[]
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aYahooFinanceSymbols.length <= 0) {
if (aSymbols.length <= 0) {
return {};
}
const yahooFinanceSymbols = aSymbols.map((symbol) =>
this.convertToYahooFinanceSymbol(symbol)
);
try {
const response: { [symbol: string]: IDataProviderResponse } = {};
@ -46,12 +49,12 @@ export class YahooFinanceService implements DataProviderInterface {
[symbol: string]: IYahooFinanceQuoteResponse;
} = await yahooFinance.quote({
modules: ['price', 'summaryProfile'],
symbols: aYahooFinanceSymbols
symbols: yahooFinanceSymbols
});
for (const [yahooFinanceSymbol, value] of Object.entries(data)) {
// Convert symbols back
const symbol = convertFromYahooFinanceSymbol(yahooFinanceSymbol);
const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol);
const { assetClass, assetSubClass } = this.parseAssetClass(value.price);
@ -93,6 +96,12 @@ export class YahooFinanceService implements DataProviderInterface {
response[symbol].countries = [{ code, weight: 1 }];
}
} catch {}
if (value.summaryProfile?.sector) {
response[symbol].sectors = [
{ name: value.summaryProfile?.sector, weight: 1 }
];
}
}
// Add url if available
@ -123,7 +132,7 @@ export class YahooFinanceService implements DataProviderInterface {
}
const yahooFinanceSymbols = aSymbols.map((symbol) => {
return convertToYahooFinanceSymbol(symbol);
return this.convertToYahooFinanceSymbol(symbol);
});
try {
@ -143,7 +152,7 @@ export class YahooFinanceService implements DataProviderInterface {
historicalData
)) {
// Convert symbols back
const symbol = convertFromYahooFinanceSymbol(yahooFinanceSymbol);
const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol);
response[symbol] = {};
timeSeries.forEach((timeSerie) => {
@ -214,6 +223,40 @@ export class YahooFinanceService implements DataProviderInterface {
return { items };
}
private convertFromYahooFinanceSymbol(aYahooFinanceSymbol: string) {
const symbol = aYahooFinanceSymbol.replace('-', '');
return symbol.replace('=X', '');
}
/**
* Converts a symbol to a Yahoo Finance symbol
*
* Currency: USDCHF -> USDCHF=X
* Cryptocurrency: BTCUSD -> BTC-USD
* DOGEUSD -> DOGE-USD
* SOL1USD -> SOL1-USD
*/
private convertToYahooFinanceSymbol(aSymbol: string) {
if (
(aSymbol.includes('CHF') ||
aSymbol.includes('EUR') ||
aSymbol.includes('USD')) &&
aSymbol.length >= 6
) {
if (isCurrency(aSymbol.substring(0, aSymbol.length - 3))) {
return `${aSymbol}=X`;
} else if (isCrypto(aSymbol) || isCrypto(aSymbol.replace('1', ''))) {
// Add a dash before the last three characters
// BTCUSD -> BTC-USD
// DOGEUSD -> DOGE-USD
// SOL1USD -> SOL1-USD
return aSymbol.replace('USD', '-USD');
}
}
return aSymbol;
}
private parseAssetClass(aPrice: IYahooFinancePrice): {
assetClass: AssetClass;
assetSubClass: AssetSubClass;
@ -247,37 +290,3 @@ export class YahooFinanceService implements DataProviderInterface {
return aString;
}
}
export const convertFromYahooFinanceSymbol = (aYahooFinanceSymbol: string) => {
const symbol = aYahooFinanceSymbol.replace('-', '');
return symbol.replace('=X', '');
};
/**
* Converts a symbol to a Yahoo Finance symbol
*
* Currency: USDCHF -> USDCHF=X
* Cryptocurrency: BTCUSD -> BTC-USD
* DOGEUSD -> DOGE-USD
* SOL1USD -> SOL1-USD
*/
export const convertToYahooFinanceSymbol = (aSymbol: string) => {
if (
(aSymbol.includes('CHF') ||
aSymbol.includes('EUR') ||
aSymbol.includes('USD')) &&
aSymbol.length >= 6
) {
if (isCurrency(aSymbol.substring(0, aSymbol.length - 3))) {
return `${aSymbol}=X`;
} else if (isCrypto(aSymbol) || isCrypto(aSymbol.replace('1', ''))) {
// Add a dash before the last three characters
// BTCUSD -> BTC-USD
// DOGEUSD -> DOGE-USD
// SOL1USD -> SOL1-USD
return aSymbol.replace('USD', '-USD');
}
}
return aSymbol;
};

@ -45,6 +45,7 @@ export interface IDataProviderResponse {
marketPrice: number;
marketState: MarketState;
name?: string;
sectors?: { name: string; weight: number }[];
url?: string;
}

Loading…
Cancel
Save