diff --git a/CHANGELOG.md b/CHANGELOG.md index a59223393..d1caeb03c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Hid unknown exchange in the position overview - Disable the base currency selector for the demo user - Refactored the portfolio unit tests to work without database +- Refactored the search functionality of the data management (aligned with data source) - Renamed shared helper to `@ghostfolio/common/helper` - Moved shared interfaces to `@ghostfolio/common/interfaces` - Moved shared types to `@ghostfolio/common/types` diff --git a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts index 689f69b91..e99cb893c 100644 --- a/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts +++ b/apps/api/src/app/symbol/interfaces/lookup-item.interface.ts @@ -1,4 +1,7 @@ +import { DataSource } from '@prisma/client'; + export interface LookupItem { + dataSource: DataSource; name: string; symbol: string; } diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts index 33a22286e..d388d0829 100644 --- a/apps/api/src/app/symbol/symbol.controller.ts +++ b/apps/api/src/app/symbol/symbol.controller.ts @@ -28,9 +28,12 @@ export class SymbolController { */ @Get('lookup') @UseGuards(AuthGuard('jwt')) - public async lookupSymbol(@Query() { query }): Promise { + public async lookupSymbol( + @Query() { query = '' } + ): Promise<{ items: LookupItem[] }> { try { - return this.symbolService.lookup(query); + const encodedQuery = encodeURIComponent(query.toLowerCase()); + return this.symbolService.lookup(encodedQuery); } catch { throw new HttpException( getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR), diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index 9da9488b9..5740ea69f 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -1,10 +1,8 @@ -import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service'; import { DataProviderService } from '@ghostfolio/api/services/data-provider.service'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { Injectable } from '@nestjs/common'; -import { Currency } from '@prisma/client'; -import * as bent from 'bent'; +import { Currency, DataSource } from '@prisma/client'; import { LookupItem } from './interfaces/lookup-item.interface'; import { SymbolItem } from './interfaces/symbol-item.interface'; @@ -27,62 +25,30 @@ export class SymbolService { }; } - public async lookup(aQuery = ''): Promise { - const query = aQuery.toLowerCase(); - const results: LookupItem[] = []; + public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> { + const results: { items: LookupItem[] } = { items: [] }; - if (!query) { + if (!aQuery) { return results; } - const get = bent( - `https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, - 'GET', - 'json', - 200 - ); - - // Add custom symbols - const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations(); - scraperConfigurations.forEach((scraperConfiguration) => { - if (scraperConfiguration.name.toLowerCase().startsWith(query)) { - results.push({ - name: scraperConfiguration.name, - symbol: scraperConfiguration.symbol - }); - } - }); - try { - const { quotes } = await get(); - - const searchResult = quotes - .filter(({ isYahooFinance }) => { - return isYahooFinance; - }) - .filter(({ quoteType }) => { - return ( - quoteType === 'CRYPTOCURRENCY' || - quoteType === 'EQUITY' || - quoteType === 'ETF' - ); - }) - .filter(({ quoteType, symbol }) => { - if (quoteType === 'CRYPTOCURRENCY') { - // Only allow cryptocurrencies in USD - return symbol.includes('USD'); - } + const { items } = await this.dataProviderService.search(aQuery); + results.items = items; + + // Add custom symbols + const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations(); + scraperConfigurations.forEach((scraperConfiguration) => { + if (scraperConfiguration.name.toLowerCase().startsWith(aQuery)) { + results.items.push({ + dataSource: DataSource.GHOSTFOLIO, + name: scraperConfiguration.name, + symbol: scraperConfiguration.symbol + }); + } + }); - return true; - }) - .map(({ longname, shortname, symbol }) => { - return { - name: longname || shortname, - symbol: convertFromYahooSymbol(symbol) - }; - }); - - return results.concat(searchResult); + return results; } catch (error) { console.error(error); diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts index 6a5af44d5..a52bf85cd 100644 --- a/apps/api/src/services/configuration.service.ts +++ b/apps/api/src/services/configuration.service.ts @@ -1,7 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { bool, cleanEnv, num, port, str } from 'envalid'; +import { bool, cleanEnv, json, num, port, str } from 'envalid'; import { Environment } from './interfaces/environment.interface'; +import { DataSource } from '.prisma/client'; @Injectable() export class ConfigurationService { @@ -12,6 +13,7 @@ export class ConfigurationService { ACCESS_TOKEN_SALT: str(), ALPHA_VANTAGE_API_KEY: str({ default: '' }), CACHE_TTL: num({ default: 1 }), + DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }), ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }), ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }), ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }), diff --git a/apps/api/src/services/data-provider.service.ts b/apps/api/src/services/data-provider.service.ts index e5d82a7a1..2183a514f 100644 --- a/apps/api/src/services/data-provider.service.ts +++ b/apps/api/src/services/data-provider.service.ts @@ -1,3 +1,4 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { isCrypto, isGhostfolioScraperApiSymbol, @@ -5,7 +6,7 @@ import { } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; -import { MarketData } from '@prisma/client'; +import { DataSource, MarketData } from '@prisma/client'; import { format } from 'date-fns'; import { ConfigurationService } from './configuration.service'; @@ -184,4 +185,19 @@ export class DataProviderService implements DataProviderInterface { return dataOfYahoo; } + + public async search(aSymbol: string) { + return this.getDataProvider().search(aSymbol); + } + + private getDataProvider() { + switch (this.configurationService.get('DATA_SOURCES')[0]) { + case DataSource.ALPHA_VANTAGE: + return this.alphaVantageService; + case DataSource.YAHOO: + return this.yahooFinanceService; + default: + throw new Error('No data provider has been found.'); + } + } } diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 696aee3c4..3046737e5 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -1,5 +1,7 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; import { isAfter, isBefore, parse } from 'date-fns'; import { ConfigurationService } from '../../configuration.service'; @@ -77,7 +79,17 @@ export class AlphaVantageService implements DataProviderInterface { } } - public search(aSymbol: string) { - return this.alphaVantage.data.search(aSymbol); + public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + const result = await this.alphaVantage.data.search(aSymbol); + + return { + items: result?.bestMatches?.map((bestMatch) => { + return { + dataSource: DataSource.ALPHA_VANTAGE, + name: bestMatch['2. name'], + symbol: bestMatch['1. symbol'] + }; + }) + }; } } diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 4568bf9a2..b6423f4a2 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -117,6 +117,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return []; } + public async search(aSymbol: string) { + return { items: [] }; + } + private extractNumberFromString(aString: string): number { try { const [numberString] = aString.match( diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index 1d31c0298..9f3943a8e 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -117,6 +117,14 @@ export class RakutenRapidApiService implements DataProviderInterface { return {}; } + public async search(aSymbol: string) { + return { items: [] }; + } + + public setPrisma(aPrismaService: PrismaService) { + this.prisma = aPrismaService; + } + private async getFearAndGreedIndex(): Promise<{ now: { value: number; valueText: string }; previousClose: { value: number; valueText: string }; @@ -147,8 +155,4 @@ export class RakutenRapidApiService implements DataProviderInterface { return undefined; } } - - public setPrisma(aPrismaService: PrismaService) { - this.prisma = aPrismaService; - } } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ deleted file mode 100644 index b18d9eaff..000000000 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___ +++ /dev/null @@ -1,24 +0,0 @@ -/* - import { Test } from '@nestjs/testing'; - - import { YahooFinanceService } from './yahoo-finance.service'; - - describe('AppService', () => { - let service: YahooFinanceService; - - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [], - providers: [YahooFinanceService] - }).compile(); - - service = app.get(YahooFinanceService); - }); - - describe('get', () => { - it('should return data for USDCHF', () => { - expect(service.get(['USDCHF'])).toEqual('{}'); - }); - }); - }); -*/ diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index d20446f29..19a969cbd 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -1,8 +1,10 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { UNKNOWN_KEY } from '@ghostfolio/common/config'; import { isCrypto, isCurrency, parseCurrency } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; +import * as bent from 'bent'; import { format } from 'date-fns'; import * as yahooFinance from 'yahoo-finance'; @@ -22,6 +24,8 @@ import { @Injectable() export class YahooFinanceService implements DataProviderInterface { + private yahooFinanceHostname = 'https://query1.finance.yahoo.com'; + public constructor() {} public async get( @@ -136,6 +140,49 @@ export class YahooFinanceService implements DataProviderInterface { } } + public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + let items = []; + + try { + const get = bent( + `${this.yahooFinanceHostname}/v1/finance/search?q=${aSymbol}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, + 'GET', + 'json', + 200 + ); + + const result = await get(); + items = result.quotes + .filter((quote) => { + return quote.isYahooFinance; + }) + .filter(({ quoteType }) => { + return ( + quoteType === 'CRYPTOCURRENCY' || + quoteType === 'EQUITY' || + quoteType === 'ETF' + ); + }) + .filter(({ quoteType, symbol }) => { + if (quoteType === 'CRYPTOCURRENCY') { + // Only allow cryptocurrencies in USD + return symbol.includes('USD'); + } + + return true; + }) + .map(({ longname, shortname, symbol }) => { + return { + dataSource: DataSource.YAHOO, + name: longname || shortname, + symbol: convertFromYahooSymbol(symbol) + }; + }); + } catch {} + + return { items }; + } + /** * Converts a symbol to a Yahoo symbol * diff --git a/apps/api/src/services/interfaces/data-provider.interface.ts b/apps/api/src/services/interfaces/data-provider.interface.ts index c3a431dce..cf861ad7b 100644 --- a/apps/api/src/services/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/interfaces/data-provider.interface.ts @@ -1,3 +1,4 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { Granularity } from '@ghostfolio/common/types'; import { @@ -16,4 +17,6 @@ export interface DataProviderInterface { ): Promise<{ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }>; + + search(aSymbol: string): Promise<{ items: LookupItem[] }>; } diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index ab73a0fe1..89ff79d33 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors { ACCESS_TOKEN_SALT: string; ALPHA_VANTAGE_API_KEY: string; CACHE_TTL: number; + DATA_SOURCES: string | string[]; // string is not correct, error in envalid? ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean; ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean; ENABLE_FEATURE_SOCIAL_LOGIN: boolean; diff --git a/apps/client/src/app/services/data.service.ts b/apps/client/src/app/services/data.service.ts index 81908d4b6..74fe3bc9c 100644 --- a/apps/client/src/app/services/data.service.ts +++ b/apps/client/src/app/services/data.service.ts @@ -102,7 +102,13 @@ export class DataService { } public fetchSymbols(aQuery: string) { - return this.http.get(`/api/symbol/lookup?query=${aQuery}`); + return this.http + .get<{ items: LookupItem[] }>(`/api/symbol/lookup?query=${aQuery}`) + .pipe( + map((respose) => { + return respose.items; + }) + ); } public fetchOrders(): Observable { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4f173a39a..eb7a084c9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -127,6 +127,7 @@ enum Currency { } enum DataSource { + ALPHA_VANTAGE GHOSTFOLIO RAKUTEN YAHOO