From b5f01c0d151cd96da47c1cdef4a679c6b748dbb0 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Fri, 11 Aug 2023 20:20:35 +0200 Subject: [PATCH] Feature/migrate requests from bent to got (#2231) * Migrate requests from bent to got * Update changelog --- CHANGELOG.md | 4 ++ apps/api/src/app/info/info.service.ts | 50 +++++++------------ apps/api/src/app/logo/logo.service.ts | 12 ++--- .../coingecko/coingecko.service.ts | 30 ++++------- .../trackinsight/trackinsight.service.ts | 32 ++++++------ .../financial-modeling-prep.service.ts | 32 ++++-------- .../data-provider/manual/manual.service.ts | 7 ++- .../rapid-api/rapid-api.service.ts | 18 +++---- package.json | 1 - yarn.lock | 14 ------ 10 files changed, 74 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 671f7a85f..5aae4e412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added more durations in the coupon system +### Changed + +- Migrated the remaining requests from `bent` to `got` + ## 1.299.1 - 2023-08-10 ### Changed diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts index 187135a35..7eba432f5 100644 --- a/apps/api/src/app/info/info.service.ts +++ b/apps/api/src/app/info/info.service.ts @@ -30,9 +30,9 @@ import { permissions } from '@ghostfolio/common/permissions'; import { SubscriptionOffer } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; -import * as bent from 'bent'; import * as cheerio from 'cheerio'; import { format, subDays } from 'date-fns'; +import got from 'got'; @Injectable() export class InfoService { @@ -172,17 +172,13 @@ export class InfoService { private async countDockerHubPulls(): Promise { try { - const get = bent( + const { pull_count } = await got( `https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio`, - 'GET', - 'json', - 200, { - 'User-Agent': 'request' + headers: { 'User-Agent': 'request' } } - ); + ).json(); - const { pull_count } = await get(); return pull_count; } catch (error) { Logger.error(error, 'InfoService'); @@ -193,16 +189,9 @@ export class InfoService { private async countGitHubContributors(): Promise { try { - const get = bent( - 'https://github.com/ghostfolio/ghostfolio', - 'GET', - 'string', - 200, - {} - ); + const { body } = await got('https://github.com/ghostfolio/ghostfolio'); - const html = await get(); - const $ = cheerio.load(html); + const $ = cheerio.load(body); return extractNumberFromString( $( @@ -218,17 +207,13 @@ export class InfoService { private async countGitHubStargazers(): Promise { try { - const get = bent( + const { stargazers_count } = await got( `https://api.github.com/repos/ghostfolio/ghostfolio`, - 'GET', - 'json', - 200, { - 'User-Agent': 'request' + headers: { 'User-Agent': 'request' } } - ); + ).json(); - const { stargazers_count } = await get(); return stargazers_count; } catch (error) { Logger.error(error, 'InfoService'); @@ -346,22 +331,21 @@ export class InfoService { PROPERTY_BETTER_UPTIME_MONITOR_ID )) as string; - const get = bent( + const { data } = await got( `https://betteruptime.com/api/v2/monitors/${monitorId}/sla?from=${format( subDays(new Date(), 90), DATE_FORMAT )}&to${format(new Date(), DATE_FORMAT)}`, - 'GET', - 'json', - 200, + { - Authorization: `Bearer ${this.configurationService.get( - 'BETTER_UPTIME_API_KEY' - )}` + headers: { + Authorization: `Bearer ${this.configurationService.get( + 'BETTER_UPTIME_API_KEY' + )}` + } } - ); + ).json(); - const { data } = await get(); return data.attributes.availability / 100; } catch (error) { Logger.error(error, 'InfoService'); diff --git a/apps/api/src/app/logo/logo.service.ts b/apps/api/src/app/logo/logo.service.ts index d2e377fc0..166143a75 100644 --- a/apps/api/src/app/logo/logo.service.ts +++ b/apps/api/src/app/logo/logo.service.ts @@ -2,7 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/sy import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { HttpException, Injectable } from '@nestjs/common'; import { DataSource } from '@prisma/client'; -import * as bent from 'bent'; +import got from 'got'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; @Injectable() @@ -41,15 +41,11 @@ export class LogoService { } private getBuffer(aUrl: string) { - const get = bent( + return got( `https://t0.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=${aUrl}&size=64`, - 'GET', - 'buffer', - 200, { - 'User-Agent': 'request' + headers: { 'User-Agent': 'request' } } - ); - return get(); + ).buffer(); } } diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts index 35083e810..492664393 100644 --- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts +++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts @@ -15,8 +15,8 @@ import { DataSource, SymbolProfile } from '@prisma/client'; -import bent from 'bent'; import { format, fromUnixTime, getUnixTime } from 'date-fns'; +import got from 'got'; @Injectable() export class CoinGeckoService implements DataProviderInterface { @@ -45,8 +45,7 @@ export class CoinGeckoService implements DataProviderInterface { }; try { - const get = bent(`${this.URL}/coins/${aSymbol}`, 'GET', 'json', 200); - const { name } = await get(); + const { name } = await got(`${this.URL}/coins/${aSymbol}`).json(); response.name = name; } catch (error) { @@ -79,17 +78,13 @@ export class CoinGeckoService implements DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { try { - const get = bent( + const { prices } = await got( `${ this.URL }/coins/${aSymbol}/market_chart/range?vs_currency=${this.baseCurrency.toLowerCase()}&from=${getUnixTime( from - )}&to=${getUnixTime(to)}`, - 'GET', - 'json', - 200 - ); - const { prices } = await get(); + )}&to=${getUnixTime(to)}` + ).json(); const result: { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; @@ -132,15 +127,11 @@ export class CoinGeckoService implements DataProviderInterface { } try { - const get = bent( + const response = await got( `${this.URL}/simple/price?ids=${aSymbols.join( ',' - )}&vs_currencies=${this.baseCurrency.toLowerCase()}`, - 'GET', - 'json', - 200 - ); - const response = await get(); + )}&vs_currencies=${this.baseCurrency.toLowerCase()}` + ).json(); for (const symbol in response) { if (Object.prototype.hasOwnProperty.call(response, symbol)) { @@ -174,8 +165,9 @@ export class CoinGeckoService implements DataProviderInterface { let items: LookupItem[] = []; try { - const get = bent(`${this.URL}/search?query=${query}`, 'GET', 'json', 200); - const { coins } = await get(); + const { coins } = await got( + `${this.URL}/search?query=${query}` + ).json(); items = coins.map(({ id: symbol, name }) => { return { diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts index ec68dd2eb..a18d82449 100644 --- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts @@ -3,9 +3,7 @@ import { Country } from '@ghostfolio/common/interfaces/country.interface'; import { Sector } from '@ghostfolio/common/interfaces/sector.interface'; import { Injectable } from '@nestjs/common'; import { SymbolProfile } from '@prisma/client'; -import bent from 'bent'; - -const getJSON = bent('json'); +import got from 'got'; @Injectable() export class TrackinsightDataEnhancerService implements DataEnhancerInterface { @@ -34,11 +32,13 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface { return response; } - const profile = await getJSON( + const profile = await got( `${TrackinsightDataEnhancerService.baseUrl}/data-api/funds/${symbol}.json` - ).catch(() => { - return {}; - }); + ) + .json() + .catch(() => { + return {}; + }); const isin = profile.isin?.split(';')?.[0]; @@ -46,15 +46,17 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface { response.isin = isin; } - const holdings = await getJSON( + const holdings = await got( `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json` - ).catch(() => { - return getJSON( - `${TrackinsightDataEnhancerService.baseUrl}/holdings/${ - symbol.split('.')?.[0] - }.json` - ); - }); + ) + .json() + .catch(() => { + return got( + `${TrackinsightDataEnhancerService.baseUrl}/holdings/${ + symbol.split('.')?.[0] + }.json` + ); + }); if (holdings?.weight < 0.95) { // Skip if data is inaccurate diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts index ed65c3f65..3d7ba65f4 100644 --- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts +++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts @@ -10,8 +10,8 @@ import { DataProviderInfo } from '@ghostfolio/common/interfaces'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import bent from 'bent'; import { format, isAfter, isBefore, isSameDay } from 'date-fns'; +import got from 'got'; @Injectable() export class FinancialModelingPrepService implements DataProviderInterface { @@ -64,13 +64,9 @@ export class FinancialModelingPrepService implements DataProviderInterface { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; }> { try { - const get = bent( - `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`, - 'GET', - 'json', - 200 - ); - const { historical } = await get(); + const { historical } = await got( + `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}` + ).json(); const result: { [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; @@ -115,13 +111,9 @@ export class FinancialModelingPrepService implements DataProviderInterface { } try { - const get = bent( - `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`, - 'GET', - 'json', - 200 - ); - const response = await get(); + const response = await got( + `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}` + ).json(); for (const { price, symbol } of response) { results[symbol] = { @@ -153,13 +145,9 @@ export class FinancialModelingPrepService implements DataProviderInterface { let items: LookupItem[] = []; try { - const get = bent( - `${this.URL}/search?query=${query}&apikey=${this.apiKey}`, - 'GET', - 'json', - 200 - ); - const result = await get(); + const result = await got( + `${this.URL}/search?query=${query}&apikey=${this.apiKey}` + ).json(); items = result.map(({ currency, name, symbol }) => { return { diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 7b3933532..adf14e43f 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -14,10 +14,10 @@ import { import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import bent from 'bent'; import * as cheerio from 'cheerio'; import { isUUID } from 'class-validator'; import { addDays, format, isBefore } from 'date-fns'; +import got from 'got'; @Injectable() export class ManualService implements DataProviderInterface { @@ -95,10 +95,9 @@ export class ManualService implements DataProviderInterface { return {}; } - const get = bent(url, 'GET', 'string', 200, headers); + const { body } = await got(url, { headers }); - const html = await get(); - const $ = cheerio.load(html); + const $ = cheerio.load(body); const value = extractNumberFromString($(selector).text()); diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts index 053391a8a..307855aaf 100644 --- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts @@ -10,8 +10,8 @@ import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import bent from 'bent'; import { format } from 'date-fns'; +import got from 'got'; @Injectable() export class RapidApiService implements DataProviderInterface { @@ -135,19 +135,17 @@ export class RapidApiService implements DataProviderInterface { oneYearAgo: { value: number; valueText: string }; }> { try { - const get = bent( + const { fgi } = await got( `https://fear-and-greed-index.p.rapidapi.com/v1/fgi`, - 'GET', - 'json', - 200, { - useQueryString: true, - 'x-rapidapi-host': 'fear-and-greed-index.p.rapidapi.com', - 'x-rapidapi-key': this.configurationService.get('RAPID_API_API_KEY') + headers: { + useQueryString: 'true', + 'x-rapidapi-host': 'fear-and-greed-index.p.rapidapi.com', + 'x-rapidapi-key': this.configurationService.get('RAPID_API_API_KEY') + } } - ); + ).json(); - const { fgi } = await get(); return fgi; } catch (error) { Logger.error(error, 'RapidApiService'); diff --git a/package.json b/package.json index 20f28fa44..9057f4328 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,6 @@ "@simplewebauthn/server": "5.2.1", "@stripe/stripe-js": "1.47.0", "alphavantage": "2.2.0", - "bent": "7.3.12", "big.js": "6.2.1", "body-parser": "1.20.1", "bootstrap": "4.6.0", diff --git a/yarn.lock b/yarn.lock index 44a88e311..fddf81de8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7339,15 +7339,6 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" -bent@7.3.12: - version "7.3.12" - resolved "https://registry.yarnpkg.com/bent/-/bent-7.3.12.tgz#e0a2775d4425e7674c64b78b242af4f49da6b035" - integrity sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w== - dependencies: - bytesish "^0.4.1" - caseless "~0.12.0" - is-stream "^2.0.0" - better-opn@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-2.1.1.tgz#94a55b4695dc79288f31d7d0e5f658320759f7c6" @@ -7646,11 +7637,6 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== -bytesish@^0.4.1: - version "0.4.4" - resolved "https://registry.yarnpkg.com/bytesish/-/bytesish-0.4.4.tgz#f3b535a0f1153747427aee27256748cff92347e6" - integrity sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ== - cacache@17.1.3, cacache@^17.0.0: version "17.1.3" resolved "https://registry.yarnpkg.com/cacache/-/cacache-17.1.3.tgz#c6ac23bec56516a7c0c52020fd48b4909d7c7044"