From 075431d868e964fbfe750bb3253d0c6238c6ed38 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sat, 8 Jan 2022 18:19:25 +0100 Subject: [PATCH] Feature/add google sheets as data source (#620) * Add google sheets as data source * Update changelog --- CHANGELOG.md | 8 + apps/api/src/app/symbol/symbol.controller.ts | 5 +- apps/api/src/app/symbol/symbol.service.ts | 26 --- .../api/src/services/configuration.service.ts | 4 + .../alpha-vantage/alpha-vantage.service.ts | 6 +- .../data-provider/data-provider.module.ts | 5 + .../data-provider/data-provider.service.ts | 6 +- .../ghostfolio-scraper-api.service.ts | 46 ++++- .../google-sheets/google-sheets.service.ts | 172 ++++++++++++++++++ .../interfaces/data-provider.interface.ts | 9 +- .../rakuten-rapid-api.service.ts | 10 +- .../yahoo-finance/yahoo-finance.service.ts | 10 +- .../interfaces/environment.interface.ts | 4 + package.json | 2 + .../migration.sql | 2 + prisma/schema.prisma | 1 + yarn.lock | 130 ++++++++++++- 17 files changed, 383 insertions(+), 63 deletions(-) create mode 100644 apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts create mode 100644 prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 395fdc199..7f3c5eda3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `GOOGLE_SHEETS` as a new data source type + ### Changed - Excluded the url pattern of shared portfolios in the `robots.txt` file +### Todo + +- Apply data migration (`yarn database:migrate`) + ## 1.100.0 - 05.01.2022 ### Added diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts index a5c31fe6a..bbe582cea 100644 --- a/apps/api/src/app/symbol/symbol.controller.ts +++ b/apps/api/src/app/symbol/symbol.controller.ts @@ -13,7 +13,7 @@ import { } from '@nestjs/common'; import { REQUEST } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; -import { DataSource, MarketData } from '@prisma/client'; +import { DataSource } from '@prisma/client'; import { StatusCodes, getReasonPhrase } from 'http-status-codes'; import { isDate, isEmpty } from 'lodash'; @@ -37,8 +37,7 @@ export class SymbolController { @Query() { query = '' } ): Promise<{ items: LookupItem[] }> { try { - const encodedQuery = encodeURIComponent(query.toLowerCase()); - return this.symbolService.lookup(encodedQuery); + return this.symbolService.lookup(query.toLowerCase()); } catch { throw new HttpException( getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR), diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts index b120d7ea2..8c95ce947 100644 --- a/apps/api/src/app/symbol/symbol.service.ts +++ b/apps/api/src/app/symbol/symbol.service.ts @@ -93,32 +93,6 @@ export class SymbolService { try { const { items } = await this.dataProviderService.search(aQuery); results.items = items; - - // Add custom symbols - const ghostfolioSymbolProfiles = - await this.prismaService.symbolProfile.findMany({ - select: { - currency: true, - dataSource: true, - name: true, - symbol: true - }, - where: { - AND: [ - { - dataSource: DataSource.GHOSTFOLIO, - name: { - startsWith: aQuery - } - } - ] - } - }); - - for (const ghostfolioSymbolProfile of ghostfolioSymbolProfiles) { - results.items.push(ghostfolioSymbolProfile); - } - return results; } catch (error) { Logger.error(error); diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts index b2d9c65fb..0c358f099 100644 --- a/apps/api/src/services/configuration.service.ts +++ b/apps/api/src/services/configuration.service.ts @@ -13,6 +13,7 @@ export class ConfigurationService { ACCESS_TOKEN_SALT: str(), ALPHA_VANTAGE_API_KEY: str({ default: '' }), CACHE_TTL: num({ default: 1 }), + DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }), DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }), ENABLE_FEATURE_BLOG: bool({ default: false }), ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }), @@ -25,6 +26,9 @@ export class ConfigurationService { ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }), GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }), GOOGLE_SECRET: str({ default: 'dummySecret' }), + GOOGLE_SHEETS_ACCOUNT: str({ default: '' }), + GOOGLE_SHEETS_ID: str({ default: '' }), + GOOGLE_SHEETS_PRIVATE_KEY: str({ default: '' }), JWT_SECRET_KEY: str({}), MAX_ITEM_IN_CACHE: num({ default: 9999 }), MAX_ORDERS_TO_IMPORT: num({ default: Number.MAX_SAFE_INTEGER }), diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts index 625bbc627..838f1ae6e 100644 --- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts +++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts @@ -88,13 +88,13 @@ export class AlphaVantageService implements DataProviderInterface { return DataSource.ALPHA_VANTAGE; } - public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { - const result = await this.alphaVantage.data.search(aSymbol); + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { + const result = await this.alphaVantage.data.search(aQuery); return { items: result?.bestMatches?.map((bestMatch) => { return { - dataSource: DataSource.ALPHA_VANTAGE, + dataSource: this.getName(), name: bestMatch['2. name'], symbol: bestMatch['1. symbol'] }; diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 37afb3abf..c05570932 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -1,6 +1,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; +import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service'; import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; @@ -21,12 +22,14 @@ import { DataProviderService } from './data-provider.service'; AlphaVantageService, DataProviderService, GhostfolioScraperApiService, + GoogleSheetsService, RakutenRapidApiService, YahooFinanceService, { inject: [ AlphaVantageService, GhostfolioScraperApiService, + GoogleSheetsService, RakutenRapidApiService, YahooFinanceService ], @@ -34,11 +37,13 @@ import { DataProviderService } from './data-provider.service'; useFactory: ( alphaVantageService, ghostfolioScraperApiService, + googleSheetsService, rakutenRapidApiService, yahooFinanceService ) => [ alphaVantageService, ghostfolioScraperApiService, + googleSheetsService, rakutenRapidApiService, yahooFinanceService ] diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 15a8a6efb..98237af8b 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -149,13 +149,13 @@ export class DataProviderService { return result; } - public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { const promises: Promise<{ items: LookupItem[] }>[] = []; let lookupItems: LookupItem[] = []; for (const dataSource of this.configurationService.get('DATA_SOURCES')) { promises.push( - this.getDataProvider(DataSource[dataSource]).search(aSymbol) + this.getDataProvider(DataSource[dataSource]).search(aQuery) ); } @@ -176,7 +176,7 @@ export class DataProviderService { } public getPrimaryDataSource(): DataSource { - return DataSource[this.configurationService.get('DATA_SOURCES')[0]]; + return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')]; } private getDataProvider(providerName: DataSource) { diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 77d406fdb..a34f7cf92 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -1,4 +1,10 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; +import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; +import { + IDataProviderHistoricalResponse, + IDataProviderResponse, + MarketState +} from '@ghostfolio/api/services/interfaces/interfaces'; import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service'; import { @@ -13,13 +19,6 @@ import * as bent from 'bent'; import * as cheerio from 'cheerio'; import { format } from 'date-fns'; -import { - IDataProviderHistoricalResponse, - IDataProviderResponse, - MarketState -} from '../../interfaces/interfaces'; -import { DataProviderInterface } from '../interfaces/data-provider.interface'; - @Injectable() export class GhostfolioScraperApiService implements DataProviderInterface { private static NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g; @@ -59,7 +58,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface { [symbol]: { marketPrice, currency: symbolProfile?.currency, - dataSource: DataSource.GHOSTFOLIO, + dataSource: this.getName(), marketState: MarketState.delayed } }; @@ -116,8 +115,35 @@ export class GhostfolioScraperApiService implements DataProviderInterface { return DataSource.GHOSTFOLIO; } - public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { - return { items: [] }; + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { + const items = await this.prismaService.symbolProfile.findMany({ + select: { + currency: true, + dataSource: true, + name: true, + symbol: true + }, + where: { + OR: [ + { + dataSource: this.getName(), + name: { + mode: 'insensitive', + startsWith: aQuery + } + }, + { + dataSource: this.getName(), + symbol: { + mode: 'insensitive', + startsWith: aQuery + } + } + ] + } + }); + + return { items }; } private extractNumberFromString(aString: string): number { diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts new file mode 100644 index 000000000..81b369279 --- /dev/null +++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts @@ -0,0 +1,172 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; +import { + IDataProviderHistoricalResponse, + IDataProviderResponse, + MarketState +} from '@ghostfolio/api/services/interfaces/interfaces'; +import { PrismaService } from '@ghostfolio/api/services/prisma.service'; +import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service'; +import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { Granularity } from '@ghostfolio/common/types'; +import { Injectable, Logger } from '@nestjs/common'; +import { DataSource } from '@prisma/client'; +import { format } from 'date-fns'; +import { GoogleSpreadsheet } from 'google-spreadsheet'; + +@Injectable() +export class GoogleSheetsService implements DataProviderInterface { + public constructor( + private readonly configurationService: ConfigurationService, + private readonly prismaService: PrismaService, + private readonly symbolProfileService: SymbolProfileService + ) {} + + public canHandle(symbol: string) { + return true; + } + + public async get( + aSymbols: string[] + ): Promise<{ [symbol: string]: IDataProviderResponse }> { + if (aSymbols.length <= 0) { + return {}; + } + + try { + const [symbol] = aSymbols; + const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles( + [symbol] + ); + + const sheet = await this.getSheet({ + sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'), + symbol + }); + const marketPrice = parseFloat( + (await sheet.getCellByA1('B1').value) as string + ); + + return { + [symbol]: { + marketPrice, + currency: symbolProfile?.currency, + dataSource: this.getName(), + marketState: MarketState.delayed + } + }; + } catch (error) { + Logger.error(error); + } + + return {}; + } + + public async getHistorical( + aSymbols: string[], + aGranularity: Granularity = 'day', + from: Date, + to: Date + ): Promise<{ + [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; + }> { + if (aSymbols.length <= 0) { + return {}; + } + + try { + const [symbol] = aSymbols; + + const sheet = await this.getSheet({ + symbol, + sheetId: this.configurationService.get('GOOGLE_SHEETS_ID') + }); + + const rows = await sheet.getRows(); + + const historicalData: { + [date: string]: IDataProviderHistoricalResponse; + } = {}; + + rows + .filter((row, index) => { + return index >= 1; + }) + .forEach((row) => { + const date = new Date(row._rawData[0]); + const close = parseFloat(row._rawData[1]); + + historicalData[format(date, DATE_FORMAT)] = { marketPrice: close }; + }); + + return { + [symbol]: historicalData + }; + } catch (error) { + Logger.error(error); + } + + return {}; + } + + public getName(): DataSource { + return DataSource.GOOGLE_SHEETS; + } + + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { + const items = await this.prismaService.symbolProfile.findMany({ + select: { + currency: true, + dataSource: true, + name: true, + symbol: true + }, + where: { + OR: [ + { + dataSource: this.getName(), + name: { + mode: 'insensitive', + startsWith: aQuery + } + }, + { + dataSource: this.getName(), + symbol: { + mode: 'insensitive', + startsWith: aQuery + } + } + ] + } + }); + + return { items }; + } + + private async getSheet({ + sheetId, + symbol + }: { + sheetId: string; + symbol: string; + }) { + const doc = new GoogleSpreadsheet(sheetId); + + await doc.useServiceAccountAuth({ + client_email: this.configurationService.get('GOOGLE_SHEETS_ACCOUNT'), + private_key: this.configurationService + .get('GOOGLE_SHEETS_PRIVATE_KEY') + .replace(/\\n/g, '\n') + }); + + await doc.loadInfo(); + + const sheet = doc.sheetsByTitle[symbol]; + + await sheet.loadCells(); + + return sheet; + } +} diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts index 5f99c8614..c5cf4c330 100644 --- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts +++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts @@ -1,11 +1,10 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; -import { Granularity } from '@ghostfolio/common/types'; -import { DataSource } from '@prisma/client'; - import { IDataProviderHistoricalResponse, IDataProviderResponse -} from '../../interfaces/interfaces'; +} from '@ghostfolio/api/services/interfaces/interfaces'; +import { Granularity } from '@ghostfolio/common/types'; +import { DataSource } from '@prisma/client'; export interface DataProviderInterface { canHandle(symbol: string): boolean; @@ -23,5 +22,5 @@ export interface DataProviderInterface { getName(): DataSource; - search(aSymbol: string): Promise<{ items: LookupItem[] }>; + search(aQuery: string): Promise<{ items: LookupItem[] }>; } diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index dea52e74d..bdfc147dd 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -45,7 +45,7 @@ export class RakutenRapidApiService implements DataProviderInterface { return { [ghostfolioFearAndGreedIndexSymbol]: { currency: undefined, - dataSource: DataSource.RAKUTEN, + dataSource: this.getName(), marketPrice: fgi.now.value, marketState: MarketState.open, name: RakutenRapidApiService.FEAR_AND_GREED_INDEX_NAME @@ -85,7 +85,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, - dataSource: DataSource.RAKUTEN, + dataSource: this.getName(), date: subWeeks(getToday(), 1), marketPrice: fgi.oneWeekAgo.value } @@ -94,7 +94,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, - dataSource: DataSource.RAKUTEN, + dataSource: this.getName(), date: subMonths(getToday(), 1), marketPrice: fgi.oneMonthAgo.value } @@ -103,7 +103,7 @@ export class RakutenRapidApiService implements DataProviderInterface { await this.prismaService.marketData.create({ data: { symbol, - dataSource: DataSource.RAKUTEN, + dataSource: this.getName(), date: subYears(getToday(), 1), marketPrice: fgi.oneYearAgo.value } @@ -129,7 +129,7 @@ export class RakutenRapidApiService implements DataProviderInterface { return DataSource.RAKUTEN; } - public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { return { items: [] }; } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index 0717a699e..2b4fe8f92 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -103,7 +103,7 @@ export class YahooFinanceService implements DataProviderInterface { assetClass, assetSubClass, currency: value.price?.currency, - dataSource: DataSource.YAHOO, + dataSource: this.getName(), exchange: this.parseExchange(value.price?.exchangeName), marketState: value.price?.marketState === 'REGULAR' || @@ -221,12 +221,14 @@ export class YahooFinanceService implements DataProviderInterface { return DataSource.YAHOO; } - public async search(aSymbol: string): Promise<{ items: LookupItem[] }> { + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { const items: LookupItem[] = []; try { const get = bent( - `${this.yahooFinanceHostname}/v1/finance/search?q=${aSymbol}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, + `${this.yahooFinanceHostname}/v1/finance/search?q=${encodeURIComponent( + aQuery + )}&lang=en-US®ion=US"esCount=8&newsCount=0&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`, 'GET', 'json', 200 @@ -268,7 +270,7 @@ export class YahooFinanceService implements DataProviderInterface { items.push({ symbol, currency: value.currency, - dataSource: DataSource.YAHOO, + dataSource: this.getName(), name: value.name }); } diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 56f8fe822..86b049546 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors { ACCESS_TOKEN_SALT: string; ALPHA_VANTAGE_API_KEY: string; CACHE_TTL: number; + DATA_SOURCE_PRIMARY: string; DATA_SOURCES: string | string[]; // string is not correct, error in envalid? ENABLE_FEATURE_BLOG: boolean; ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean; @@ -16,6 +17,9 @@ export interface Environment extends CleanedEnvAccessors { ENABLE_FEATURE_SYSTEM_MESSAGE: boolean; GOOGLE_CLIENT_ID: string; GOOGLE_SECRET: string; + GOOGLE_SHEETS_ACCOUNT: string; + GOOGLE_SHEETS_ID: string; + GOOGLE_SHEETS_PRIVATE_KEY: string; JWT_SECRET_KEY: string; MAX_ITEM_IN_CACHE: number; MAX_ORDERS_TO_IMPORT: number; diff --git a/package.json b/package.json index b3de40d0e..3d1a24d97 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "cryptocurrencies": "7.0.0", "date-fns": "2.22.1", "envalid": "7.2.1", + "google-spreadsheet": "3.2.0", "http-status-codes": "2.2.0", "ionicons": "5.5.1", "lodash": "4.17.21", @@ -143,6 +144,7 @@ "@types/big.js": "6.1.2", "@types/cache-manager": "3.4.2", "@types/color": "3.0.2", + "@types/google-spreadsheet": "3.1.5", "@types/jest": "27.0.2", "@types/lodash": "4.14.174", "@types/node": "14.14.33", diff --git a/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql b/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql new file mode 100644 index 000000000..f8923af93 --- /dev/null +++ b/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "DataSource" ADD VALUE 'GOOGLE_SHEETS'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b0d7a313c..0c89f8ec5 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -184,6 +184,7 @@ enum AssetSubClass { enum DataSource { ALPHA_VANTAGE GHOSTFOLIO + GOOGLE_SHEETS RAKUTEN YAHOO } diff --git a/yarn.lock b/yarn.lock index c4f08c700..bce12e97b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4487,6 +4487,11 @@ "@types/minimatch" "*" "@types/node" "*" +"@types/google-spreadsheet@3.1.5": + version "3.1.5" + resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-3.1.5.tgz#2bdc6f9f5372551e0506cb6ef3f562adcf44fc2e" + integrity sha512-7N+mDtZ1pmya2RRFPPl4KYc2TRgiqCNBLUZfyrKfER+u751JgCO+C24/LzF70UmUm/zhHUbzRZ5mtfaxekQ1ZQ== + "@types/graceful-fs@^4.1.2": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -5284,6 +5289,13 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" @@ -5774,7 +5786,7 @@ array.prototype.map@^1.0.3: es-array-method-boxes-properly "^1.0.0" is-string "^1.0.5" -arrify@^2.0.1: +arrify@^2.0.0, arrify@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== @@ -5903,6 +5915,13 @@ axios@0.24.0: dependencies: follow-redirects "^1.14.4" +axios@^0.21.4: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + axobject-query@2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.0.2.tgz#ea187abe5b9002b377f925d8bf7d1c561adf38f9" @@ -6124,7 +6143,7 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base64-js@^1.0.2, base64-js@^1.2.0, base64-js@^1.3.1: +base64-js@^1.0.2, base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -6190,6 +6209,11 @@ big.js@^5.2.2: resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== +bignumber.js@^9.0.0: + version "9.0.2" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.2.tgz#71c6c6bed38de64e24a65ebe16cfcf23ae693673" + integrity sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw== + bignumber.js@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" @@ -8233,7 +8257,7 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -ecdsa-sig-formatter@1.0.11: +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== @@ -9064,6 +9088,11 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter-asyncresource@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" @@ -9233,7 +9262,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@^3.0.0, extend@~3.0.2: +extend@^3.0.0, extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -9324,6 +9353,11 @@ fast-safe-stringify@2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-text-encoding@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" + integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== + fastparse@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/fastparse/-/fastparse-1.1.2.tgz#91728c5a5942eced8531283c79441ee4122c35a9" @@ -9587,6 +9621,11 @@ follow-redirects@^1.0.0: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.3.tgz#6ada78118d8d24caee595595accdc0ac6abd022e" integrity sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw== +follow-redirects@^1.14.0: + version "1.14.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.6.tgz#8cfb281bbc035b3c067d6cd975b0f6ade6e855cd" + integrity sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A== + follow-redirects@^1.14.4: version "1.14.5" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.5.tgz#f09a5848981d3c772b5392309778523f8d85c381" @@ -9831,6 +9870,25 @@ gauge@^4.0.0: strip-ansi "^6.0.1" wide-align "^1.1.2" +gaxios@^4.0.0: + version "4.3.2" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.2.tgz#845827c2dc25a0213c8ab4155c7a28910f5be83f" + integrity sha512-T+ap6GM6UZ0c4E6yb1y/hy2UB6hTrqhglp3XfmU9qbLCGRYhLVV5aRPpC4EmoG8N8zOnkYCgoBz+ScvGAARY6Q== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.6.1" + +gcp-metadata@^4.2.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9" + integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -10086,11 +10144,51 @@ globby@^9.0.0, globby@^9.2.0: pify "^4.0.1" slash "^2.0.0" +google-auth-library@^6.1.3: + version "6.1.6" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.6.tgz#deacdcdb883d9ed6bac78bb5d79a078877fdf572" + integrity sha512-Q+ZjUEvLQj/lrVHF/IQwRo6p3s8Nc44Zk/DALsN+ac3T4HY/g/3rrufkgtl+nZ1TW7DNAw5cTChdVp4apUXVgQ== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + +google-p12-pem@^3.0.3: + version "3.1.2" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.2.tgz#c3d61c2da8e10843ff830fdb0d2059046238c1d4" + integrity sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A== + dependencies: + node-forge "^0.10.0" + +google-spreadsheet@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.2.0.tgz#ce8aa75c15705aa950ad52b091a6fc4d33dcb329" + integrity sha512-z7XMaqb+26rdo8p51r5O03u8aPLAPzn5YhOXYJPcf2hdMVr0dUbIARgdkRdmGiBeoV/QoU/7VNhq1MMCLZv3kQ== + dependencies: + axios "^0.21.4" + google-auth-library "^6.1.3" + lodash "^4.17.21" + graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6: version "4.2.8" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== +gtoken@^5.0.4: + version "5.3.1" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.1.tgz#c1c2598a826f2b5df7c6bb53d7be6cf6d50c3c78" + integrity sha512-yqOREjzLHcbzz1UrQoxhBtpk8KjrVhuqPE7od1K2uhyxG2BHjKZetlbLw/SPZak/QqTIQW+addS+EcjqQsZbwQ== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.0.3" + jws "^4.0.0" + gzip-size@5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-5.1.1.tgz#cb9bee692f87c0612b232840a873904e4c135274" @@ -12175,6 +12273,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -12295,6 +12400,15 @@ jwa@^1.4.1: ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + jwk-to-pem@^2.0.4: version "2.0.5" resolved "https://registry.yarnpkg.com/jwk-to-pem/-/jwk-to-pem-2.0.5.tgz#151310bcfbcf731adc5ad9f379cbc8b395742906" @@ -12312,6 +12426,14 @@ jws@^3.2.2: jwa "^1.4.1" safe-buffer "^5.0.1" +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + karma-source-map-support@1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/karma-source-map-support/-/karma-source-map-support-1.4.0.tgz#58526ceccf7e8730e56effd97a4de8d712ac0d6b"