Feature/improve import (#657)

* Improve import

* Update changelog

Co-Authored-By: Ronald Konjer <ronaldkonjer@gmail.com>

CHANGELOG.md

@@ -7,11 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
-- Added support for cryptocurrency _Mina Protocol_ (`MINA-USD`)
+### Added
+
+- Added support for the (optional) `accountId` in the import functionality for activities
+- Added support for the (optional) `dataSource` in the import functionality for activities
+- Added support for the cryptocurrency _Mina Protocol_ (`MINA-USD`)
 
 ### Changed
 
 - Improved the consistent use of `symbol` in combination with `dataSource`
+- Removed the primary data source from the client
 
 ## 1.108.0 - 27.01.2022
 
@@ -208,7 +213,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added support for cryptocurrency _Solana_ (`SOL-USD`)
+- Added support for the cryptocurrency _Solana_ (`SOL-USD`)
 - Extended the documentation for self-hosting with the [official Ghostfolio Docker image](https://hub.docker.com/r/ghostfolio/ghostfolio)
 
 ### Fixed

apps/api/src/app/import/import.service.ts

@@ -20,6 +20,11 @@ export class ImportService {
     orders: Partial<Order>[];
     userId: string;
   }): Promise<void> {
+    for (const order of orders) {
+      order.dataSource =
+        order.dataSource ?? this.dataProviderService.getPrimaryDataSource();
+    }
+
     await this.validateOrders({ orders, userId });
 
     for (const {
@@ -34,6 +39,7 @@ export class ImportService {
       unitPrice
     } of orders) {
       await this.orderService.createOrder({
+        accountId,
         currency,
         dataSource,
         fee,
@@ -41,11 +47,7 @@ export class ImportService {
         symbol,
         type,
         unitPrice,
-        Account: {
-          connect: {
-            id_userId: { userId, id: accountId }
-          }
-        },
+        userId,
         date: parseISO(<string>(<unknown>date)),
         SymbolProfile: {
           connectOrCreate: {

apps/api/src/app/info/info.service.ts

@@ -1,7 +1,6 @@
 import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
-import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
@@ -27,7 +26,6 @@ export class InfoService {
 
   public constructor(
     private readonly configurationService: ConfigurationService,
-    private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly dataGatheringService: DataGatheringService,
     private readonly jwtService: JwtService,
@@ -92,7 +90,6 @@ export class InfoService {
       currencies: this.exchangeRateDataService.getCurrencies(),
       demoAuthToken: this.getDemoAuthToken(),
       lastDataGathering: await this.getLastDataGathering(),
-      primaryDataSource: this.dataProviderService.getPrimaryDataSource(),
       statistics: await this.getStatistics(),
       subscriptions: await this.getSubscriptions()
     };

apps/api/src/app/order/create-order.dto.ts

@@ -1,14 +1,22 @@
 import { DataSource, Type } from '@prisma/client';
-import { IsEnum, IsISO8601, IsNumber, IsString } from 'class-validator';
+import {
+  IsEnum,
+  IsISO8601,
+  IsNumber,
+  IsOptional,
+  IsString
+} from 'class-validator';
 
 export class CreateOrderDto {
   @IsString()
+  @IsOptional()
   accountId: string;
 
   @IsString()
   currency: string;
 
   @IsEnum(DataSource, { each: true })
+  @IsOptional()
   dataSource: DataSource;
 
   @IsISO8601()

apps/api/src/app/order/order.controller.ts

@@ -114,19 +114,9 @@ export class OrderController {
       );
     }
 
-    const date = parseISO(data.date);
-
-    const accountId = data.accountId;
-    delete data.accountId;
-
     return this.orderService.createOrder({
       ...data,
-      date,
-      Account: {
-        connect: {
-          id_userId: { id: accountId, userId: this.request.user.id }
-        }
-      },
+      date: parseISO(data.date),
       SymbolProfile: {
         connectOrCreate: {
           create: {
@@ -141,7 +131,8 @@ export class OrderController {
           }
         }
       },
-      User: { connect: { id: this.request.user.id } }
+      User: { connect: { id: this.request.user.id } },
+      userId: this.request.user.id
     });
   }
 

apps/api/src/app/order/order.module.ts

@@ -1,3 +1,4 @@
+import { AccountService } from '@ghostfolio/api/app/account/account.service';
 import { CacheService } from '@ghostfolio/api/app/cache/cache.service';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { UserModule } from '@ghostfolio/api/app/user/user.module';
@@ -24,7 +25,7 @@ import { OrderService } from './order.service';
     UserModule
   ],
   controllers: [OrderController],
-  providers: [CacheService, OrderService],
+  providers: [AccountService, CacheService, OrderService],
   exports: [OrderService]
 })
 export class OrderModule {}

apps/api/src/app/order/order.service.ts

@@ -1,3 +1,4 @@
+import { AccountService } from '@ghostfolio/api/app/account/account.service';
 import { CacheService } from '@ghostfolio/api/app/cache/cache.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
@@ -13,6 +14,7 @@ import { Activity } from './interfaces/activities.interface';
 @Injectable()
 export class OrderService {
   public constructor(
+    private readonly accountService: AccountService,
     private readonly cacheService: CacheService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly dataGatheringService: DataGatheringService,
@@ -47,7 +49,24 @@ export class OrderService {
     });
   }
 
-  public async createOrder(data: Prisma.OrderCreateInput): Promise<Order> {
+  public async createOrder(
+    data: Prisma.OrderCreateInput & { accountId?: string; userId: string }
+  ): Promise<Order> {
+    const defaultAccount = (
+      await this.accountService.getAccounts(data.userId)
+    ).find((account) => {
+      return account.isDefault === true;
+    });
+
+    const Account = {
+      connect: {
+        id_userId: {
+          userId: data.userId,
+          id: data.accountId ?? defaultAccount?.id
+        }
+      }
+    };
+
     const isDraft = isAfter(data.date as Date, endOfToday());
 
     // Convert the symbol to uppercase to avoid case-sensitive duplicates
@@ -70,9 +89,15 @@ export class OrderService {
 
     await this.cacheService.flush();
 
+    delete data.accountId;
+    delete data.userId;
+
+    const orderData: Prisma.OrderCreateInput = data;
+
     return this.prismaService.order.create({
       data: {
-        ...data,
+        ...orderData,
+        Account,
         isDraft,
         symbol
       }

apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts

@@ -39,7 +39,6 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
   public routeQueryParams: Subscription;
   public user: User;
 
-  private primaryDataSource: DataSource;
   private unsubscribeSubject = new Subject<void>();
 
   /**
@@ -57,9 +56,6 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
     private snackBar: MatSnackBar,
     private userService: UserService
   ) {
-    const { primaryDataSource } = this.dataService.fetchInfo();
-    this.primaryDataSource = primaryDataSource;
-
     this.routeQueryParams = route.queryParams
       .pipe(takeUntil(this.unsubscribeSubject))
       .subscribe((params) => {
@@ -209,8 +205,7 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
             try {
               await this.importTransactionsService.importCsv({
                 fileContent,
-                primaryDataSource: this.primaryDataSource,
-                user: this.user
+                userAccounts: this.user.accounts
               });
 
               this.handleImportSuccess();

apps/client/src/app/services/import-transactions.service.ts

@@ -1,20 +1,20 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
-import { DataSource, Type } from '@prisma/client';
+import { Account, DataSource, Type } from '@prisma/client';
 import { parse } from 'date-fns';
-import { parse as csvToJson } from 'papaparse';
 import { isNumber } from 'lodash';
+import { parse as csvToJson } from 'papaparse';
 import { EMPTY } from 'rxjs';
 import { catchError } from 'rxjs/operators';
-import { User } from '@ghostfolio/common/interfaces';
 
 @Injectable({
   providedIn: 'root'
 })
 export class ImportTransactionsService {
-  private static ACCOUNT_ID = ['account', 'accountid'];
+  private static ACCOUNT_KEYS = ['account', 'accountid'];
   private static CURRENCY_KEYS = ['ccy', 'currency'];
+  private static DATA_SOURCE_KEYS = ['datasource'];
   private static DATE_KEYS = ['date'];
   private static FEE_KEYS = ['commission', 'fee'];
   private static QUANTITY_KEYS = ['qty', 'quantity', 'shares', 'units'];
@@ -26,30 +26,23 @@ export class ImportTransactionsService {
 
   public async importCsv({
     fileContent,
-    primaryDataSource,
-    user
+    userAccounts
   }: {
     fileContent: string;
-    primaryDataSource: DataSource;
-    user: User;
+    userAccounts: Account[];
   }) {
-    let content: any[] = [];
-
-    csvToJson(fileContent, {
+    const content = csvToJson(fileContent, {
       dynamicTyping: true,
       header: true,
-      skipEmptyLines: true,
-      complete: (parsedData) => {
-        content = parsedData.data.filter((item) => item['date'] != null);
-      }
-    });
+      skipEmptyLines: true
+    }).data;
 
     const orders: CreateOrderDto[] = [];
     for (const [index, item] of content.entries()) {
       orders.push({
-        accountId: this.parseAccount({ content, index, item, user }),
+        accountId: this.parseAccount({ item, userAccounts }),
         currency: this.parseCurrency({ content, index, item }),
-        dataSource: primaryDataSource,
+        dataSource: this.parseDataSource({ item }),
         date: this.parseDate({ content, index, item }),
         fee: this.parseFee({ content, index, item }),
         quantity: this.parseQuantity({ content, index, item }),
@@ -89,35 +82,26 @@ export class ImportTransactionsService {
   }
 
   private parseAccount({
-    content,
-    index,
     item,
-    user
+    userAccounts
   }: {
-    content: any[];
-    index: number;
     item: any;
-    user: User;
+    userAccounts: Account[];
   }) {
     item = this.lowercaseKeys(item);
-    for (const key of ImportTransactionsService.ACCOUNT_ID) {
+
+    for (const key of ImportTransactionsService.ACCOUNT_KEYS) {
       if (item[key]) {
-        let accountid = user.accounts.find((account) => {
+        return userAccounts.find((account) => {
           return (
-            account.name.toLowerCase() === item[key].toLowerCase() ||
-            account.id == item[key]
+            account.id === item[key] ||
+            account.name.toLowerCase() === item[key].toLowerCase()
           );
         })?.id;
-        if (!accountid) {
-          accountid = user?.accounts.find((account) => {
-            return account.isDefault;
-          })?.id;
-        }
-        return accountid;
       }
     }
 
-    throw { message: `orders.${index}.account is not valid`, orders: content };
+    return undefined;
   }
 
   private parseCurrency({
@@ -140,6 +124,18 @@ export class ImportTransactionsService {
     throw { message: `orders.${index}.currency is not valid`, orders: content };
   }
 
+  private parseDataSource({ item }: { item: any }) {
+    item = this.lowercaseKeys(item);
+
+    for (const key of ImportTransactionsService.DATA_SOURCE_KEYS) {
+      if (item[key]) {
+        return DataSource[item[key].toUpperCase()];
+      }
+    }
+
+    return undefined;
+  }
+
   private parseDate({
     content,
     index,

libs/common/src/lib/interfaces/info-item.interface.ts

@@ -1,5 +1,3 @@
-import { DataSource } from '@prisma/client';
-
 import { Statistics } from './statistics.interface';
 import { Subscription } from './subscription.interface';
 
@@ -10,7 +8,6 @@ export interface InfoItem {
   isReadOnlyMode?: boolean;
   lastDataGathering?: Date;
   platforms: { id: string; name: string }[];
-  primaryDataSource: DataSource;
   statistics: Statistics;
   stripePublicKey?: string;
   subscriptions: Subscription[];