Fix missing CSV parser for dune-mock

.env.devnet

@@ -2,6 +2,6 @@ NETWORK=devnet
 API_URL=https://devnet-api.multiversx.com
 DATA_API_CEX_URL=https://data-api.multiversx.com/v1/quotes/cex
 DATA_API_XEXCHANGE_URL=https://data-api.multiversx.com/v1/quotes/xexchange
-DUNE_MOCK_URL=localhost:3001/dune-mock
+DUNE_MOCK_URL=http://localhost:3001/dune-mock
 DUNE_NAMESPACE=
 DUNE_API_KEY=

.env.mainnet

@@ -2,6 +2,6 @@ NETWORK=mainnet
 API_URL=https://api.multiversx.com
 DATA_API_CEX_URL=https://data-api.multiversx.com/v1/quotes/cex
 DATA_API_XEXCHANGE_URL=https://data-api.multiversx.com/v1/quotes/xexchange
-DUNE_MOCK_API_URL=localhost:3001/dune-mock
+DUNE_MOCK_API_URL=http://localhost:3001/dune-mock
 DUNE_NAMESPACE=nicolaemogage
 DUNE_API_KEY=

.env.testnet

@@ -2,6 +2,6 @@ NETWORK=testnet
 API_URL=https://testnet-api.multiversx.com
 DATA_API_CEX_URL=https://data-api.multiversx.com/v1/quotes/cex
 DATA_API_XEXCHANGE_URL=https://data-api.multiversx.com/v1/quotes/xexchange
-DUNE_MOCK_URL=localhost:3001/dune-mock
+DUNE_MOCK_URL=http://localhost:3001/dune-mock
 DUNE_NAMESPACE=
 DUNE_API_KEY=

apps/dune-mock/src/endpoints/dune-mock/csv.middleware.ts

@@ -0,0 +1,20 @@
+import { Injectable, NestMiddleware } from '@nestjs/common';
+import { Request, Response, NextFunction } from 'express';
+import csv from 'csv-parser';
+
+@Injectable()
+export class CsvParserMiddleware implements NestMiddleware {
+  use(req: Request, _: Response, next: NextFunction) {
+    if (req.headers['content-type'] === 'text/csv') {
+      const results: any = [];
+      req.pipe(csv())
+        .on('data', (data: any) => results.push(data))
+        .on('end', () => {
+          req.body = results;
+          next();
+        });
+    } else {
+      next();
+    }
+  }
+}

apps/dune-mock/src/endpoints/dune-mock/dune-mock.controller.ts

@@ -3,7 +3,7 @@ import { Body, Controller, Param, Post } from "@nestjs/common";
 import { ApiTags } from "@nestjs/swagger";
 import { CreateTableBody } from "./entities";
 
-@Controller('')
+@Controller('/dune-mock')
 @ApiTags('dune-mock')
 export class DuneMockController {
     constructor(
@@ -22,6 +22,8 @@ export class DuneMockController {
         @Param('table_name') tableName: string,
         @Body() body: Buffer,
     ): Promise<void> {
-        await this.duneMockService.insertIntoTable(tableName, body);
+        console.log('inserting into table', tableName);
+        console.log(body);
+        // await this.duneMockService.insertIntoTable(tableName, body);
     }
 }

apps/dune-mock/src/endpoints/dune-mock/dune-mock.module.ts

@@ -1,6 +1,7 @@
-import { Module } from "@nestjs/common";
+import { Module, MiddlewareConsumer, RequestMethod } from "@nestjs/common";
 import { ServicesModule } from "@libs/services/services.module";
 import { DuneMockController } from "./dune-mock.controller";
+import { CsvParserMiddleware } from "./csv.middleware";
 
 @Module({
     imports: [
@@ -10,4 +11,10 @@ import { DuneMockController } from "./dune-mock.controller";
         DuneMockController,
     ],
 })
-export class DuneMockModule { }
+export class DuneMockModule {
+    configure(consumer: MiddlewareConsumer) {
+        consumer
+        .apply(CsvParserMiddleware)
+        .forRoutes({ path: 'dune-mock/:table_name/insert', method: RequestMethod.POST });
+    }
+ }

apps/dune-mock/src/public.app.module.ts

@@ -4,15 +4,15 @@ import { DynamicModuleUtils } from '@libs/common';
 import { LoggingModule } from '@multiversx/sdk-nestjs-common';
 import { CommonConfigModule } from '@libs/common/config/common.config.module';
 import { DuneMockConfigModule } from './config/dune-mock-config.module';
-import { ScheduleModule } from '@nestjs/schedule';
+// import { ScheduleModule } from '@nestjs/schedule';
 
 @Module({
   imports: [
     LoggingModule,
     EndpointsModule,
     DuneMockConfigModule,
     CommonConfigModule,
-    ScheduleModule.forRoot(),
+    // ScheduleModule.forRoot(),
   ],
   providers: [
     DynamicModuleUtils.getNestJsApiConfigService(),

libs/services/src/dune-sender/dune-sender.service.ts

@@ -77,9 +77,8 @@ export class DuneSenderService {
 
     async insertCsvDataToLocalTable(tableName: string, data: Buffer): Promise<boolean> {
         await axios.post(`${this.appConfigService.getDuneMockApiUrl()}/${tableName}/insert`, data, {
-            headers: { 'Content-Type': ContentType.Csv }
+            headers: { 'Content-Type': ContentType.Csv },
         });
-        // console.log(response);
 
         return true;
     }

package.json

@@ -60,6 +60,7 @@
     "class-validator": "^0.14.1",
     "cookie-parser": "^1.4.6",
     "cron": "^3.1.6",
+    "csv-parser": "^3.0.0",
     "csv-writer": "^1.6.0",
     "ioredis": "^5.3.2",
     "jsonwebtoken": "^9.0.0",
@@ -132,4 +133,4 @@
       "^@libs/common": "<rootDir>/libs/common"
     }
   }
-}
+}