diff --git a/govtool/metadata-validation/src/app.module.ts b/govtool/metadata-validation/src/app.module.ts index 0c013915..0f6e9d7b 100644 --- a/govtool/metadata-validation/src/app.module.ts +++ b/govtool/metadata-validation/src/app.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; import { HttpModule } from '@nestjs/axios'; import { ConfigModule } from '@nestjs/config'; +import * as http from 'http'; +import * as https from 'https'; import { AppController } from './app.controller'; import { AppService } from './app.service'; @@ -12,8 +14,17 @@ import { HealthModule } from './health/health.module'; isGlobal: true, }), HttpModule.register({ - timeout: 5000, - maxRedirects: 5, + timeout: 10000, + maxContentLength: 10 * 1024 * 1024, // Max content length 10MB + maxBodyLength: 10 * 1024 * 1024, // Max body length 10MB + responseType: 'text', + headers: { + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + Expires: '0', + }, + httpAgent: new http.Agent({ keepAlive: true }), + httpsAgent: new https.Agent({ keepAlive: true }), }), HealthModule, ], diff --git a/govtool/metadata-validation/src/app.service.ts b/govtool/metadata-validation/src/app.service.ts index 7698a3c4..1ce809d1 100644 --- a/govtool/metadata-validation/src/app.service.ts +++ b/govtool/metadata-validation/src/app.service.ts @@ -1,8 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; -import { catchError, firstValueFrom, timeout } from 'rxjs'; +import { catchError, finalize, firstValueFrom } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import * as blake from 'blakejs'; -import { AxiosRequestConfig } from 'axios'; import * as jsonld from 'jsonld'; import { ValidateMetadataDTO } from '@dto'; @@ -10,13 +9,6 @@ import { LoggerMessage, MetadataValidationStatus } from '@enums'; import { validateMetadataStandard, parseMetadata, getStandard } from '@utils'; import { ValidateMetadataResult } from '@types'; -const axiosConfig: AxiosRequestConfig = { - timeout: 5000, - maxContentLength: 10 * 1024 * 1024, // Max content length 10MB - maxBodyLength: 10 * 1024 * 1024, // Max body length 10MB - responseType: 'text', -}; - @Injectable() export class AppService { constructor(private readonly httpService: HttpService) {} @@ -30,9 +22,10 @@ export class AppService { try { const { data: rawData } = await firstValueFrom( - this.httpService.get(url, axiosConfig).pipe( - timeout(5000), - catchError(() => { + this.httpService.get(url).pipe( + finalize(() => Logger.log(`Fetching ${url} completed`)), + catchError((error) => { + Logger.error(error, JSON.stringify(error)); throw MetadataValidationStatus.URL_NOT_FOUND; }), ),