From ca97cabb3a5c28f4a43e53d381f3aabc86522b43 Mon Sep 17 00:00:00 2001 From: OliveGerste Date: Mon, 6 Mar 2023 16:08:26 +0100 Subject: [PATCH 1/3] Only receive required car data in smart charging --- .../SapSmartChargingIntegration.ts | 2 +- src/storage/mongodb/TransactionStorage.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts b/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts index 9616164d84..7cea8d337c 100644 --- a/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts +++ b/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts @@ -119,7 +119,7 @@ export default class SapSmartChargingIntegration extends SmartChargingIntegratio this.tenant, { chargingStationIDs: chargingStationIDs, profilePurposeType: ChargingProfilePurposeType.TX_PROFILE }, Constants.DB_PARAMS_MAX_LIMIT); const currentChargingProfiles = currentChargingProfilesResponse.result; // Get all transactions of the site areas - const transactions = await TransactionStorage.getTransactions(this.tenant, { transactionIDs, withCar: true }, Constants.DB_PARAMS_MAX_LIMIT); + const transactions = await TransactionStorage.getTransactions(this.tenant, { transactionIDs, withSmartChargingData: true }, Constants.DB_PARAMS_MAX_LIMIT); // Build request const request = await this.buildOptimizerRequest(rootSiteArea, excludedChargingStations, false, currentChargingProfiles, transactions.result); // Call optimizer diff --git a/src/storage/mongodb/TransactionStorage.ts b/src/storage/mongodb/TransactionStorage.ts index 22d222f2e0..b8ff2581fa 100644 --- a/src/storage/mongodb/TransactionStorage.ts +++ b/src/storage/mongodb/TransactionStorage.ts @@ -328,7 +328,7 @@ export default class TransactionStorage { ocpiSessionID?: string; ocpiAuthorizationID?: string; ocpiSessionDateFrom?: Date; ocpiSessionDateTo?: Date; ocpiCdrDateFrom?: Date; ocpiCdrDateTo?: Date; ocpiSessionChecked?: boolean; ocpiCdrChecked?: boolean; oicpSessionID?: string; withSite?: boolean; withSiteArea?: boolean; withCompany?: boolean; statistics?: TransactionStatisticsType; refundStatus?: RefundStatus[]; withTag?: boolean; hasUserID?: boolean; withUser?: boolean; withCar?: boolean; - transactionsToStop?: boolean; siteOwnerIDs?: string[]; + transactionsToStop?: boolean; siteOwnerIDs?: string[]; withSmartChargingData?: boolean }, dbParams: DbParams, projectFields?: string[]): Promise { const startTime = Logging.traceDatabaseRequestStart(); @@ -753,6 +753,17 @@ export default class TransactionStorage { foreignField: '_id', oneToOneCardinality: true }); } + // Smart Charging Data + if (params.withSmartChargingData) { + DatabaseUtils.pushCarLookupInAggregation({ + tenantID: tenant.id, aggregation: aggregation, asField: 'car', localField: 'carID', + foreignField: '_id', oneToOneCardinality: true, oneToOneCardinalityNotNull: false, projectFields:['converter.amperagePerPhase'] + }); + DatabaseUtils.pushCarCatalogLookupInAggregation({ + tenantID: Constants.DEFAULT_TENANT_ID, aggregation: aggregation, asField: 'carCatalog', localField: 'carCatalogID', + foreignField: '_id', oneToOneCardinality: true, projectFields:['fastChargePowerMax', 'batteryCapacityFull'] + }); + } // Rename ID DatabaseUtils.pushRenameDatabaseIDToNumber(aggregation); // Convert Object ID to string From 25c235921cf99a5448a336f6fae62e9b43a3dc17 Mon Sep 17 00:00:00 2001 From: ClaudeROSSI Date: Tue, 7 Mar 2023 09:49:22 +0100 Subject: [PATCH 2/3] CheckChargingStationTemplateTask - pagination + shrink logged data --- .../tasks/CheckChargingStationTemplateTask.ts | 68 ++++++++++--------- src/server/ocpp/utils/OCPPUtils.ts | 24 ++++--- src/storage/mongodb/ChargingStationStorage.ts | 8 ++- src/utils/Constants.ts | 4 +- src/utils/LoggingHelper.ts | 21 +++++- 5 files changed, 80 insertions(+), 45 deletions(-) diff --git a/src/scheduler/tasks/CheckChargingStationTemplateTask.ts b/src/scheduler/tasks/CheckChargingStationTemplateTask.ts index 61d4650bc7..a618d0877e 100644 --- a/src/scheduler/tasks/CheckChargingStationTemplateTask.ts +++ b/src/scheduler/tasks/CheckChargingStationTemplateTask.ts @@ -32,39 +32,45 @@ export default class CheckChargingStationTemplateTask extends TenantSchedulerTas private async applyTemplateToChargingStations(tenant: Tenant) { let updated = 0; - // Bypass perf tenant - if (tenant.subdomain === 'testperf') { - await Logging.logWarning({ - tenantID: tenant.id, - action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, - module: MODULE_NAME, method: 'applyTemplateToChargingStations', - message: `Bypassed tenant ${Utils.buildTenantName(tenant)})` + // Prepare pagination + const limit = Constants.BATCH_PAGE_SIZE; // Avoid loading too much data in one shot + const sort = null; // No sort + let skip = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + // Get the charging stations + const chargingStations = await ChargingStationStorage.getChargingStations(tenant, { + issuer: true, + manualConfiguration: false, // template cannot be applied when a manual configuration is being used + // withSiteArea: true, // Site area data is loaded only when necessary + }, { + limit, skip, sort }); - return; - } - // Get the charging stations - const chargingStations = await ChargingStationStorage.getChargingStations(tenant, { - issuer: true, withSiteArea: true - }, Constants.DB_PARAMS_MAX_LIMIT); - // Update - for (const chargingStation of chargingStations.result) { - try { - // Apply template - const chargingStationTemplateUpdateResult = await OCPPUtils.checkAndApplyTemplateToChargingStation(tenant, chargingStation); - // Save - if (chargingStationTemplateUpdateResult.chargingStationUpdated) { - await ChargingStationStorage.saveChargingStation(tenant, chargingStation); - updated++; + if (Utils.isEmptyArray(chargingStations.result)) { + break; + } + // Increment skip for next round + skip += limit; + // Update + for (const chargingStation of chargingStations.result) { + try { + // Apply template + const chargingStationTemplateUpdateResult = await OCPPUtils.checkAndApplyTemplateToChargingStation(tenant, chargingStation); + // Save + if (chargingStationTemplateUpdateResult.chargingStationUpdated) { + await ChargingStationStorage.saveChargingStation(tenant, chargingStation); + updated++; + } + } catch (error) { + await Logging.logError({ + ...LoggingHelper.getChargingStationProperties(chargingStation), + tenantID: tenant.id, + action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, + module: MODULE_NAME, method: 'applyTemplateToChargingStations', + message: `Template update error in Tenant ${Utils.buildTenantName(tenant)}): ${error.message as string}`, + detailedMessages: { error: error.stack } + }); } - } catch (error) { - await Logging.logError({ - ...LoggingHelper.getChargingStationProperties(chargingStation), - tenantID: tenant.id, - action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, - module: MODULE_NAME, method: 'applyTemplateToChargingStations', - message: `Template update error in Tenant ${Utils.buildTenantName(tenant)}): ${error.message as string}`, - detailedMessages: { error: error.stack } - }); } } if (updated > 0) { diff --git a/src/server/ocpp/utils/OCPPUtils.ts b/src/server/ocpp/utils/OCPPUtils.ts index 61cb38573a..a73ca740fe 100644 --- a/src/server/ocpp/utils/OCPPUtils.ts +++ b/src/server/ocpp/utils/OCPPUtils.ts @@ -1530,13 +1530,12 @@ export default class OCPPUtils { if (chargingStationTemplate) { // Already updated? if (chargingStation.templateHash !== chargingStationTemplate.hash) { - await Logging.logInfo({ + await Logging.logDebug({ ...LoggingHelper.getChargingStationProperties(chargingStation), tenantID: tenant.id, action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, module: MODULE_NAME, method: 'enrichChargingStationWithTemplate', - message: `Template ID '${chargingStationTemplate.id}' is been applied...`, - detailedMessages: { chargingStationTemplate, chargingStation } + message: `Template ID '${chargingStationTemplate.id}' is being applied...` }); // Check Technical templateUpdateResult.technicalUpdated = @@ -1559,16 +1558,21 @@ export default class OCPPUtils { action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, module: MODULE_NAME, method: 'enrichChargingStationWithTemplate', message: `Template ID '${chargingStationTemplate.id}' has been applied with success`, - detailedMessages: { templateUpdateResult, chargingStationTemplate, chargingStation } + detailedMessages: { + templateUpdateResult, + templateData: LoggingHelper.shrinkTemplateProperties(chargingStationTemplate), + } }); } else { - await Logging.logInfo({ + await Logging.logDebug({ ...LoggingHelper.getChargingStationProperties(chargingStation), tenantID: tenant.id, action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, module: MODULE_NAME, method: 'enrichChargingStationWithTemplate', message: `Template ID '${chargingStationTemplate.id}' has already been applied`, - detailedMessages: { chargingStationTemplate, chargingStation } + detailedMessages: { + templateData: LoggingHelper.shrinkTemplateProperties(chargingStationTemplate) + } }); } // Master/Slave: always override the charge point @@ -1583,10 +1587,12 @@ export default class OCPPUtils { tenantID: tenant.id, action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE, module: MODULE_NAME, method: 'enrichChargingStationWithTemplate', - message: 'No Template has been found for this Charging Station', - detailedMessages: { chargingStation } + message: 'No template has been found', + detailedMessages: { + chargingStationData: LoggingHelper.shrinkChargingStationProperties(chargingStation) + } }); - chargingStation.manualConfiguration = true; + chargingStation.manualConfiguration = true; // To be clarified! - Why is this changed here??? } return templateUpdateResult; } diff --git a/src/storage/mongodb/ChargingStationStorage.ts b/src/storage/mongodb/ChargingStationStorage.ts index 9b629625b9..140955f100 100644 --- a/src/storage/mongodb/ChargingStationStorage.ts +++ b/src/storage/mongodb/ChargingStationStorage.ts @@ -135,7 +135,7 @@ export default class ChargingStationStorage { connectorStatuses?: ChargePointStatus[]; connectorTypes?: ConnectorType[]; statusChangedBefore?: Date; withSiteArea?: boolean; withUser?: boolean; ocpiEvseUid?: string; ocpiLocationID?: string; oicpEvseID?: string; siteIDs?: string[]; companyIDs?: string[]; withSite?: boolean; includeDeleted?: boolean; offlineSince?: Date; issuer?: boolean; - locCoordinates?: number[]; locMaxDistanceMeters?: number; public?: boolean; + locCoordinates?: number[]; locMaxDistanceMeters?: number; public?: boolean; manualConfiguration?: boolean; }, dbParams: DbParams, projectFields?: string[]): Promise { const startTime = Logging.traceDatabaseRequestStart(); @@ -180,9 +180,13 @@ export default class ChargingStationStorage { filters.deleted = { '$ne': true }; } // Public Charging Stations - if (Utils.objectHasProperty(params, 'public') && Utils.isBoolean(params.public)) { + if (Utils.isBoolean(params.public)) { filters.public = params.public; } + // Charging Station + if (Utils.isBoolean(params.manualConfiguration)) { + filters.manualConfiguration = params.manualConfiguration; + } // Charging Stations if (!Utils.isEmptyArray(params.chargingStationIDs)) { filters._id = { diff --git a/src/utils/Constants.ts b/src/utils/Constants.ts index 4ba12c4de2..bbdf28523b 100644 --- a/src/utils/Constants.ts +++ b/src/utils/Constants.ts @@ -38,8 +38,8 @@ export default class Constants { public static readonly EXPORT_PAGE_SIZE = 1000; public static readonly EXPORT_RECORD_MAX_COUNT = 100000; public static readonly IMPORT_PAGE_SIZE = 1000; - public static readonly IMPORT_BATCH_INSERT_SIZE = 250; - public static readonly BATCH_PAGE_SIZE = 1000; + public static readonly IMPORT_BATCH_INSERT_SIZE = 256; + public static readonly BATCH_PAGE_SIZE = 256; public static readonly SFDP_LATTITUDE = 43.585784; public static readonly SFDP_LONGITUDE = 7.0377592; diff --git a/src/utils/LoggingHelper.ts b/src/utils/LoggingHelper.ts index 2c66868b27..0dd83ec6a9 100644 --- a/src/utils/LoggingHelper.ts +++ b/src/utils/LoggingHelper.ts @@ -1,6 +1,7 @@ +import ChargingStation, { ChargingStationTemplate } from '../types/ChargingStation'; + import Asset from '../types/Asset'; import { Car } from '../types/Car'; -import ChargingStation from '../types/ChargingStation'; import { PricingContext } from '../types/Pricing'; import RegistrationToken from '../types/RegistrationToken'; import Site from '../types/Site'; @@ -110,4 +111,22 @@ export default class LoggingHelper { stopTimestamp: transaction?.stop?.timestamp }; } + + public static shrinkChargingStationProperties(chargingStation: ChargingStation) { + return { + id: chargingStation?.id, + chargePointVendor: chargingStation?.chargePointVendor, + chargePointModel: chargingStation?.chargePointModel, + chargePointSerialNumber: chargingStation?.chargePointSerialNumber, + firmwareVersion: chargingStation?.firmwareVersion, + }; + } + + public static shrinkTemplateProperties(templateContainer: ChargingStationTemplate) { + return { + id: templateContainer?.id, + chargePointVendor: templateContainer?.template?.chargePointVendor, + extraFilters: templateContainer?.template?.extraFilters, + }; + } } From 51d7ebf49ce6cde02dd94e84bcbfaca9e89b3a8e Mon Sep 17 00:00:00 2001 From: ClaudeROSSI Date: Tue, 7 Mar 2023 09:50:08 +0100 Subject: [PATCH 3/3] pm2 config - troubleshooting memory leaks --- .vscode/launch.json | 24 +++++++++++++++++++++--- ecosystem.config.js | 24 ++++++++++++++++++++++++ package.json | 1 + 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 ecosystem.config.js diff --git a/.vscode/launch.json b/.vscode/launch.json index cc40977476..157c024388 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,7 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { "type": "node", "name": "vscode-jest-tests", @@ -100,9 +101,26 @@ ], "stopOnEntry": true }, - - - + { + "name": "start:dev:debug:batch", + "type": "pwa-node", + "request": "launch", + "cwd": "${workspaceFolder}", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "run-script", + "start:dev:debug:batch" + ], + "sourceMaps": true, + "resolveSourceMapLocations": [ + "${workspaceFolder}/**", + "!**/node_modules/**" + ], + "skipFiles": [ + "/**" + ], + "stopOnEntry": true + }, { "name": "Debug Tests 'createContext'", "type": "pwa-node", diff --git a/ecosystem.config.js b/ecosystem.config.js new file mode 100644 index 0000000000..6ea5867201 --- /dev/null +++ b/ecosystem.config.js @@ -0,0 +1,24 @@ +module.exports = { + apps: [{ + env: { + NODE_ENV: "development", + SERVER_ROLE: "ocppj" + }, + name: "pm2 - OCPP", + script: "./dist/start.js" + }, { + env: { + NODE_ENV: "development", + SERVER_ROLE: "rest" + }, + name: "pm2 - REST", + script: "./dist/start.js" + }, { + env: { + NODE_ENV: "development", + SERVER_ROLE: "batch" + }, + name: "pm2 - BATCH", + script: "./dist/start.js" + }] +} diff --git a/package.json b/package.json index 5dc6bcb19f..afbba662cf 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "start:dev:debug": "npm version && cross-env NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts", "start:dev:debug:ocppj": "npm version && cross-env SERVER_ROLE=ocppj NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts", "start:dev:debug:rest": "npm version && cross-env SERVER_ROLE=rest NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts", + "start:dev:debug:batch": "npm version && cross-env SERVER_ROLE=batch NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts", "start:dev:debug:nodemon": "npm version && nodemon --exec \"ts-node --files\" src/start.ts 9229", "start:dev:nodemon": "npm version && nodemon --exec \"ts-node --files\" src/start.ts", "start:dev:prof": "npm version && cross-env NODE_OPTIONS=\"--max-old-space-size=4096\" NODE_ENV=development node -r source-map-support/register --prof -- dist/start.js",