Merge pull request #4143 from sap-labs-france/master-labs

Version 2.7.5 - Performance improvements
sap-labs-france · Mar 8, 2023 · 33ff54b · 33ff54b
2 parents 4e27c25 + 13a3bd4
commit 33ff54b
Show file tree

Hide file tree

Showing 10 changed files with 139 additions and 50 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,6 +4,7 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+
     {
       "type": "node",
       "name": "vscode-jest-tests",
@@ -100,9 +101,26 @@
       ],
       "stopOnEntry": true
     },
-
-
-
+    {
+      "name": "start:dev:debug:batch",
+      "type": "pwa-node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": [
+        "run-script",
+        "start:dev:debug:batch"
+      ],
+      "sourceMaps": true,
+      "resolveSourceMapLocations": [
+        "${workspaceFolder}/**",
+        "!**/node_modules/**"
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "stopOnEntry": true
+    },
     {
       "name": "Debug Tests 'createContext'",
       "type": "pwa-node",

diff --git a/ecosystem.config.js b/ecosystem.config.js
@@ -0,0 +1,24 @@
+module.exports = {
+  apps: [{
+    env: {
+      NODE_ENV: "development",
+      SERVER_ROLE: "ocppj"
+    },
+    name: "pm2 - OCPP",
+    script: "./dist/start.js"
+  }, {
+    env: {
+      NODE_ENV: "development",
+      SERVER_ROLE: "rest"
+    },
+    name: "pm2 - REST",
+    script: "./dist/start.js"
+  }, {
+    env: {
+      NODE_ENV: "development",
+      SERVER_ROLE: "batch"
+    },
+    name: "pm2 - BATCH",
+    script: "./dist/start.js"
+  }]
+}
diff --git a/package.json b/package.json
@@ -32,6 +32,7 @@
     "start:dev:debug": "npm version && cross-env NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts",
     "start:dev:debug:ocppj": "npm version && cross-env SERVER_ROLE=ocppj NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts",
     "start:dev:debug:rest": "npm version && cross-env SERVER_ROLE=rest NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts",
+    "start:dev:debug:batch": "npm version && cross-env SERVER_ROLE=batch NODE_ENV=development ts-node-dev --inspect --files --max-old-space-size=4096 -- src/start.ts",
     "start:dev:debug:nodemon": "npm version && nodemon --exec \"ts-node --files\" src/start.ts 9229",
     "start:dev:nodemon": "npm version && nodemon --exec \"ts-node --files\" src/start.ts",
     "start:dev:prof": "npm version && cross-env NODE_OPTIONS=\"--max-old-space-size=4096\" NODE_ENV=development node -r source-map-support/register --prof -- dist/start.js",

diff --git a/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts b/src/integration/smart-charging/sap-smart-charging/SapSmartChargingIntegration.ts
@@ -119,7 +119,7 @@ export default class SapSmartChargingIntegration extends SmartChargingIntegratio
       this.tenant, { chargingStationIDs: chargingStationIDs, profilePurposeType:  ChargingProfilePurposeType.TX_PROFILE }, Constants.DB_PARAMS_MAX_LIMIT);
     const currentChargingProfiles = currentChargingProfilesResponse.result;
     // Get all transactions of the site areas
-    const transactions = await TransactionStorage.getTransactions(this.tenant, { transactionIDs, withCar: true }, Constants.DB_PARAMS_MAX_LIMIT);
+    const transactions = await TransactionStorage.getTransactions(this.tenant, { transactionIDs, withSmartChargingData: true }, Constants.DB_PARAMS_MAX_LIMIT);
     // Build request
     const request = await this.buildOptimizerRequest(rootSiteArea, excludedChargingStations, false, currentChargingProfiles, transactions.result);
     // Call optimizer

diff --git a/src/scheduler/tasks/CheckChargingStationTemplateTask.ts b/src/scheduler/tasks/CheckChargingStationTemplateTask.ts
@@ -32,39 +32,45 @@ export default class CheckChargingStationTemplateTask extends TenantSchedulerTas
 
   private async applyTemplateToChargingStations(tenant: Tenant) {
     let updated = 0;
-    // Bypass perf tenant
-    if (tenant.subdomain === 'testperf') {
-      await Logging.logWarning({
-        tenantID: tenant.id,
-        action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
-        module: MODULE_NAME, method: 'applyTemplateToChargingStations',
-        message: `Bypassed tenant ${Utils.buildTenantName(tenant)})`
+    // Prepare pagination
+    const limit = Constants.BATCH_PAGE_SIZE; // Avoid loading too much data in one shot
+    const sort = null; // No sort
+    let skip = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      // Get the charging stations
+      const chargingStations = await ChargingStationStorage.getChargingStations(tenant, {
+        issuer: true,
+        manualConfiguration: false, // template cannot be applied when a manual configuration is being used
+        // withSiteArea: true, // Site area data is loaded only when necessary
+      }, {
+        limit, skip, sort
       });
-      return;
-    }
-    // Get the charging stations
-    const chargingStations = await ChargingStationStorage.getChargingStations(tenant, {
-      issuer: true, withSiteArea: true
-    }, Constants.DB_PARAMS_MAX_LIMIT);
-    // Update
-    for (const chargingStation of chargingStations.result) {
-      try {
-        // Apply template
-        const chargingStationTemplateUpdateResult = await OCPPUtils.checkAndApplyTemplateToChargingStation(tenant, chargingStation);
-        // Save
-        if (chargingStationTemplateUpdateResult.chargingStationUpdated) {
-          await ChargingStationStorage.saveChargingStation(tenant, chargingStation);
-          updated++;
+      if (Utils.isEmptyArray(chargingStations.result)) {
+        break;
+      }
+      // Increment skip for next round
+      skip += limit;
+      // Update
+      for (const chargingStation of chargingStations.result) {
+        try {
+          // Apply template
+          const chargingStationTemplateUpdateResult = await OCPPUtils.checkAndApplyTemplateToChargingStation(tenant, chargingStation);
+          // Save
+          if (chargingStationTemplateUpdateResult.chargingStationUpdated) {
+            await ChargingStationStorage.saveChargingStation(tenant, chargingStation);
+            updated++;
+          }
+        } catch (error) {
+          await Logging.logError({
+            ...LoggingHelper.getChargingStationProperties(chargingStation),
+            tenantID: tenant.id,
+            action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
+            module: MODULE_NAME, method: 'applyTemplateToChargingStations',
+            message: `Template update error in Tenant ${Utils.buildTenantName(tenant)}): ${error.message as string}`,
+            detailedMessages: { error: error.stack }
+          });
         }
-      } catch (error) {
-        await Logging.logError({
-          ...LoggingHelper.getChargingStationProperties(chargingStation),
-          tenantID: tenant.id,
-          action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
-          module: MODULE_NAME, method: 'applyTemplateToChargingStations',
-          message: `Template update error in Tenant ${Utils.buildTenantName(tenant)}): ${error.message as string}`,
-          detailedMessages: { error: error.stack }
-        });
       }
     }
     if (updated > 0) {

diff --git a/src/server/ocpp/utils/OCPPUtils.ts b/src/server/ocpp/utils/OCPPUtils.ts
@@ -1530,13 +1530,12 @@ export default class OCPPUtils {
     if (chargingStationTemplate) {
       // Already updated?
       if (chargingStation.templateHash !== chargingStationTemplate.hash) {
-        await Logging.logInfo({
+        await Logging.logDebug({
           ...LoggingHelper.getChargingStationProperties(chargingStation),
           tenantID: tenant.id,
           action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
           module: MODULE_NAME, method: 'enrichChargingStationWithTemplate',
-          message: `Template ID '${chargingStationTemplate.id}' is been applied...`,
-          detailedMessages: { chargingStationTemplate, chargingStation }
+          message: `Template ID '${chargingStationTemplate.id}' is being applied...`
         });
         // Check Technical
         templateUpdateResult.technicalUpdated =
@@ -1559,16 +1558,21 @@ export default class OCPPUtils {
           action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
           module: MODULE_NAME, method: 'enrichChargingStationWithTemplate',
           message: `Template ID '${chargingStationTemplate.id}' has been applied with success`,
-          detailedMessages: { templateUpdateResult, chargingStationTemplate, chargingStation }
+          detailedMessages: {
+            templateUpdateResult,
+            templateData: LoggingHelper.shrinkTemplateProperties(chargingStationTemplate),
+          }
         });
       } else {
-        await Logging.logInfo({
+        await Logging.logDebug({
           ...LoggingHelper.getChargingStationProperties(chargingStation),
           tenantID: tenant.id,
           action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
           module: MODULE_NAME, method: 'enrichChargingStationWithTemplate',
           message: `Template ID '${chargingStationTemplate.id}' has already been applied`,
-          detailedMessages: { chargingStationTemplate, chargingStation }
+          detailedMessages: {
+            templateData: LoggingHelper.shrinkTemplateProperties(chargingStationTemplate)
+          }
         });
       }
       // Master/Slave: always override the charge point
@@ -1583,10 +1587,12 @@ export default class OCPPUtils {
         tenantID: tenant.id,
         action: ServerAction.UPDATE_CHARGING_STATION_WITH_TEMPLATE,
         module: MODULE_NAME, method: 'enrichChargingStationWithTemplate',
-        message: 'No Template has been found for this Charging Station',
-        detailedMessages: { chargingStation }
+        message: 'No template has been found',
+        detailedMessages: {
+          chargingStationData: LoggingHelper.shrinkChargingStationProperties(chargingStation)
+        }
       });
-      chargingStation.manualConfiguration = true;
+      chargingStation.manualConfiguration = true; // To be clarified! - Why is this changed here???
     }
     return templateUpdateResult;
   }

diff --git a/src/storage/mongodb/ChargingStationStorage.ts b/src/storage/mongodb/ChargingStationStorage.ts
@@ -135,7 +135,7 @@ export default class ChargingStationStorage {
         connectorStatuses?: ChargePointStatus[]; connectorTypes?: ConnectorType[]; statusChangedBefore?: Date; withSiteArea?: boolean; withUser?: boolean;
         ocpiEvseUid?: string; ocpiLocationID?: string; oicpEvseID?: string;
         siteIDs?: string[]; companyIDs?: string[]; withSite?: boolean; includeDeleted?: boolean; offlineSince?: Date; issuer?: boolean;
-        locCoordinates?: number[]; locMaxDistanceMeters?: number; public?: boolean;
+        locCoordinates?: number[]; locMaxDistanceMeters?: number; public?: boolean; manualConfiguration?: boolean;
       },
       dbParams: DbParams, projectFields?: string[]): Promise<ChargingStationDataResult> {
     const startTime = Logging.traceDatabaseRequestStart();
@@ -180,9 +180,13 @@ export default class ChargingStationStorage {
       filters.deleted = { '$ne': true };
     }
     // Public Charging Stations
-    if (Utils.objectHasProperty(params, 'public') && Utils.isBoolean(params.public)) {
+    if (Utils.isBoolean(params.public)) {
       filters.public = params.public;
     }
+    // Charging Station
+    if (Utils.isBoolean(params.manualConfiguration)) {
+      filters.manualConfiguration = params.manualConfiguration;
+    }
     // Charging Stations
     if (!Utils.isEmptyArray(params.chargingStationIDs)) {
       filters._id = {

diff --git a/src/storage/mongodb/TransactionStorage.ts b/src/storage/mongodb/TransactionStorage.ts
@@ -328,7 +328,7 @@ export default class TransactionStorage {
         ocpiSessionID?: string; ocpiAuthorizationID?: string; ocpiSessionDateFrom?: Date; ocpiSessionDateTo?: Date; ocpiCdrDateFrom?: Date; ocpiCdrDateTo?: Date;
         ocpiSessionChecked?: boolean; ocpiCdrChecked?: boolean; oicpSessionID?: string; withSite?: boolean; withSiteArea?: boolean; withCompany?: boolean;
         statistics?: TransactionStatisticsType; refundStatus?: RefundStatus[]; withTag?: boolean; hasUserID?: boolean; withUser?: boolean; withCar?: boolean;
-        transactionsToStop?: boolean; siteOwnerIDs?: string[];
+        transactionsToStop?: boolean; siteOwnerIDs?: string[]; withSmartChargingData?: boolean
       },
       dbParams: DbParams, projectFields?: string[]): Promise<TransactionDataResult> {
     const startTime = Logging.traceDatabaseRequestStart();
@@ -753,6 +753,17 @@ export default class TransactionStorage {
         foreignField: '_id', oneToOneCardinality: true
       });
     }
+    // Smart Charging Data
+    if (params.withSmartChargingData) {
+      DatabaseUtils.pushCarLookupInAggregation({
+        tenantID: tenant.id, aggregation: aggregation, asField: 'car', localField: 'carID',
+        foreignField: '_id', oneToOneCardinality: true, oneToOneCardinalityNotNull: false, projectFields:['converter.amperagePerPhase']
+      });
+      DatabaseUtils.pushCarCatalogLookupInAggregation({
+        tenantID: Constants.DEFAULT_TENANT_ID, aggregation: aggregation, asField: 'carCatalog', localField: 'carCatalogID',
+        foreignField: '_id', oneToOneCardinality: true, projectFields:['fastChargePowerMax', 'batteryCapacityFull']
+      });
+    }
     // Rename ID
     DatabaseUtils.pushRenameDatabaseIDToNumber(aggregation);
     // Convert Object ID to string

diff --git a/src/utils/Constants.ts b/src/utils/Constants.ts
@@ -38,8 +38,8 @@ export default class Constants {
   public static readonly EXPORT_PAGE_SIZE = 1000;
   public static readonly EXPORT_RECORD_MAX_COUNT = 100000;
   public static readonly IMPORT_PAGE_SIZE = 1000;
-  public static readonly IMPORT_BATCH_INSERT_SIZE = 250;
-  public static readonly BATCH_PAGE_SIZE = 1000;
+  public static readonly IMPORT_BATCH_INSERT_SIZE = 256;
+  public static readonly BATCH_PAGE_SIZE = 256;
 
   public static readonly SFDP_LATTITUDE = 43.585784;
   public static readonly SFDP_LONGITUDE = 7.0377592;

diff --git a/src/utils/LoggingHelper.ts b/src/utils/LoggingHelper.ts
@@ -1,6 +1,7 @@
+import ChargingStation, { ChargingStationTemplate } from '../types/ChargingStation';
+
 import Asset from '../types/Asset';
 import { Car } from '../types/Car';
-import ChargingStation from '../types/ChargingStation';
 import { PricingContext } from '../types/Pricing';
 import RegistrationToken from '../types/RegistrationToken';
 import Site from '../types/Site';
@@ -110,4 +111,22 @@ export default class LoggingHelper {
       stopTimestamp: transaction?.stop?.timestamp
     };
   }
+
+  public static shrinkChargingStationProperties(chargingStation: ChargingStation) {
+    return {
+      id: chargingStation?.id,
+      chargePointVendor: chargingStation?.chargePointVendor,
+      chargePointModel: chargingStation?.chargePointModel,
+      chargePointSerialNumber: chargingStation?.chargePointSerialNumber,
+      firmwareVersion: chargingStation?.firmwareVersion,
+    };
+  }
+
+  public static shrinkTemplateProperties(templateContainer: ChargingStationTemplate) {
+    return {
+      id: templateContainer?.id,
+      chargePointVendor: templateContainer?.template?.chargePointVendor,
+      extraFilters: templateContainer?.template?.extraFilters,
+    };
+  }
 }