feat(seeding): add initial wallet seeding (#109)

Refs: #108
Co-authored-by: Evelyn Gurschler <evelyn.gurschler@bmw.de>
Reviewed-by: Evelyn Gurschler <evelyn.gurschler@bmw.de>

charts/dim/README.md

@@ -51,13 +51,14 @@ dependencies:
 | dim.healthChecks.readyness.path | string | `"/ready"` |  |
 | dim.swaggerEnabled | bool | `false` |  |
 | dim.operatorId | string | `"00000000-0000-0000-0000-000000000000"` |  |
+| dim.bpn | string | `"empty"` | The bpn of the operator |
+| dim.didDocumentLocationAddress | string | `"https://example.org"` | The did document location address for the operator wallet |
 | migrations.name | string | `"migrations"` |  |
 | migrations.image.name | string | `"ghcr.io/sap/ssi-dim-middle-layer_dim-migrations"` |  |
 | migrations.image.tag | string | `""` |  |
 | migrations.imagePullPolicy | string | `"IfNotPresent"` |  |
 | migrations.resources | object | `{"limits":{"cpu":"45m","memory":"200M"},"requests":{"cpu":"15m","memory":"200M"}}` | We recommend to review the default resource limits as this should a conscious choice. |
-| migrations.seeding.testDataEnvironments | string | `""` |  |
-| migrations.seeding.testDataPaths | string | `"Seeder/Data"` |  |
+| migrations.seeding.useInitial | bool | `true` | Enables dynamic seeding of bpn (dim.bpn) and did document location address (dim.didDocumentLocationAddress) of the operator; If set to `true` the data configured in the config map 'configmap-seeding-initialdata.yaml' will be taken to insert the initial data; |
 | migrations.logging.default | string | `"Information"` |  |
 | processesworker.name | string | `"processesworker"` |  |
 | processesworker.image.name | string | `"ghcr.io/sap/ssi-dim-middle-layer_dim-processes-worker"` |  |

charts/dim/templates/configmap-seeding-initialdata.yaml

@@ -0,0 +1,40 @@
+{{- /*
+* Copyright (c) 2024 BMW Group AG
+* Copyright 2024 SAP SE or an SAP affiliate company and ssi-dim-middle-layer contributors.
+*
+* See the NOTICE file(s) distributed with this work for additional
+* information regarding copyright ownership.
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+*/}}
+
+{{- if (.Values.migrations.seeding.useInitial) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name:  {{ include "dim.fullname" . }}-seeding-initialdata
+  namespace: {{ .Release.Namespace }}
+data:
+  tenants.json: |-
+    [
+      {
+        "id": "5c9a4f56-0609-49a5-ab86-dd8f93dfd3fa",
+        "company_name": "Catena-X",
+        "bpn": "{{ .Values.dim.bpn }}",
+        "did_document_location": "{{ .Values.dim.didDocumentLocationAddress }}",
+        "process_id": "dd371565-9489-4907-a2e4-b8cbfe7a8cd1",
+        "is_issuer": true,
+        "operator_id": "{{ .Values.dim.operatorId }}"
+      }
+    ]
+{{- end }}

charts/dim/templates/job-migrations.yaml

@@ -38,7 +38,7 @@ spec:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
-            - ALL
+              - ALL
           readOnlyRootFilesystem: true
           runAsNonRoot: true
         image: "{{ .Values.migrations.image.name }}:{{ .Values.migrations.image.tag | default .Chart.AppVersion }}"
@@ -64,15 +64,31 @@ spec:
           - name: "CONNECTIONSTRINGS__DIMDB"
             value: "Server={{ .Values.externalDatabase.host }};Database={{ .Values.externalDatabase.database }};Port={{ .Values.externalDatabase.port }};User Id={{ .Values.externalDatabase.username }};Password=$(DIM_PASSWORD);Ssl Mode={{ .Values.dbConnection.sslMode }};"
           {{- end }}
-          - name: "SEEDING__TESTDATAENVIRONMENTS__0"
-            value: "{{ .Values.migrations.seeding.testDataEnvironments }}"
           - name: "SEEDING__DATAPATHS__0"
-            value: "{{ .Values.migrations.seeding.testDataPaths }}"
+            value: "Seeder/Data/"
+          - name: "SEEDING__TESTDATAENVIRONMENTS__0"
+            value: ""
+          - name: "SEEDING__DATAPATHS__1"
+            value: "Seeder/Data/overwrite"
+          - name: "SEEDING__TESTDATAENVIRONMENTS__1"
+            value: ""
           - name: "SERILOG__MINIMUMLEVEL__Default"
             value: "{{ .Values.migrations.logging.default }}"
         ports:
-        - name: http
-          containerPort: {{ .Values.portContainer }}
-          protocol: TCP
+          - name: http
+            containerPort: {{ .Values.portContainer }}
+            protocol: TCP
         resources:
           {{- toYaml .Values.migrations.resources | nindent 10 }}
+        {{- if (.Values.migrations.seeding.useInitial) }}
+        volumeMounts:
+          - name: data-initial
+            mountPath: /migrations/Seeder/Data/overwrite
+        {{- end }}
+      {{- if (.Values.migrations.seeding.useInitial) }}
+      volumes:
+        - name: data-initial
+          configMap:
+            name: "{{ include "dim.fullname" . }}-seeding-initialdata"
+            optional: true
+      {{- end }}

charts/dim/values.yaml

@@ -43,6 +43,10 @@ dim:
       path: "/ready"
   swaggerEnabled: false
   operatorId: "00000000-0000-0000-0000-000000000000"
+  # -- The bpn of the operator
+  bpn: "empty"
+  # -- The did document location address for the operator wallet
+  didDocumentLocationAddress: "https://example.org"
 
 migrations:
   name: "migrations"
@@ -59,8 +63,9 @@ migrations:
       cpu: 45m
       memory: 200M
   seeding:
-    testDataEnvironments: ""
-    testDataPaths: "Seeder/Data"
+    # -- Enables dynamic seeding of bpn (dim.bpn) and did document location address (dim.didDocumentLocationAddress) of the operator;
+    # If set to `true` the data configured in the config map 'configmap-seeding-initialdata.yaml' will be taken to insert the initial data;
+    useInitial: true
   logging:
     default: "Information"
 

environments/helm-values/values-int.yaml

@@ -41,7 +41,9 @@ ingress:
 
 dim:
   swaggerEnabled: true
-  operatorId: "d6cd4e2e-1053-4ba6-888e-1cd56509958a"
+  operatorId: "d6cd4e2e-1053-4ba6-888e-1cd56509958a"   
+  bpn: "BPNL00000003CRHK"
+  didDocumentLocationAddress: "https://portal-backend.int.catena-x.net/api/administration/staticdata/did/BPNL00000003CRHK/did.json"
 
 migrations:
   logging:

src/database/Dim.Migrations/Dim.Migrations.csproj

@@ -54,14 +54,16 @@
     <Content Include="appsettings.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
-  </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\Dim.Entities\Dim.Entities.csproj" />
+    <None Remove="Seeder\Data" />
+    <Content Include="Seeder\Data\**" LinkBase="Seeder\Data" CopyToOutputDirectory="Always" />
+    <Content Update="Seeder\Data\overwriteable\tenants.json">
+      <LinkBase>Seeder/Data/</LinkBase>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>
-    <Folder Include="Migrations\" />
+    <ProjectReference Include="..\Dim.Entities\Dim.Entities.csproj" />
   </ItemGroup>
 
 </Project>

tests/database/Dim.DbAccess.Tests/Seeder/BatchInsertSeeder.cs → src/database/Dim.Migrations/Seeder/BatchInsertSeeder.cs

@@ -24,29 +24,15 @@
 using Microsoft.Extensions.Options;
 using Org.Eclipse.TractusX.Portal.Backend.Framework.Seeding;
 
-namespace Dim.DbAccess.Tests.Seeder;
+namespace Dim.Migrations.Seeder;
 
 /// <summary>
 /// Seeder to seed the base entities (those with an id as primary key)
 /// </summary>
-public class BatchInsertSeeder : ICustomSeeder
+public class BatchInsertSeeder(DimDbContext context, ILogger<BatchInsertSeeder> logger, IOptions<SeederSettings> options)
+    : ICustomSeeder
 {
-    private readonly DimDbContext _context;
-    private readonly ILogger<BatchInsertSeeder> _logger;
-    private readonly SeederSettings _settings;
-
-    /// <summary>
-    /// Constructor
-    /// </summary>
-    /// <param name="context">The database context</param>
-    /// <param name="logger">The logger</param>
-    /// <param name="options">The options</param>
-    public BatchInsertSeeder(DimDbContext context, ILogger<BatchInsertSeeder> logger, IOptions<SeederSettings> options)
-    {
-        _context = context;
-        _logger = logger;
-        _settings = options.Value;
-    }
+    private readonly SeederSettings _settings = options.Value;
 
     /// <inheritdoc />
     public int Order => 1;
@@ -56,37 +42,36 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
         if (!_settings.DataPaths.Any())
         {
-            _logger.LogInformation("There a no data paths configured, therefore the {SeederName} will be skipped", nameof(BatchInsertSeeder));
+            logger.LogInformation("There a no data paths configured, therefore the {SeederName} will be skipped", nameof(BatchInsertSeeder));
             return;
         }
 
-        _logger.LogInformation("Start BaseEntityBatch Seeder");
+        logger.LogInformation("Start BaseEntityBatch Seeder");
         await SeedTable<Tenant>("tenants", x => x.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
         await SeedTable<ProcessStep>("process_steps", x => x.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
         await SeedTable<Process>("processes", x => x.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
         await SeedTable<TechnicalUser>("technical_users", x => x.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
 
-        await _context.SaveChangesAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        _logger.LogInformation("Finished BaseEntityBatch Seeder");
+        await context.SaveChangesAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+        logger.LogInformation("Finished BaseEntityBatch Seeder");
     }
 
     private async Task SeedTable<T>(string fileName, Func<T, object> keySelector, CancellationToken cancellationToken) where T : class
     {
-        _logger.LogInformation("Start seeding {Filename}", fileName);
+        logger.LogInformation("Start seeding {Filename}", fileName);
         var additionalEnvironments = _settings.TestDataEnvironments ?? Enumerable.Empty<string>();
-        var data = await SeederHelper.GetSeedData<T>(_logger, fileName, _settings.DataPaths, cancellationToken, additionalEnvironments.ToArray()).ConfigureAwait(ConfigureAwaitOptions.None);
-        _logger.LogInformation("Found {ElementCount} data", data.Count);
+        var data = await SeederHelper.GetSeedData<T>(logger, fileName, _settings.DataPaths, cancellationToken, additionalEnvironments.ToArray()).ConfigureAwait(ConfigureAwaitOptions.None);
+        logger.LogInformation("Found {ElementCount} data", data.Count);
         if (data.Any())
         {
             var typeName = typeof(T).Name;
-            _logger.LogInformation("Started to Seed {TableName}", typeName);
-            data = data.GroupJoin(_context.Set<T>(), keySelector, keySelector, (d, dbEntry) => new { d, dbEntry })
+            logger.LogInformation("Started to Seed {TableName}", typeName);
+            data = data.GroupJoin(context.Set<T>(), keySelector, keySelector, (d, dbEntry) => new { d, dbEntry })
                 .SelectMany(t => t.dbEntry.DefaultIfEmpty(), (t, x) => new { t, x })
                 .Where(t => t.x == null)
                 .Select(t => t.t.d).ToList();
-            _logger.LogInformation("Seeding {DataCount} {TableName}", data.Count, typeName);
-            await _context.Set<T>().AddRangeAsync(data, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-            _logger.LogInformation("Seeded {TableName}", typeName);
+            await context.Set<T>().AddRangeAsync(data, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+            logger.LogInformation("Seeded {TableName}", typeName);
         }
     }
 }

src/database/Dim.Migrations/Seeder/BatchUpdateSeeder.cs

@@ -0,0 +1,88 @@
+/********************************************************************************
+ * Copyright (c) 2024 BMW Group AG
+ * Copyright 2024 SAP SE or an SAP affiliate company and ssi-dim-middle-layer contributors.
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+using Dim.Entities;
+using Dim.Entities.Entities;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Org.Eclipse.TractusX.Portal.Backend.Framework.Seeding;
+
+namespace Dim.Migrations.Seeder;
+
+/// <summary>
+/// Seeder to seed the base entities (those with an id as primary key)
+/// </summary>
+public class BatchUpdateSeeder(DimDbContext context, ILogger<BatchInsertSeeder> logger, IOptions<SeederSettings> options)
+    : ICustomSeeder
+{
+    private readonly SeederSettings _settings = options.Value;
+
+    /// <inheritdoc />
+    public int Order => 2;
+
+    /// <inheritdoc />
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        if (!_settings.DataPaths.Any())
+        {
+            logger.LogInformation("There a no data paths configured, therefore the {SeederName} will be skipped", nameof(BatchUpdateSeeder));
+            return;
+        }
+
+        logger.LogInformation("Start BaseEntityBatch Seeder");
+        await SeedTable<Tenant>("tenants",
+            x => x.Id,
+            x => x.dataEntity.CompanyName != x.dbEntity.CompanyName || x.dataEntity.Bpn != x.dbEntity.Bpn || x.dataEntity.DidDocumentLocation != x.dbEntity.DidDocumentLocation,
+            (dbEntry, entry) =>
+            {
+                dbEntry.Bpn = entry.Bpn;
+                dbEntry.CompanyName = entry.CompanyName;
+                dbEntry.DidDocumentLocation = entry.DidDocumentLocation;
+            }, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+
+        await context.SaveChangesAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+        logger.LogInformation("Finished BaseEntityBatch Seeder");
+    }
+
+    private async Task SeedTable<T>(string fileName, Func<T, object> keySelector, Func<(T dataEntity, T dbEntity), bool> whereClause, Action<T, T> updateEntries, CancellationToken cancellationToken) where T : class
+    {
+        logger.LogInformation("Start seeding {Filename}", fileName);
+        var additionalEnvironments = _settings.TestDataEnvironments ?? Enumerable.Empty<string>();
+        var data = await SeederHelper.GetSeedData<T>(logger, fileName, _settings.DataPaths, cancellationToken, additionalEnvironments.ToArray()).ConfigureAwait(ConfigureAwaitOptions.None);
+        logger.LogInformation("Found {ElementCount} data", data.Count);
+        if (data.Any())
+        {
+            var typeName = typeof(T).Name;
+            var entriesForUpdate = data
+                .Join(context.Set<T>(), keySelector, keySelector, (dataEntry, dbEntry) => (DataEntry: dataEntry, DbEntry: dbEntry))
+                .Where(whereClause.Invoke)
+                .ToList();
+            if (entriesForUpdate.Any())
+            {
+                logger.LogInformation("Started to Update {EntryCount} entries of {TableName}", entriesForUpdate.Count, typeName);
+                foreach (var entry in entriesForUpdate)
+                {
+                    updateEntries.Invoke(entry.DbEntry, entry.DataEntry);
+                }
+                logger.LogInformation("Updated {TableName}", typeName);
+            }
+        }
+    }
+}

src/database/Dim.Migrations/Seeder/Data/overwrite/tenants.json

@@ -0,0 +1,11 @@
+[
+  {
+    "id": "5c9a4f56-0609-49a5-ab86-dd8f93dfd3fa",
+    "company_name": "issuer company",
+    "bpn": "BPNL000001ISSUER",
+    "did_document_location": "https://example.org/did.json",
+    "process_id": "dd371565-9489-4907-a2e4-b8cbfe7a8cd1",
+    "is_issuer": true,
+    "operator_id": "dd371565-9489-4907-a2e4-b8cbfe7a8cd1"
+  }
+]

src/database/Dim.Migrations/Seeder/Data/process_steps.json

@@ -0,0 +1,10 @@
+[
+  {
+    "id": "5c9a4f56-0609-49a5-ab86-dd8f93dfd3fa",
+    "process_step_type_id": 1,
+    "process_step_status_id": 1,
+    "process_id": "dd371565-9489-4907-a2e4-b8cbfe7a8cd1",
+    "date_created": "2023-02-21 08:15:20.479000 +00:00",
+    "date_last_changed": null
+  }
+]

src/database/Dim.Migrations/Seeder/Data/processes.json

@@ -0,0 +1,8 @@
+[
+  {
+    "id": "dd371565-9489-4907-a2e4-b8cbfe7a8cd1",
+    "process_type_id" : 1,
+    "lock_expiry_date" : "2023-03-01 00:00:00.000000 +00:00",
+    "version" : "deadbeef-dead-beef-dead-beefdeadbeef"
+  }
+]

src/database/Dim.Migrations/appsettings.json

@@ -28,6 +28,8 @@
   },
   "Seeding": {
     "DataPaths": [
+      "Seeder/Data",
+      "Seeder/Data/overwrite"
     ],
     "TestDataEnvironments": []
   }

tests/database/Dim.DbAccess.Tests/Dim.DbAccess.Tests.csproj

@@ -56,8 +56,4 @@
     <Content Include="Seeder\Data\**" LinkBase="Seeder\Data" CopyToOutputDirectory="Always" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Seeder\" />
-  </ItemGroup>
-
 </Project>