fix: added SleepAfterImport to possibly address MERGE conflicts

miracum · May 28, 2024 · e16fd4a · e16fd4a
1 parent 302544a
commit e16fd4a
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 2 deletions.
diff --git a/src/PathlingS3Import.Tests.E2E/Tests.cs b/src/PathlingS3Import.Tests.E2E/Tests.cs
@@ -82,6 +82,6 @@ public async Task StartImportTool_WithRunningPathlingServerAndMinio_ShouldCreate
         );
 
         response.Should().NotBeNull();
-        response!.Total.Should().Be(5007);
+        response!.Total.Should().Be(50007);
     }
 }
diff --git a/src/PathlingS3Import/Commands/ImportCommand.cs b/src/PathlingS3Import/Commands/ImportCommand.cs
@@ -1,3 +1,4 @@
+using System;
 using System.Diagnostics;
 using System.Reactive.Linq;
 using System.Text;
@@ -79,6 +80,9 @@ public ImportCommand()
     [CliOption(Description = "Name of the checkpoint file", Name = "--checkpoint-file-name")]
     public string CheckpointFileName { get; set; } = "_last-import-checkpoint.json";
 
+    [CliOption(Description = "Delay to wait after importing a bundle")]
+    public TimeSpan SleepAfterImport { get; set; } = TimeSpan.FromSeconds(10);
+
     public async Task RunAsync()
     {
         log.LogInformation(
@@ -425,6 +429,13 @@ await retryPipeline.ExecuteAsync(async token =>
                     importedCount,
                     objectsToProcessCount
                 );
+
+                log.LogInformation(
+                    "Sleeping after import for {SleepAfterImportSeconds} s",
+                    SleepAfterImport.TotalSeconds
+                );
+
+                await Task.Delay(SleepAfterImport);
             }
         }
 

diff --git a/src/PathlingS3Import/Commands/MergeCommand.cs b/src/PathlingS3Import/Commands/MergeCommand.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using System.Reactive.Linq;
 using System.Text;
 using System.Text.Json;
@@ -95,7 +96,7 @@ await minio.ListObjectsAsync(listArgs).ToList()
             })
             .ToList();
 
-        var currentMergedResources = new Dictionary<string, string>();
+        var currentMergedResources = new ConcurrentDictionary<string, string>();
         var estimatedSizeInBytes = 0;
         foreach (var item in objectsToProcess)
         {