optimise loading and write

NateKomodo · Feb 5, 2021 · 731c45c · 731c45c
1 parent 748a578
commit 731c45c
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 11 deletions.
diff --git a/DiscordToMoon/Program.cs b/DiscordToMoon/Program.cs
@@ -1,9 +1,11 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Drawing;
 using System.Drawing.Imaging;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
 
 namespace DiscordToMoon
@@ -27,22 +29,24 @@ public static void Main(string[] args)
         private static void ToFile(string jsonPath, string imagePath, ImageFormat format)
         {
             Console.WriteLine($"Loading json from {jsonPath}/*");
-
-            var json = Directory.GetFiles(jsonPath).Select(file => JObject.Parse(File.ReadAllText(file))).ToList();
+
+            var json = new ConcurrentStack<JObject>();
+
+            Parallel.ForEach(Directory.GetFiles(jsonPath), file => json.Push(JObject.Parse(File.ReadAllText(file))));
 
             Console.WriteLine($"Loaded {json.Count} files");
 
-            var manifest = new Dictionary<string, List<string>>();
-            
-            foreach (var j in json)
+            var manifest = new ConcurrentDictionary<string, List<string>>();
+
+            Parallel.ForEach(json, j =>
             {
                 var name = j.SelectToken("channel.name")?.ToObject<string>();
                 var messages = j.SelectToken("messages", false)?.ToObject<JArray>();
-                
+
                 if (string.IsNullOrEmpty(name) || messages == null) throw new NullReferenceException();
-                
+
                 var messageList = new List<string>();
-                
+
                 foreach (var message in messages)
                 {
                     var content = message.SelectToken("content", false)?.ToObject<string>();
@@ -55,12 +59,12 @@ private static void ToFile(string jsonPath, string imagePath, ImageFormat format
                 }
 
                 name = "#" + name;
-                
+
                 if (manifest.ContainsKey(name))
                     manifest[name].AddRange(messageList);
                 else
-                    manifest.Add(name, messageList);
-            }
+                    manifest.TryAdd(name, messageList);
+            });
 
             Console.WriteLine($"Loaded {manifest.Keys.Count} channels and {manifest.Values.Select(m => m.Count).Sum()} messages");
             foreach (var kvp in manifest) Console.WriteLine($" - {kvp.Key}: {kvp.Value.Count}");

diff --git a/DiscordToMoon/build.sh b/DiscordToMoon/build.sh
@@ -0,0 +1,3 @@
+dotnet publish -r ubuntu.16.04-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true --self-contained
+dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true --self-contained
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		dotnet publish -r ubuntu.16.04-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true --self-contained
		dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true --self-contained