Faster ChorusNotes handling for LfMerge (#401)

* Move some LfMerge model classes into LfMergeBridge

This will allow these classes to be passed by reference into the
ChorusNotes action handler, saving two large JSON serialization steps.

* Allow passing comments to/from LfMerge w/o JSON

We use the ConditionalWeakTable class, designed to allow compilers to
attach metadata to objects, as an extra input/output mechanism that
doesn't require serializing the comments and replies to JSON strings.
This will save quite a lot of RAM when doing a Send/Receive of a project
with lots and lots of comments.

This implements the LfMergeBridge side of the process; a corresponding
change will be needed in LfMerge.

* Allow FLExBridge to compile on Linux

Linux requires GenerateResourceUsePreserializedResources to be set to
true before `dotnet build` will allow non-string resources to be
compiled into assemblies.

src/FLEx-ChorusPlugin/FLEx-ChorusPlugin.csproj

@@ -4,6 +4,7 @@
     <RootNamespace>FLEx_ChorusPlugin</RootNamespace>
     <AssemblyTitle>FLEx-ChorusPlugin</AssemblyTitle>
     <PackageId>SIL.ChorusPlugin.FLEx</PackageId>
+    <GenerateResourceUsePreserializedResources>true</GenerateResourceUsePreserializedResources>
   </PropertyGroup>
 
   <ItemGroup>

src/FwdataNester/FwdataTestApp.csproj

@@ -5,6 +5,7 @@
     <AssemblyTitle>FwdataTestApp</AssemblyTitle>
     <OutputType>WinExe</OutputType>
     <IsPackable>false</IsPackable>
+    <GenerateResourceUsePreserializedResources>true</GenerateResourceUsePreserializedResources>
   </PropertyGroup>
 
   <ItemGroup>

src/LfMergeBridge/GetChorusNotesInput.cs

@@ -0,0 +1,13 @@
+// Copyright (c) 2010-2024 SIL International
+// This software is licensed under the MIT License (http://opensource.org/licenses/MIT)
+
+using System.Collections.Generic;
+using LfMergeBridge.LfMergeModel;
+
+namespace LfMergeBridge
+{
+    public class GetChorusNotesInput
+    {
+		public List<LfComment> LfComments;
+    }
+}

src/LfMergeBridge/GetChorusNotesResponse.cs

@@ -0,0 +1,16 @@
+// Copyright (c) 2010-2024 SIL International
+// This software is licensed under the MIT License (http://opensource.org/licenses/MIT)
+
+using System;
+using System.Collections.Generic;
+using LfMergeBridge.LfMergeModel;
+
+namespace LfMergeBridge
+{
+    public class GetChorusNotesResponse
+    {
+		public List<LfComment> LfComments;
+		public List<Tuple<string, List<LfCommentReply>>> LfReplies;
+		public List<KeyValuePair<string, Tuple<string, string>>> LfStatusChanges;
+    }
+}

src/LfMergeBridge/LanguageForgeGetChorusNotesActionHandler.cs

@@ -14,6 +14,7 @@
 using LibTriboroughBridgeChorusPlugin.Infrastructure;
 using SIL.Progress;
 using FLEx_ChorusPlugin.Infrastructure.ActionHandlers;
+using LfMergeBridge.LfMergeModel;
 
 namespace LfMergeBridge
 {
@@ -41,86 +42,97 @@ void IBridgeActionTypeHandler.StartWorking(IProgress progress, Dictionary<string
 			ProjectName = Path.GetFileNameWithoutExtension(pOption);
 			ProjectDir = Path.GetDirectoryName(pOption);
 
-			string inputFilename = options[LfMergeBridgeUtilities.serializedCommentsFromLfMerge];
-			List<SerializableLfComment> commentsFromLF = LfMergeBridgeUtilities.DecodeJsonFile<List<SerializableLfComment>>(inputFilename);
-			Dictionary<string, SerializableLfComment> commentsFromLFByGuid = commentsFromLF.Where(comment => comment.Guid != null).ToDictionary(comment => comment.Guid);
-			var knownReplyGuids = new HashSet<string>(commentsFromLF.Where(comment => comment.Replies != null).SelectMany(comment => comment.Replies.Where(reply => reply.Guid != null).Select(reply => reply.Guid)));
+			List<LfComment> commentsFromLF;
+			if (LfMergeBridge.ExtraInputData.TryGetValue(options, out var extraData) && extraData is GetChorusNotesInput inputData)
+			{
+				commentsFromLF = inputData.LfComments;
+			}
+			else
+			{
+				LfMergeBridgeUtilities.AppendLineToSomethingForClient(ref somethingForClient, "No ExtraInputData passed, or it was the wrong type (should be GetChorusNotesInput). Aborting operation.");
+				return;
+			}
 
-			var lfComments = new List<SerializableLfComment>();
-			var lfReplies = new List<Tuple<string, List<SerializableLfCommentReply>>>();
+			Dictionary<Guid, LfComment> commentsFromLFByGuid = commentsFromLF.Where(comment => comment.Guid != null).ToDictionary(comment => comment.Guid.Value);
+			var knownReplyGuids = new HashSet<Guid>(commentsFromLF.Where(comment => comment.Replies != null).SelectMany(comment => comment.Replies.Where(reply => reply.Guid != null).Select(reply => reply.Guid.Value)));
+
+			var lfComments = new List<LfComment>();
+			var lfReplies = new List<Tuple<string, List<LfCommentReply>>>();
 			var lfStatusChanges = new List<KeyValuePair<string, Tuple<string, string>>>();
 			// TODO: See if we want to suppress progress messages here by using a NullProgress instance instead of the IProgress instance we were given...
 			foreach (Annotation ann in GetAllAnnotations(progress, ProjectDir))
 			{
-				if (ann.Guid != null && commentsFromLFByGuid.ContainsKey(ann.Guid))
+				Guid? annGuid = null;
+				if (ann.Guid != null && Guid.TryParse(ann.Guid, out var parsedGuid)) { annGuid = parsedGuid; }
+				if (annGuid != null && commentsFromLFByGuid.ContainsKey(annGuid.Value))
 				{
-					var lfComment = commentsFromLFByGuid[ann.Guid];
-					// Known comment; only serialize new replies
-					List<SerializableLfCommentReply> repliesNotYetInLf =
+					var lfComment = commentsFromLFByGuid[annGuid.Value];
+					// Known comment; only return new replies
+					List<LfCommentReply> repliesNotYetInLf =
 						ann
 							.Messages
 							.Skip(1)  // First message translates to the LF *comment*, while subsequent messages are *replies* in LF
 							.Where(m => ! String.IsNullOrWhiteSpace(m.Text))
-							.Where(m => ! knownReplyGuids.Contains(m.Guid))
+							.Where(m => m.Guid != null && Guid.TryParse(m.Guid, out var mGuid) && ! knownReplyGuids.Contains(mGuid))
 							.Select(ReplyFromChorusMsg)
 							.ToList();
 					if (repliesNotYetInLf.Count > 0)
 					{
-						lfReplies.Add(new Tuple<string, List<SerializableLfCommentReply>>(ann.Guid, repliesNotYetInLf));
+						lfReplies.Add(new Tuple<string, List<LfCommentReply>>(ann.Guid, repliesNotYetInLf));
 					}
 					// But also need to check for status updates
 					string chorusStatus = ChorusStatusToLfStatus(ann.Status);
-					if (chorusStatus != lfComment.Status || lfComment.StatusGuid != ann.StatusGuid)
+					Guid annStatusGuid = Guid.Empty;
+					if (ann.StatusGuid != null) { Guid.TryParse(ann.StatusGuid, out annStatusGuid); }
+					if (chorusStatus != lfComment.Status || (lfComment.StatusGuid ?? Guid.Empty) != annStatusGuid)
 					{
-						lfStatusChanges.Add(new KeyValuePair<string, Tuple<string, string>>(lfComment.Guid, new Tuple<string, string>(chorusStatus, ann.StatusGuid)));
-				}
+						lfStatusChanges.Add(new KeyValuePair<string, Tuple<string, string>>(lfComment.Guid?.ToString(), new Tuple<string, string>(chorusStatus, ann.StatusGuid)));
+					}
 				}
 				else
 				{
-					// New comment: serialize everything
+					// New comment: return all replies
 					var msg = ann.Messages.FirstOrDefault();
-					var lfComment = new SerializableLfComment {
-						Guid = ann.Guid,
+					var lfComment = new LfComment {
+						Guid = annGuid,
 						// AuthorNameAlternate = msg?.Author ?? string.Empty,  // C# 6 syntax would be simpler if we could count on a C# 6 compiler everywhere
 						AuthorNameAlternate = (msg == null) ? string.Empty : msg.Author,
 						DateCreated = ann.Date,
 						DateModified = ann.Date,
 						// Content = msg?.Text ?? string.Empty,  // C# 6 syntax would be simpler if we could count on a C# 6 compiler everywhere
 						Content = (msg == null) ? string.Empty : msg.Text,
 						Status = ChorusStatusToLfStatus(ann.Status),
-						StatusGuid = ann.StatusGuid,
-						Replies = new List<SerializableLfCommentReply>(ann.Messages.Skip(1).Where(m => ! String.IsNullOrWhiteSpace(m.Text)).Select(ReplyFromChorusMsg)),
-						IsDeleted = false
-					};
-					lfComment.Regarding = new SerializableLfCommentRegarding {
-						TargetGuid = ExtractGuidFromChorusRef(ann.RefStillEscaped),
-						// Word and Meaning will be set in LfMerge, but set them to something vaguely sensible here as a fallback
-						Word = ann.LabelOfThingAnnotated,
-						Meaning = string.Empty
+						StatusGuid = Guid.TryParse(ann.StatusGuid, out var annStatusGuid) ? annStatusGuid : Guid.Empty,
+						Replies = new List<LfCommentReply>(ann.Messages.Skip(1).Where(m => !String.IsNullOrWhiteSpace(m.Text)).Select(ReplyFromChorusMsg)),
+						IsDeleted = false,
+						Regarding = new LfCommentRegarding {
+							TargetGuid = ExtractGuidFromChorusRef(ann.RefStillEscaped),
+							// Word and Meaning will be set in LfMerge, but set them to something vaguely sensible here as a fallback
+							Word = ann.LabelOfThingAnnotated,
+							Meaning = string.Empty
+						}
 					};
 					lfComments.Add(lfComment);
 				}
 			}
-			var serializedComments = new StringBuilder("New comments not yet in LF: ");
-			serializedComments.Append(JsonConvert.SerializeObject(lfComments));
-			LfMergeBridgeUtilities.AppendLineToSomethingForClient(ref somethingForClient, serializedComments.ToString());
-
-			var serializedReplies = new StringBuilder("New replies on comments already in LF: ");
-			serializedReplies.Append(JsonConvert.SerializeObject(lfReplies));
-			LfMergeBridgeUtilities.AppendLineToSomethingForClient(ref somethingForClient, serializedReplies.ToString());
 
-			var serializedStatusChanges = new StringBuilder("New status changes on comments already in LF: ");
-			serializedStatusChanges.Append(JsonConvert.SerializeObject(lfStatusChanges));
-			LfMergeBridgeUtilities.AppendLineToSomethingForClient(ref somethingForClient, serializedStatusChanges.ToString());
+			var response = new GetChorusNotesResponse {
+				LfComments = lfComments,
+				LfReplies = lfReplies,
+				LfStatusChanges = lfStatusChanges,
+			};
+			// LfMergeBridge.ExtraOutputData.AddOrUpdate(options, response); // Not available in netstandard2.0
+			LfMergeBridge.ExtraOutputData.Remove(options); // Available in netstandard2.0, does not throw if value does not exist
+			LfMergeBridge.ExtraOutputData.Add(options, response); // Now this is guaranteed safe (would have thrown if previous value had not been removed)
 		}
 
-		private SerializableLfCommentReply ReplyFromChorusMsg(Message msg)
+		private LfCommentReply ReplyFromChorusMsg(Message msg)
 		{
-			var reply = new SerializableLfCommentReply();
-			reply.Guid = msg.Guid;
+			var reply = new LfCommentReply();
+			reply.Guid = Guid.Parse(msg.Guid); // We already know it's parseable by now
 			reply.AuthorNameAlternate = msg.Author;
 			if (reply.AuthorInfo == null)
-				reply.AuthorInfo = new SerializableLfAuthorInfo();
+				reply.AuthorInfo = new LfAuthorInfo();
 			reply.AuthorInfo.CreatedDate = msg.Date;
 			reply.AuthorInfo.ModifiedDate = msg.Date;
 			reply.Content = msg.Text;

src/LfMergeBridge/LfMergeBridge.cs

@@ -5,6 +5,7 @@
 using System.ComponentModel.Composition.Hosting;
 using System.IO;
 using System.Reflection;
+using System.Runtime.CompilerServices;
 using Chorus.VcsDrivers.Mercurial;
 using LibTriboroughBridgeChorusPlugin.Infrastructure;
 using LibTriboroughBridgeChorusPlugin.Infrastructure.ActionHandlers;
@@ -18,6 +19,9 @@ namespace LfMergeBridge
 	/// </summary>
 	public static class LfMergeBridge
 	{
+		public static ConditionalWeakTable<object, object> ExtraInputData { get; } = new ConditionalWeakTable<object, object>();
+		public static ConditionalWeakTable<object, object> ExtraOutputData { get; } = new ConditionalWeakTable<object, object>();
+
 		/// <summary>
 		/// This is the only 'uniform interface' public API neded to support current and future needs of LfMerge.
 		/// </summary>

src/LfMergeBridge/LfMergeBridge.csproj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <PackageReference Include="GitVersion.MsBuild" Version="5.10.3" PrivateAssets="all" />
     <PackageReference Include="SIL.Chorus.LibChorus" Version="$(ChorusVersion)" />
+    <PackageReference Include="MongoDB.Bson.signed" Version="2.13.*" />
     <PackageReference Include="SIL.ReleaseTasks" Version="2.5.0" PrivateAssets="all" />
   </ItemGroup>
 

src/LfMergeBridge/LfMergeModel/IHasNullableGuid.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	public interface IHasNullableGuid
+	{
+//		[BsonRepresentation(BsonType.String)]
+		Guid? Guid { get; set; }
+	}
+}
+

src/LfMergeBridge/LfMergeModel/LfAuthorInfo.cs

@@ -0,0 +1,14 @@
+using System;
+using MongoDB.Bson;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	public class LfAuthorInfo
+	{
+		public ObjectId? CreatedByUserRef { get; set; }
+		public DateTime CreatedDate { get; set; }
+		public ObjectId? ModifiedByUserRef { get; set; }
+		public DateTime ModifiedDate { get; set; }
+	}
+}
+

src/LfMergeBridge/LfMergeModel/LfComment.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	public class LfComment : IHasNullableGuid
+	{
+		public ObjectId Id { get; set; }
+		[BsonRepresentation(BsonType.String)]
+		public Guid? Guid { get; set; }
+		public LfAuthorInfo AuthorInfo { get; set; }
+		public string AuthorNameAlternate { get; set; } // Used in sending comments to FW; should be null when serializing to Mongo
+		public LfCommentRegarding Regarding { get; set; }
+		public DateTime DateCreated { get; set; }
+		public DateTime DateModified { get; set; }
+		public string Content { get; set; }
+		public string Status { get; set; }
+		[BsonRepresentation(BsonType.String)]
+		public Guid? StatusGuid { get; set; }
+		public bool IsDeleted { get; set; }
+		public List<LfCommentReply> Replies { get; set; }
+		public ObjectId EntryRef { get; set; }
+		public int Score { get; set; }
+		public string ContextGuid { get; set; } // not really a GUID
+
+		public bool ShouldSerializeGuid() { return (Guid != null && Guid.Value != System.Guid.Empty); }
+		public bool ShouldSerializeDateCreated() { return true; }
+		public bool ShouldSerializeDateModified() { return true; }
+		public bool ShouldSerializeContent() { return ( ! String.IsNullOrEmpty(Content)); }
+		public bool ShouldSerializeAuthorNameAlternate() { return ( ! String.IsNullOrEmpty(AuthorNameAlternate)); }
+		public bool ShouldSerializeStatusGuid() { return (StatusGuid != null && StatusGuid.Value != System.Guid.Empty); }
+		public bool ShouldSerializeReplies() { return (Replies != null && Replies.Count > 0); }
+
+		public LfComment() {
+			Replies = new List<LfCommentReply>();
+		}
+	}
+}
+

src/LfMergeBridge/LfMergeModel/LfCommentRegarding.cs

@@ -0,0 +1,26 @@
+using System;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	public class LfCommentRegarding
+	{
+		public string TargetGuid { get; set; }
+		public string Field { get; set; }
+		public string FieldNameForDisplay { get; set; }
+		public string FieldValue { get; set; }
+		public string InputSystem { get; set; }
+		public string InputSystemAbbreviation { get; set; }
+		public string Word { get; set; }
+		public string Meaning { get; set; }
+
+		public bool ShouldSerializeEntryGuid() { return ( ! String.IsNullOrEmpty(TargetGuid)); }
+		public bool ShouldSerializeField() { return ( ! String.IsNullOrEmpty(Field)); }
+		public bool ShouldSerializeFieldNameForDisplay() { return ( ! String.IsNullOrEmpty(FieldNameForDisplay)); }
+		public bool ShouldSerializeFieldValue() { return ( ! String.IsNullOrEmpty(FieldValue)); }
+		public bool ShouldSerializeInputSystem() { return ( ! String.IsNullOrEmpty(InputSystem)); }
+		public bool ShouldSerializeInputSystemAbbreviation() { return ( ! String.IsNullOrEmpty(InputSystemAbbreviation)); }
+		public bool ShouldSerializeWord() { return ( ! String.IsNullOrEmpty(Word)); }
+		public bool ShouldSerializeMeaning() { return ( ! String.IsNullOrEmpty(Meaning)); }
+	}
+}
+

src/LfMergeBridge/LfMergeModel/LfCommentReply.cs

@@ -0,0 +1,27 @@
+using System;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	[BsonIgnoreExtraElements] // WARNING: Beware of using FindOneAndReplace() with IgnoreExtraElements, as you can lose data
+	public class LfCommentReply : IHasNullableGuid
+	{
+		[BsonRepresentation(BsonType.String)]
+		public Guid? Guid { get; set; }
+		public LfAuthorInfo AuthorInfo { get; set; }
+		public string AuthorNameAlternate { get; set; } // Used in sending comments to FW; should be null when serializing to Mongo
+		public string Content { get; set; }
+		[BsonElement("id")]
+		public string UniqId { get; set; }  // If we name this field "Id", the C# driver tries to map it to _id and always thinks it is null
+		public bool IsDeleted { get; set; }
+
+		public bool ShouldSerializeGuid() { return (Guid != null && Guid.Value != System.Guid.Empty); }
+		public bool ShouldSerializeContent() { return ( ! String.IsNullOrEmpty(Content)); }
+		public bool ShouldSerializeAuthorNameAlternate() { return ( ! String.IsNullOrEmpty(AuthorNameAlternate)); }
+		public bool ShouldSerializeId() { return ( ! String.IsNullOrEmpty(UniqId)); }
+		// We almost always want to store the IsDeleted value, unless the reply is pretty much empty of any useful content.
+		public bool ShouldSerializeIsDeleted() { return IsDeleted || ShouldSerializeGuid() || ShouldSerializeContent() || ShouldSerializeId(); }
+	}
+}
+

src/LfMergeBridge/LfMergeModel/LfInputSystemRecord.cs

@@ -0,0 +1,18 @@
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	[BsonIgnoreExtraElements]
+	public class LfInputSystemRecord
+	{
+		public string Abbreviation { get; set; }
+		public string Tag { get; set; }
+		public string LanguageName { get; set; }
+		public bool IsRightToLeft { get; set; }
+
+		// We'll store vernacular / analysis writing system info when
+		// importing LCM projects, but LF won't be using this information
+		public bool VernacularWS { get; set; }
+		public bool AnalysisWS { get; set; }
+	}
+}

src/LfMergeBridge/LfMergeModel/LfOptionList.cs

@@ -0,0 +1,24 @@
+using MongoDB.Bson;
+using System;
+using System.Collections.Generic;
+
+namespace LfMergeBridge.LfMergeModel
+{
+	public class LfOptionList
+	{
+		public ObjectId Id { get; set; }
+		public string Name { get; set; }
+		public string Code { get; set; }
+		public DateTime DateCreated { get; set; }
+		public DateTime DateModified { get; set; }
+		public List<LfOptionListItem> Items { get; set; }
+		public string DefaultItemKey { get; set; }
+		public bool CanDelete { get; set; }
+
+		public LfOptionList()
+		{
+			Items = new List<LfOptionListItem>();
+		}
+	}
+}
+