Add caching for popular parts of git repositories

Add infrastructure for apps to speed up loading artifacts from paths into git repositories

+ Implement a method using a git program from the environment to perform a partial clone of a repository.
+ Provide a global switch to enable a cache for partial git files by commit.
+ Add an HTTP server for zip archives of partial git clone by commit.

implement/Dockerfile

@@ -15,6 +15,10 @@ COPY --from=build-env /app/elm-fullstack/out /elm-fullstack/dotnet/
 # Build the process with a deployment for the default app.
 FROM binaries AS build-default-config
 
+# Support partial clone of git repositories: Install git as fallback implementation for cloning.
+RUN apt update
+RUN apt install -y git
+
 COPY ./example-apps/docker-image-default-app /docker-image-default-app/
 
 RUN dotnet "/elm-fullstack/dotnet/elm-fs.dll"  deploy  /docker-image-default-app/  /elm-fullstack/process-store  --init-app-state

implement/elm-fullstack/Pine/CacheByFileName.cs

@@ -0,0 +1,37 @@
+using System.IO;
+
+namespace Pine;
+
+public class CacheByFileName
+{
+    public string CacheDirectory { init; get; }
+
+    public byte[] GetOrUpdate(string fileName, System.Func<byte[]> getNew)
+    {
+        var cacheFilePath = Path.Combine(CacheDirectory, fileName);
+
+        if (File.Exists(cacheFilePath))
+        {
+            try
+            {
+                return File.ReadAllBytes(cacheFilePath);
+            }
+            catch { }
+        }
+
+        var file = getNew();
+
+        try
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(cacheFilePath));
+
+            File.WriteAllBytes(cacheFilePath, file);
+        }
+        catch (System.Exception e)
+        {
+            System.Console.WriteLine("Failed to write cache entry: " + e.ToString());
+        }
+
+        return file;
+    }
+}

implement/elm-fullstack/Pine/ExecutableFile.cs

@@ -135,7 +135,7 @@ string writeEnvironmentFile(KeyValuePair<IImmutableList<string>, IReadOnlyList<b
         {
             Directory.Delete(path: containerDirectory, recursive: true);
         }
-        // Avoid crash in scenario like https://forum.botengine.org/t/farm-manager-tribal-wars-2-farmbot/3038/170
+        // Avoid crash in scenario like https://forum.botlab.org/t/farm-manager-tribal-wars-2-farmbot/3038/170
         catch (UnauthorizedAccessException)
         {
         }

implement/elm-fullstack/Pine/Filesystem.cs

@@ -50,4 +50,33 @@ static public string CreateRandomDirectoryInTempDirectory()
 
     static public string MakePlatformSpecificPath(IImmutableList<string> path) =>
         string.Join(Path.DirectorySeparatorChar.ToString(), path);
+
+    /// <summary>
+    /// https://github.com/libgit2/libgit2sharp/issues/769#issuecomment-198833179
+    /// </summary>
+    static public void DeleteLocalDirectoryRecursive(string directoryPath)
+    {
+        if (!Directory.Exists(directoryPath))
+        {
+            return;
+        }
+
+        var files = Directory.GetFiles(directoryPath);
+        var directories = Directory.GetDirectories(directoryPath);
+
+        foreach (var file in files)
+        {
+            File.SetAttributes(file, FileAttributes.Normal);
+            File.Delete(file);
+        }
+
+        foreach (var dir in directories)
+        {
+            DeleteLocalDirectoryRecursive(dir);
+        }
+
+        File.SetAttributes(directoryPath, FileAttributes.Normal);
+
+        Directory.Delete(directoryPath, false);
+    }
 }

implement/elm-fullstack/Pine/GitPartialForCommitServer.cs

@@ -0,0 +1,90 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Pine;
+
+/// <summary>
+/// This server packages popular parts of git repositories into zip archives and caches them.
+/// 
+/// https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
+/// </summary>
+public class GitPartialForCommitServer
+{
+    static public string ZipArchivePathPrefix => "/git/partial-for-commit/zip/";
+
+    static public string ZipArchivePathFromCommit(string commit) => ZipArchivePathPrefix + commit;
+
+    static public Task Run(
+        IReadOnlyList<string> urls,
+        IReadOnlyList<string> gitCloneUrlPrefixes,
+        string fileCacheDirectory)
+    {
+        var builder = WebApplication.CreateBuilder();
+        var app = builder.Build();
+
+        app.Urls.Clear();
+        urls.ToList().ForEach(app.Urls.Add);
+
+        var fileCache = new CacheByFileName
+        {
+            CacheDirectory = Path.Combine(fileCacheDirectory, ZipArchivePathPrefix.TrimStart('/'))
+        };
+
+        /*
+         * https://docs.microsoft.com/en-us/aspnet/core/fundamentals/minimal-apis?view=aspnetcore-6.0
+         * */
+
+        app.MapGet(ZipArchivePathPrefix + "{commitId}", (string commitId, HttpRequest httpRequest) =>
+        {
+            using var bodyReader = new StreamReader(httpRequest.Body);
+
+            var bodyString = bodyReader.ReadToEndAsync().Result;
+
+            var cloneUrls = bodyString.Split(new[] { '\n', '\r' }, System.StringSplitOptions.RemoveEmptyEntries);
+
+            var supportedCloneUrls =
+                cloneUrls
+                .Where(c => gitCloneUrlPrefixes.Any(prefix => c.ToLowerInvariant().StartsWith(prefix.ToLowerInvariant())))
+                .ToImmutableList();
+
+            if (!cloneUrls.Any())
+            {
+                return Results.BadRequest("Missing clone URL. Use one line in the request body for each clone URL.");
+            }
+
+            if (!supportedCloneUrls.Any())
+            {
+                return Results.BadRequest(
+                    "None of the given clone URLs is enabled here. Only URLs with the following " +
+                    gitCloneUrlPrefixes.Count + " prefixes are supported: " + string.Join(", ", gitCloneUrlPrefixes));
+            }
+
+            byte[] loadWithFreshClone()
+            {
+                var files =
+                    LoadFromGitHubOrGitLab.GetRepositoryFilesPartialForCommitViaEnvironmentGitCheckout(
+                        cloneUrl: supportedCloneUrls[0],
+                        commit: commitId);
+
+                var zipArchive = ZipArchive.ZipArchiveFromEntries(files);
+
+                System.Console.WriteLine(
+                    "Cloned for commit " + commitId + ": Got " + files.Count + " files. Size of zip archive: " + zipArchive.Length + " bytes.");
+
+                return zipArchive;
+            }
+
+            return Results.Bytes(
+                contents: fileCache.GetOrUpdate(commitId, loadWithFreshClone),
+                contentType: "application/zip",
+                fileDownloadName: "git-partial-for-commit-" + commitId + ".zip");
+        });
+
+        return app.RunAsync();
+    }
+}