wip: Restructure Git sync

packages/backend/src/lib/git-sync/github/commit.ts

@@ -0,0 +1,54 @@
+import { commitChanges } from "./requests";
+import { GitSyncConfiguration } from "../integration";
+import { errors } from "#lib/errors";
+
+const commit: GitSyncConfiguration["commit"] = async ({
+  ctx,
+  gitData,
+  message,
+  additions,
+  deletions
+}) => {
+  if (!gitData.github) throw errors.notFound("githubData");
+
+  const octokit = await ctx.fastify.github.getInstallationOctokit(gitData?.github.installationId);
+  const { baseDirectory } = gitData.github!;
+  const result = await commitChanges({
+    githubData: gitData.github!,
+    octokit,
+    payload: {
+      additions: additions.map((addition, index) => {
+        return {
+          contents: Buffer.from(addition.contents).toString("base64"),
+          path: [...baseDirectory.split("/"), ...addition.path.split("/")].filter(Boolean).join("/")
+        };
+      }),
+      deletions: deletions.map((deletion) => {
+        return {
+          ...deletion,
+          path: [...baseDirectory.split("/"), ...deletion.path.split("/")].filter(Boolean).join("/")
+        };
+      }),
+      message,
+      expectedCommitId: gitData.lastCommitId!
+    }
+  });
+
+  if (!result) throw errors.serverError();
+
+  if (result.status === "stale-data") {
+    return {
+      status: "stale"
+    };
+  }
+
+  return {
+    commit: {
+      id: result.oid,
+      date: result.committedDate
+    },
+    status: "success"
+  };
+};
+
+export { commit };

packages/backend/src/lib/git-sync/github/get-records.ts

@@ -0,0 +1,12 @@
+import { GitSyncConfiguration } from "../integration";
+import { minimatch } from "minimatch";
+
+const getRecords: GitSyncConfiguration["getRecords"] = ({ gitData }) => {
+  if (!gitData.github) return [];
+
+  return gitData.records.filter((record) => {
+    return minimatch(record.path, gitData.github!.matchPattern);
+  });
+};
+
+export { getRecords };

packages/backend/src/lib/git-sync/github/get-transformer.ts

@@ -0,0 +1,9 @@
+import { GitSyncConfiguration } from "../integration";
+
+const getTransformer: GitSyncConfiguration["getTransformer"] = ({ gitData }) => {
+  if (!gitData.github) return "markdown";
+
+  return gitData.github.transformer;
+};
+
+export { getTransformer };

packages/backend/src/lib/git-sync/github/index.ts

@@ -0,0 +1,16 @@
+import { commit } from "./commit";
+import { initialSync } from "./initial-sync";
+import { getRecords } from "./get-records";
+import { pull } from "./pull";
+import { getTransformer } from "./get-transformer";
+import { createGitSyncIntegration } from "../integration";
+
+const useGitHubIntegration = createGitSyncIntegration({
+  getTransformer,
+  getRecords,
+  commit,
+  initialSync,
+  pull
+});
+
+export { useGitHubIntegration };

packages/backend/src/lib/git-sync/github/initial-sync.ts

@@ -0,0 +1,128 @@
+import { getDirectory, getLastCommit } from "./requests";
+import { createInputContentProcessor } from "../process-content";
+import { createSyncedPieces } from "../synced-pieces";
+import { GitSyncConfiguration } from "../integration";
+import { LexoRank } from "lexorank";
+import { minimatch } from "minimatch";
+import { ObjectId } from "mongodb";
+import {
+  FullContentGroup,
+  FullContentPiece,
+  FullContents,
+  GitRecord,
+  GitDirectory
+} from "#collections";
+import { errors } from "#lib/errors";
+import { UnderscoreID } from "#lib/mongo";
+
+const initialSync: GitSyncConfiguration["initialSync"] = async ({ ctx, gitData }) => {
+  if (!gitData?.github) throw errors.notFound("gitData");
+
+  const newContentGroups: UnderscoreID<FullContentGroup<ObjectId>>[] = [];
+  const newContentPieces: UnderscoreID<FullContentPiece<ObjectId>>[] = [];
+  const newContents: UnderscoreID<FullContents<ObjectId>>[] = [];
+  const newRecords: Array<GitRecord<ObjectId>> = [];
+  const newDirectories: Array<GitDirectory<ObjectId>> = [];
+  const octokit = await ctx.fastify.github.getInstallationOctokit(gitData?.github.installationId);
+  const { baseDirectory } = gitData.github;
+  const basePath = baseDirectory.startsWith("/") ? baseDirectory.slice(1) : baseDirectory;
+  const inputContentProcessor = await createInputContentProcessor(ctx, gitData.github.transformer);
+  const syncDirectory = async (
+    path: string,
+    ancestors: ObjectId[]
+  ): Promise<UnderscoreID<FullContentGroup<ObjectId>>> => {
+    const syncedPath = path.startsWith("/") ? path.slice(1) : path;
+    const recordPath = path.replace(basePath, "").split("/").filter(Boolean).join("/");
+    const entries = await getDirectory({
+      githubData: gitData.github!,
+      octokit,
+      payload: { path: syncedPath }
+    });
+    const name = recordPath.split("/").pop() || gitData.github?.repositoryName || "";
+    const contentGroupId = new ObjectId();
+    const descendants: ObjectId[] = [];
+    const createSyncedPiecesSource: Array<{
+      path: string;
+      content: string;
+      workspaceId: ObjectId;
+      contentGroupId: ObjectId;
+      order: string;
+    }> = [];
+
+    let order = LexoRank.min();
+
+    for await (const entry of entries) {
+      if (entry.type === "tree") {
+        const descendantContentGroup = await syncDirectory(
+          syncedPath.split("/").filter(Boolean).concat(entry.name).join("/"),
+          [...ancestors, contentGroupId]
+        );
+
+        descendants.push(descendantContentGroup._id);
+      } else if (
+        entry.type === "blob" &&
+        entry.object.text &&
+        minimatch(entry.name, gitData.github!.matchPattern)
+      ) {
+        createSyncedPiecesSource.push({
+          content: entry.object.text,
+          path: [...recordPath.split("/"), entry.name].filter(Boolean).join("/"),
+          workspaceId: ctx.auth.workspaceId,
+          contentGroupId,
+          order: order.toString()
+        });
+        order = order.genNext();
+      }
+    }
+
+    const syncedPieces = await createSyncedPieces(createSyncedPiecesSource, inputContentProcessor);
+
+    syncedPieces.forEach(({ contentPiece, content, contentHash }, index) => {
+      const { path } = createSyncedPiecesSource[index];
+
+      newContentPieces.push(contentPiece);
+      newContents.push(content);
+      newRecords.push({
+        contentPieceId: contentPiece._id,
+        currentHash: contentHash,
+        syncedHash: contentHash,
+        path
+      });
+    });
+
+    const contentGroup: UnderscoreID<FullContentGroup<ObjectId>> = {
+      _id: contentGroupId,
+      workspaceId: ctx.auth.workspaceId,
+      name,
+      ancestors,
+      descendants
+    };
+
+    newContentGroups.push(contentGroup);
+    newDirectories.push({
+      path: recordPath,
+      contentGroupId
+    });
+
+    return contentGroup;
+  };
+  const topContentGroup = await syncDirectory(basePath, []);
+  const latestGitHubCommit = await getLastCommit({ octokit, githubData: gitData.github! });
+
+  if (!latestGitHubCommit) throw errors.notFound("lastCommit");
+
+  return {
+    newContentGroups,
+    newContentPieces,
+    newContents,
+    newRecords,
+    newDirectories,
+    topContentGroup,
+    lastCommit: {
+      date: latestGitHubCommit.committedDate,
+      id: latestGitHubCommit.oid
+    }
+  };
+};
+
+export { initialSync };

packages/backend/src/lib/git-sync/github/pull.ts

@@ -0,0 +1,90 @@
+import { getCommitsSince, getFilesChangedInCommit, getDirectory } from "./requests";
+import { GitSyncConfiguration } from "../integration";
+import { minimatch } from "minimatch";
+import crypto from "node:crypto";
+import { errors } from "#lib/errors";
+
+const pull: GitSyncConfiguration["pull"] = async ({ ctx, gitData }) => {
+  if (!gitData.github) throw errors.notFound("githubData");
+
+  const octokit = await ctx.fastify.github.getInstallationOctokit(gitData?.github.installationId);
+  const changedRecordsByDirectory = new Map<
+    string,
+    Array<{ fileName: string; status: string; content?: string; hash: string }>
+  >();
+  const lastCommits = await getCommitsSince({
+    payload: { since: gitData.lastCommitDate! },
+    githubData: gitData.github!,
+    octokit
+  });
+  const { baseDirectory } = gitData.github!;
+  const basePath = baseDirectory.startsWith("/") ? baseDirectory.slice(1) : baseDirectory;
+
+  for await (const commit of lastCommits) {
+    const filesChangedInCommit = await getFilesChangedInCommit({
+      payload: { commitId: commit.oid },
+      githubData: gitData.github!,
+      octokit
+    });
+
+    filesChangedInCommit.forEach((file) => {
+      if (
+        !file.filename.startsWith(basePath) ||
+        !minimatch(file.filename, gitData.github!.matchPattern)
+      ) {
+        return;
+      }
+
+      const recordPath = file.filename.replace(basePath, "").split("/").filter(Boolean).join("/");
+      const directory = recordPath.split("/").slice(0, -1).join("/");
+      const fileName = recordPath.split("/").pop() || "";
+      const { status } = file;
+      const directoryRecords = changedRecordsByDirectory.get(directory) || [];
+      const existingRecordIndex = directoryRecords.findIndex(
+        (record) => record.fileName === fileName
+      );
+
+      if (existingRecordIndex === -1) {
+        directoryRecords.push({ fileName, status, hash: "" });
+      } else {
+        directoryRecords[existingRecordIndex].status = status;
+      }
+
+      changedRecordsByDirectory.set(directory, directoryRecords);
+    });
+  }
+
+  for await (const [directory, files] of changedRecordsByDirectory.entries()) {
+    const directoryEntries = await getDirectory({
+      githubData: gitData.github!,
+      octokit,
+      payload: {
+        path: [...basePath.split("/"), ...directory.split("/")].filter(Boolean).join("/")
+      }
+    });
+
+    for await (const entry of directoryEntries) {
+      const file = files.find((file) => file.fileName === entry.name);
+
+      if (entry.type === "blob" && file && entry.object.text) {
+        file.content = entry.object.text;
+        file.hash = crypto.createHash("md5").update(entry.object.text).digest("hex");
+      }
+    }
+  }
+
+  const lastCommit = lastCommits.at(-1) || {
+    committedDate: gitData.lastCommitDate || "",
+    oid: gitData.lastCommitId || ""
+  };
+
+  return {
+    changedRecordsByDirectory,
+    lastCommit: {
+      date: lastCommit.committedDate,
+      id: lastCommit.oid
+    }
+  };
+};
+
+export { pull };

packages/backend/src/lib/git-sync/index.ts

@@ -0,0 +1,21 @@
+import { useGitHubIntegration } from "./github";
+import { UseGitSyncIntegration } from "./integration";
+import { ObjectId } from "mongodb";
+import { FullGitData } from "#collections";
+import { AuthenticatedContext } from "#lib/middleware";
+import { UnderscoreID } from "#lib/mongo";
+
+const useGitSyncIntegration = (
+  ctx: AuthenticatedContext,
+  gitData: UnderscoreID<FullGitData<ObjectId>>
+): ReturnType<UseGitSyncIntegration> | null => {
+  if (gitData.github) {
+    return useGitHubIntegration(ctx, gitData);
+  }
+
+  return null;
+};
+
+export { useGitSyncIntegration };
+export * from "./process-content";
+export * from "./process-pulled-records";