From 8664ce9ae031933ec77501cdb3742936a55c87ef Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Wed, 17 Jul 2024 19:56:15 -0400 Subject: [PATCH 01/15] Prevent empty commits if files did not change Cousin of https://github.com/huggingface/huggingface_hub/pull/2389 --- packages/hub/package.json | 2 +- packages/hub/src/lib/commit.ts | 135 +++++++++++------- packages/hub/src/types/api/api-commit.ts | 5 + .../src/utils/{sha256-node.ts => sha-node.ts} | 21 ++- packages/hub/src/utils/sha1.ts | 31 ++++ packages/hub/src/utils/sha256.ts | 4 +- 6 files changed, 143 insertions(+), 55 deletions(-) rename packages/hub/src/utils/{sha256-node.ts => sha-node.ts} (61%) create mode 100644 packages/hub/src/utils/sha1.ts diff --git a/packages/hub/package.json b/packages/hub/package.json index 3015df1dc..0f08adb56 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -18,7 +18,7 @@ } }, "browser": { - "./src/utils/sha256-node.ts": false, + "./src/utils/sha-node.ts": false, "./src/utils/FileBlob.ts": false, "./dist/index.js": "./dist/browser/index.js", "./dist/index.mjs": "./dist/browser/index.mjs" diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index a66af74c1..66590a740 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -15,6 +15,7 @@ import { checkCredentials } from "../utils/checkCredentials"; import { chunk } from "../utils/chunk"; import { promisesQueue } from "../utils/promisesQueue"; import { promisesQueueStreaming } from "../utils/promisesQueueStreaming"; +import { sha1 } from "../utils/sha1"; import { sha256 } from "../utils/sha256"; import { toRepoId } from "../utils/toRepoId"; import { WebBlob } from "../utils/WebBlob"; @@ -125,8 +126,6 @@ export async function* commitIter(params: CommitParams): AsyncGenerator(); - const abortController = new AbortController(); const abortSignal = abortController.signal; @@ -168,6 +167,9 @@ export async function* commitIter(params: CommitParams): AsyncGenerator op.path === ".gitattributes")?.content; + const deleteOps = allOperations.filter((op): op is CommitDeletedEntry => !isFileOperation(op)); + const fileOpsToCommit: (CommitBlob & + ({ uploadMode: "lfs"; sha: string } | { uploadMode: "regular"; newSha: string | null }))[] = []; for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { const payload: ApiPreuploadRequest = { gitAttributes: gitAttributes && (await gitAttributes.text()), @@ -203,43 +205,50 @@ export async function* commitIter(params: CommitParams): AsyncGenerator( + (yieldCallback) => { + return promisesQueue( + json.files.map((file) => async () => { + const op = operations.find((op) => op.path === file.path); + if (!op) { + return; // this should never happen, server-side we always return one entry per operation + } + + const iterator = + file.uploadMode === "lfs" + ? sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }) + : sha1(op.content, { abortSignal }); + let res: IteratorResult; + do { + res = await iterator.next(); + if (!res.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res.value, state: "hashing" }); + } + } while (!res.done); + + if (res.value === null || res.value !== file.oid) { + fileOpsToCommit.push({ + ...op, + uploadMode: file.uploadMode, + sha: res.value, + } as (typeof fileOpsToCommit)[number]); + // ^^ we know `newSha` can only be `null` in the case of `sha1` as expected, but TS doesn't + } + }), + CONCURRENT_SHAS + ); } - } + ); + + abortSignal?.throwIfAborted(); } yield { event: "phase", phase: "uploadingLargeFiles" }; - for (const operations of chunk( - allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + for (const lfsOpsToCommit of chunk( + fileOpsToCommit.filter((op): op is CommitBlob & { uploadMode: "lfs"; sha: string } => op.uploadMode === "lfs"), 100 )) { - const shas = yield* eventToGenerator< - { event: "fileProgress"; state: "hashing"; path: string; progress: number }, - string[] - >((yieldCallback, returnCallback, rejectCallack) => { - return promisesQueue( - operations.map((op) => async () => { - const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal: abortSignal }); - let res: IteratorResult; - do { - res = await iterator.next(); - if (!res.done) { - yieldCallback({ event: "fileProgress", path: op.path, progress: res.value, state: "hashing" }); - } - } while (!res.done); - const sha = res.value; - lfsShas.set(op.path, res.value); - return sha; - }), - CONCURRENT_SHAS - ).then(returnCallback, rejectCallack); - }); - - abortSignal?.throwIfAborted(); - const payload: ApiLfsBatchRequest = { operation: "upload", // multipart is a custom protocol for HF @@ -250,8 +259,8 @@ export async function* commitIter(params: CommitParams): AsyncGenerator ({ - oid: shas[i], + objects: lfsOpsToCommit.map((op) => ({ + oid: op.sha, size: op.content.size, })), }; @@ -279,7 +288,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator [shas[i], op])); + const shaToOperation = new Map(lfsOpsToCommit.map((op) => [op.sha, op])); yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { return promisesQueueStreaming( @@ -293,9 +302,9 @@ export async function* commitIter(params: CommitParams): AsyncGenerator( async (yieldCallback, returnCallback, rejectCallback) => (params.fetch ?? fetch)( @@ -481,17 +515,16 @@ export async function* commitIter(params: CommitParams): AsyncGenerator { + allOpsToCommit.map((operation) => { if (isFileOperation(operation)) { - const sha = lfsShas.get(operation.path); - if (sha) { + if (operation.uploadMode === "lfs") { return { key: "lfsFile", value: { path: operation.path, algo: "sha256", size: operation.content.size, - oid: sha, + oid: operation.sha, } satisfies ApiCommitLfsFile, }; } @@ -509,8 +542,8 @@ export async function* commitIter(params: CommitParams): AsyncGenerator { // For now, we display equal progress for all files // We could compute the progress based on the size of `convertOperationToNdJson` for each of the files instead - for (const op of allOperations) { - if (isFileOperation(op) && !lfsShas.has(op.path)) { + for (const op of allOpsToCommit) { + if (isFileOperation(op) && op.uploadMode === "regular") { yieldCallback({ event: "fileProgress", path: op.path, diff --git a/packages/hub/src/types/api/api-commit.ts b/packages/hub/src/types/api/api-commit.ts index 404ad5494..38ce6c3e6 100644 --- a/packages/hub/src/types/api/api-commit.ts +++ b/packages/hub/src/types/api/api-commit.ts @@ -130,6 +130,11 @@ export interface ApiPreuploadResponse { files: Array<{ path: string; uploadMode: "lfs" | "regular"; + /** + * The oid of the blob if it already exists in the repository + * in case of blob is a lfs file, it'll be the lfs oid + */ + oid?: string; }>; } diff --git a/packages/hub/src/utils/sha256-node.ts b/packages/hub/src/utils/sha-node.ts similarity index 61% rename from packages/hub/src/utils/sha256-node.ts rename to packages/hub/src/utils/sha-node.ts index b068d1a21..68c40e8af 100644 --- a/packages/hub/src/utils/sha256-node.ts +++ b/packages/hub/src/utils/sha-node.ts @@ -8,7 +8,26 @@ export async function* sha256Node( abortSignal?: AbortSignal; } ): AsyncGenerator { - const sha256Stream = createHash("sha256"); + return yield* shaNode("sha256", buffer, opts); +} + +export async function* sha1Node( + buffer: ArrayBuffer | Blob, + opts?: { + abortSignal?: AbortSignal; + } +): AsyncGenerator { + return yield* shaNode("sha1", buffer, opts); +} + +async function* shaNode( + algorithm: "sha1" | "sha256", + buffer: ArrayBuffer | Blob, + opts?: { + abortSignal?: AbortSignal; + } +): AsyncGenerator { + const sha256Stream = createHash(algorithm); const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; let done = 0; const readable = diff --git a/packages/hub/src/utils/sha1.ts b/packages/hub/src/utils/sha1.ts new file mode 100644 index 000000000..de0062072 --- /dev/null +++ b/packages/hub/src/utils/sha1.ts @@ -0,0 +1,31 @@ +import { hexFromBytes } from "./hexFromBytes"; +import { isFrontend } from "./isFrontend"; + +export async function* sha1(buffer: Blob, opts?: { abortSignal?: AbortSignal }): AsyncGenerator { + yield 0; + + if (globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-1", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + + yield 1; + return res; + } + + if (isFrontend) { + yield 1; + return null; // unsupported if we're here + } + + if (!cryptoModule) { + cryptoModule = await import("./sha-node"); + } + + return yield* cryptoModule.sha1Node(buffer, { abortSignal: opts?.abortSignal }); +} + +// eslint-disable-next-line @typescript-eslint/consistent-type-imports +let cryptoModule: typeof import("./sha-node"); diff --git a/packages/hub/src/utils/sha256.ts b/packages/hub/src/utils/sha256.ts index d432909b3..6bc2a40ae 100644 --- a/packages/hub/src/utils/sha256.ts +++ b/packages/hub/src/utils/sha256.ts @@ -154,13 +154,13 @@ export async function* sha256( } if (!cryptoModule) { - cryptoModule = await import("./sha256-node"); + cryptoModule = await import("./sha-node"); } return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); } // eslint-disable-next-line @typescript-eslint/consistent-type-imports -let cryptoModule: typeof import("./sha256-node"); +let cryptoModule: typeof import("./sha-node"); // eslint-disable-next-line @typescript-eslint/consistent-type-imports let wasmModule: typeof import("../vendor/hash-wasm/sha256-wrapper"); From 6038ed59fa1d1d6a3988ef36a1444b264c4f7252 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Wed, 17 Jul 2024 20:00:51 -0400 Subject: [PATCH 02/15] forgot to rename this --- packages/hub/src/utils/sha-node.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/hub/src/utils/sha-node.ts b/packages/hub/src/utils/sha-node.ts index 68c40e8af..7eb3cac75 100644 --- a/packages/hub/src/utils/sha-node.ts +++ b/packages/hub/src/utils/sha-node.ts @@ -27,19 +27,19 @@ async function* shaNode( abortSignal?: AbortSignal; } ): AsyncGenerator { - const sha256Stream = createHash(algorithm); + const shaStream = createHash(algorithm); const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; let done = 0; const readable = buffer instanceof Blob ? Readable.fromWeb(buffer.stream() as ReadableStream) : Readable.from(Buffer.from(buffer)); for await (const buffer of readable) { - sha256Stream.update(buffer); + shaStream.update(buffer); done += buffer.length; yield done / size; opts?.abortSignal?.throwIfAborted(); } - return sha256Stream.digest("hex"); + return shaStream.digest("hex"); } From 01c49b95b00b6e1e2ad8a6a98c20294cb53c455e Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Fri, 19 Jul 2024 17:09:04 -0400 Subject: [PATCH 03/15] Correct sha1 --- packages/hub/src/lib/commit.ts | 3 ++- packages/hub/src/utils/sha1.ts | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index 66590a740..6618e4259 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -217,7 +217,8 @@ export async function* commitIter(params: CommitParams): AsyncGenerator; do { res = await iterator.next(); diff --git a/packages/hub/src/utils/sha1.ts b/packages/hub/src/utils/sha1.ts index de0062072..5e16fd5d7 100644 --- a/packages/hub/src/utils/sha1.ts +++ b/packages/hub/src/utils/sha1.ts @@ -6,9 +6,7 @@ export async function* sha1(buffer: Blob, opts?: { abortSignal?: AbortSignal }): if (globalThis.crypto?.subtle) { const res = hexFromBytes( - new Uint8Array( - await globalThis.crypto.subtle.digest("SHA-1", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) - ) + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-1", await buffer.arrayBuffer())) ); yield 1; From 1bb111030bc5a6974cda4e877d45a8130c10b676 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Fri, 19 Jul 2024 17:31:18 -0400 Subject: [PATCH 04/15] add test that may or may not work, in any case it's timing out for sure --- packages/hub/src/lib/commit.spec.ts | 93 ++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/packages/hub/src/lib/commit.spec.ts b/packages/hub/src/lib/commit.spec.ts index 28d46b924..bc4cc9b19 100644 --- a/packages/hub/src/lib/commit.spec.ts +++ b/packages/hub/src/lib/commit.spec.ts @@ -2,7 +2,7 @@ import { assert, it, describe } from "vitest"; import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; import type { RepoId } from "../types/public"; -import type { CommitFile } from "./commit"; +import type { CommitFile, CommitOperation } from "./commit"; import { commit } from "./commit"; import { createRepo } from "./create-repo"; import { deleteRepo } from "./delete-repo"; @@ -127,6 +127,97 @@ size ${lfsContent.length} } }, 60_000); + it("should not commit if nothing to add/update", async function () { + const tokenizerJsonUrl = new URL( + "https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json" + ); + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo: RepoId = { + name: repoName, + type: "model", + }; + + await createRepo({ + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + hubUrl: TEST_HUB_URL, + repo, + license: "mit", + }); + + try { + const readme1 = await downloadFile({ repo, path: "README.md", hubUrl: TEST_HUB_URL }); + assert.strictEqual(readme1?.status, 200); + + const nodeOperation: CommitFile[] = isFrontend + ? [] + : [ + { + operation: "addOrUpdate", + path: "tsconfig.json", + content: (await import("node:url")).pathToFileURL("./tsconfig.json") as URL, + }, + ]; + + const operations: CommitOperation[] = [ + { + operation: "addOrUpdate", + content: new Blob(["This is me"]), + path: "test.txt", + }, + { + operation: "addOrUpdate", + content: new Blob([lfsContent]), + path: "test.lfs.txt", + }, + ...nodeOperation, + { + operation: "addOrUpdate", + content: tokenizerJsonUrl, + path: "lamaral.json", + }, + ]; + + const firstOutput = await commit({ + repo, + title: "Preparation commit", + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + hubUrl: TEST_HUB_URL, + operations, + // To test web workers in the front-end + useWebWorkers: { minSize: 5_000 }, + }); + + const secondOutput = await commit({ + repo, + title: "Empty commit", + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + hubUrl: TEST_HUB_URL, + operations, + // To test web workers in the front-end + useWebWorkers: { minSize: 5_000 }, + }); + + assert.strictEqual(firstOutput.commit, secondOutput.commit); + assert.strictEqual(firstOutput.hookOutput, "Nothing to commit"); + + const currentRes: Response = await fetch(`${TEST_HUB_URL}/api/${repo.type}s/${repo.name}`); + const current = await currentRes.json(); + assert.strictEqual(firstOutput.commit.oid, current.sha); + } finally { + await deleteRepo({ + repo, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + } + }, 60_000); + it("should commit a full repo from HF with web urls", async function () { const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; const repo: RepoId = { From 57ffa9dd6b5ed0af6b3af3624ab160d801e4f018 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Fri, 19 Jul 2024 17:32:25 -0400 Subject: [PATCH 05/15] a bit better --- packages/hub/src/lib/commit.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/commit.spec.ts b/packages/hub/src/lib/commit.spec.ts index bc4cc9b19..87a93cb05 100644 --- a/packages/hub/src/lib/commit.spec.ts +++ b/packages/hub/src/lib/commit.spec.ts @@ -204,11 +204,11 @@ size ${lfsContent.length} }); assert.strictEqual(firstOutput.commit, secondOutput.commit); - assert.strictEqual(firstOutput.hookOutput, "Nothing to commit"); + assert.strictEqual(secondOutput.hookOutput, "Nothing to commit"); const currentRes: Response = await fetch(`${TEST_HUB_URL}/api/${repo.type}s/${repo.name}`); const current = await currentRes.json(); - assert.strictEqual(firstOutput.commit.oid, current.sha); + assert.strictEqual(secondOutput.commit.oid, current.sha); } finally { await deleteRepo({ repo, From cd0597574884bf799ca8ef697b94e85e854da2fc Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 18:22:17 -0400 Subject: [PATCH 06/15] I was a bit too radical it seems --- packages/hub/src/lib/commit.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index 6618e4259..2b3780d86 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -205,8 +205,8 @@ export async function* commitIter(params: CommitParams): AsyncGenerator( - (yieldCallback) => { + yield* eventToGenerator<{ event: "fileProgress"; state: "hashing"; path: string; progress: number }, void[]>( + (yieldCallback, returnCallback, rejectCallack) => { return promisesQueue( json.files.map((file) => async () => { const op = operations.find((op) => op.path === file.path); @@ -237,7 +237,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator Date: Tue, 23 Jul 2024 18:53:43 -0400 Subject: [PATCH 07/15] disappointing --- packages/hub/src/lib/commit.spec.ts | 8 ++++---- packages/hub/src/lib/commit.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/hub/src/lib/commit.spec.ts b/packages/hub/src/lib/commit.spec.ts index 87a93cb05..f046339af 100644 --- a/packages/hub/src/lib/commit.spec.ts +++ b/packages/hub/src/lib/commit.spec.ts @@ -13,8 +13,8 @@ import { isFrontend } from "../utils/isFrontend"; const lfsContent = "O123456789".repeat(100_000); -describe("commit", () => { - it("should commit to a repo with blobs", async function () { +describe.only("commit", () => { + it.skip("should commit to a repo with blobs", async function () { const tokenizerJsonUrl = new URL( "https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json" ); @@ -203,7 +203,7 @@ size ${lfsContent.length} useWebWorkers: { minSize: 5_000 }, }); - assert.strictEqual(firstOutput.commit, secondOutput.commit); + assert.deepEqual(firstOutput.commit, secondOutput.commit); assert.strictEqual(secondOutput.hookOutput, "Nothing to commit"); const currentRes: Response = await fetch(`${TEST_HUB_URL}/api/${repo.type}s/${repo.name}`); @@ -218,7 +218,7 @@ size ${lfsContent.length} } }, 60_000); - it("should commit a full repo from HF with web urls", async function () { + it.skip("should commit a full repo from HF with web urls", async function () { const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; const repo: RepoId = { name: repoName, diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index 2b3780d86..323671c25 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -217,7 +217,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator; do { From be22d1a2a94a4148f688c5e36d7a6014226e5b96 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 18:54:39 -0400 Subject: [PATCH 08/15] oopsie --- packages/hub/src/lib/commit.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/hub/src/lib/commit.spec.ts b/packages/hub/src/lib/commit.spec.ts index f046339af..976cd257b 100644 --- a/packages/hub/src/lib/commit.spec.ts +++ b/packages/hub/src/lib/commit.spec.ts @@ -13,8 +13,8 @@ import { isFrontend } from "../utils/isFrontend"; const lfsContent = "O123456789".repeat(100_000); -describe.only("commit", () => { - it.skip("should commit to a repo with blobs", async function () { +describe("commit", () => { + it("should commit to a repo with blobs", async function () { const tokenizerJsonUrl = new URL( "https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json" ); @@ -218,7 +218,7 @@ size ${lfsContent.length} } }, 60_000); - it.skip("should commit a full repo from HF with web urls", async function () { + it("should commit a full repo from HF with web urls", async function () { const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; const repo: RepoId = { name: repoName, From 234fef2c56672e6fdb9f06b8ce9892201bc702d2 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 18:57:27 -0400 Subject: [PATCH 09/15] the order changed a bit --- packages/hub/src/lib/upload-files-with-progress.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/hub/src/lib/upload-files-with-progress.spec.ts b/packages/hub/src/lib/upload-files-with-progress.spec.ts index bc082cd89..9e3281e3c 100644 --- a/packages/hub/src/lib/upload-files-with-progress.spec.ts +++ b/packages/hub/src/lib/upload-files-with-progress.spec.ts @@ -76,10 +76,6 @@ describe("uploadFilesWithProgress", () => { event: "phase", phase: "preuploading", }, - { - event: "phase", - phase: "uploadingLargeFiles", - }, { event: "fileProgress", path: "test.lfs.txt", @@ -104,6 +100,10 @@ describe("uploadFilesWithProgress", () => { progress: 1, state: "uploading", }, + { + event: "phase", + phase: "uploadingLargeFiles", + }, { event: "phase", phase: "committing", From 491ff3a2c48f3164f378a8e18c93a5a52f0c3983 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 18:57:54 -0400 Subject: [PATCH 10/15] like that rather --- packages/hub/src/lib/upload-files-with-progress.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/hub/src/lib/upload-files-with-progress.spec.ts b/packages/hub/src/lib/upload-files-with-progress.spec.ts index 9e3281e3c..f50d7ac43 100644 --- a/packages/hub/src/lib/upload-files-with-progress.spec.ts +++ b/packages/hub/src/lib/upload-files-with-progress.spec.ts @@ -88,6 +88,10 @@ describe("uploadFilesWithProgress", () => { progress: 1, state: "hashing", }, + { + event: "phase", + phase: "uploadingLargeFiles", + }, { event: "fileProgress", path: "test.lfs.txt", @@ -100,10 +104,6 @@ describe("uploadFilesWithProgress", () => { progress: 1, state: "uploading", }, - { - event: "phase", - phase: "uploadingLargeFiles", - }, { event: "phase", phase: "committing", From a7aafe06f22a3a33add8dc7b979d8a6e3275e8f7 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 19:03:07 -0400 Subject: [PATCH 11/15] better --- packages/hub/src/lib/commit.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/src/lib/commit.spec.ts b/packages/hub/src/lib/commit.spec.ts index 976cd257b..de4b2810d 100644 --- a/packages/hub/src/lib/commit.spec.ts +++ b/packages/hub/src/lib/commit.spec.ts @@ -203,7 +203,7 @@ size ${lfsContent.length} useWebWorkers: { minSize: 5_000 }, }); - assert.deepEqual(firstOutput.commit, secondOutput.commit); + assert.deepStrictEqual(firstOutput.commit, secondOutput.commit); assert.strictEqual(secondOutput.hookOutput, "Nothing to commit"); const currentRes: Response = await fetch(`${TEST_HUB_URL}/api/${repo.type}s/${repo.name}`); From 38030e7d5fa277647f65b0affd757e8be78559fc Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Tue, 23 Jul 2024 19:12:41 -0400 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=A5=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/upload-files-with-progress.spec.ts | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/packages/hub/src/lib/upload-files-with-progress.spec.ts b/packages/hub/src/lib/upload-files-with-progress.spec.ts index f50d7ac43..b0d619f5b 100644 --- a/packages/hub/src/lib/upload-files-with-progress.spec.ts +++ b/packages/hub/src/lib/upload-files-with-progress.spec.ts @@ -70,43 +70,72 @@ describe("uploadFilesWithProgress", () => { // assert(intermediateUploadEvents.length > 0, "There should be at least one intermediate upload event"); // } progressEvents = progressEvents.filter((e) => e.event !== "fileProgress" || e.progress === 0 || e.progress === 1); + const hashingEvents = progressEvents.splice(1, 6); assert.deepStrictEqual(progressEvents, [ { event: "phase", phase: "preuploading", }, + { + event: "phase", + phase: "uploadingLargeFiles", + }, { event: "fileProgress", path: "test.lfs.txt", progress: 0, - state: "hashing", + state: "uploading", }, { event: "fileProgress", path: "test.lfs.txt", progress: 1, - state: "hashing", + state: "uploading", }, { event: "phase", - phase: "uploadingLargeFiles", + phase: "committing", }, + ]); + + // order can very due to concurrent processing + assert.includeDeepMembers(hashingEvents, [ { event: "fileProgress", - path: "test.lfs.txt", + path: "file1", progress: 0, - state: "uploading", + state: "hashing", }, { event: "fileProgress", - path: "test.lfs.txt", + path: "file1", progress: 1, - state: "uploading", + state: "hashing", }, { - event: "phase", - phase: "committing", + event: "fileProgress", + path: "config.json", + progress: 0, + state: "hashing", + }, + { + event: "fileProgress", + path: "config.json", + progress: 1, + state: "hashing", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 0, + state: "hashing", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 1, + state: "hashing", }, ]); From 6d07fec483ffd4373e44fe98a9d090fcb11bff5e Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Wed, 24 Jul 2024 16:18:22 -0400 Subject: [PATCH 13/15] Update packages/hub/src/types/api/api-commit.ts Co-authored-by: Eliott C. --- packages/hub/src/types/api/api-commit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/src/types/api/api-commit.ts b/packages/hub/src/types/api/api-commit.ts index 38ce6c3e6..15ced9b21 100644 --- a/packages/hub/src/types/api/api-commit.ts +++ b/packages/hub/src/types/api/api-commit.ts @@ -132,7 +132,7 @@ export interface ApiPreuploadResponse { uploadMode: "lfs" | "regular"; /** * The oid of the blob if it already exists in the repository - * in case of blob is a lfs file, it'll be the lfs oid + * in case of blob is a lfs file, it'll be the lfs file's sha256 */ oid?: string; }>; From 325d9170d7a25013a485c8103053c9f03eed08d2 Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Thu, 25 Jul 2024 17:18:58 -0400 Subject: [PATCH 14/15] get commit oid from preupload --- packages/hub/src/lib/commit.ts | 35 ++++++++++++++---------- packages/hub/src/types/api/api-commit.ts | 4 +++ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index 323671c25..d1575f8b4 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -167,6 +167,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator op.path === ".gitattributes")?.content; + let currentCommitOid: string | undefined; const deleteOps = allOperations.filter((op): op is CommitDeletedEntry => !isFileOperation(op)); const fileOpsToCommit: (CommitBlob & ({ uploadMode: "lfs"; sha: string } | { uploadMode: "regular"; newSha: string | null }))[] = []; @@ -204,6 +205,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator( (yieldCallback, returnCallback, rejectCallack) => { @@ -467,28 +469,31 @@ export async function* commitIter(params: CommitParams): AsyncGenerator Date: Thu, 25 Jul 2024 17:21:22 -0400 Subject: [PATCH 15/15] lint/format --- packages/hub/src/lib/commit.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/commit.ts b/packages/hub/src/lib/commit.ts index d1575f8b4..45ff66059 100644 --- a/packages/hub/src/lib/commit.ts +++ b/packages/hub/src/lib/commit.ts @@ -479,11 +479,11 @@ export async function* commitIter(params: CommitParams): AsyncGenerator