From 359c0b736cad8e4375d75af4f60e97e20057e7aa Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 30 Apr 2024 10:16:16 +0100 Subject: [PATCH] fix: filecoin test use blob (#1422) Filecoin api exported tests still using CARs instead of Blobs, which means https://github.com/w3s-project/w3infra/pull/360 need to have TestContentStore to use CARs to make these tests work https://github.com/w3s-project/w3infra/pull/360/files#diff-cd843762832d0767495ff56f3f8692a15262dce245db89690794a1316868e25aR99 But, we want to use Blobs already, so let's make the tests also want Blobs across Filecoin pipeline :) note that in practise both Blobs and CARs work, but for testing we are only writing one of them and is a CAR today https://github.com/w3s-project/w3up/blob/main/packages/filecoin-api/test/events/storefront.js#L42 --- packages/filecoin-api/package.json | 2 +- .../filecoin-api/test/context/receipts.js | 2 +- packages/filecoin-api/test/utils.js | 46 +++++-------------- packages/filecoin-client/package.json | 1 - packages/filecoin-client/test/helpers/car.js | 22 --------- .../filecoin-client/test/helpers/random.js | 37 +++++++-------- pnpm-lock.yaml | 13 +++--- 7 files changed, 36 insertions(+), 87 deletions(-) delete mode 100644 packages/filecoin-client/test/helpers/car.js diff --git a/packages/filecoin-api/package.json b/packages/filecoin-api/package.json index 89c703237..8619f8f29 100644 --- a/packages/filecoin-api/package.json +++ b/packages/filecoin-api/package.json @@ -165,7 +165,7 @@ "p-map": "^6.0.0" }, "devDependencies": { - "@ipld/car": "^5.1.1", + "@types/assert": "^1.5.10", "@types/mocha": "^10.0.1", "@ucanto/client": "^9.0.1", "@ucanto/principal": "^9.0.1", diff --git a/packages/filecoin-api/test/context/receipts.js b/packages/filecoin-api/test/context/receipts.js index 1d3176562..d380827d7 100644 --- a/packages/filecoin-api/test/context/receipts.js +++ b/packages/filecoin-api/test/context/receipts.js @@ -13,7 +13,7 @@ import * as API from '../../src/types.js' * @param {API.PieceLink} context.aggregate * @param {string} context.group * @param {API.PieceLink} context.piece - * @param {API.CARLink} context.content + * @param {import('@ucanto/interface').UnknownLink} context.content * @param {import('@ucanto/interface').Block} context.piecesBlock * @param {API.InclusionProof} context.inclusionProof * @param {API.AggregateAcceptSuccess} context.aggregateAcceptStatus diff --git a/packages/filecoin-api/test/utils.js b/packages/filecoin-api/test/utils.js index ebb2dc26c..259d385dc 100644 --- a/packages/filecoin-api/test/utils.js +++ b/packages/filecoin-api/test/utils.js @@ -2,10 +2,7 @@ import { Aggregate, Piece } from '@web3-storage/data-segment' import { CID } from 'multiformats' import { webcrypto } from 'one-webcrypto' import { sha256 } from 'multiformats/hashes/sha2' -import * as CAR from '@ucanto/transport/car' import * as raw from 'multiformats/codecs/raw' -import { CarWriter } from '@ipld/car' -import { Blob } from '@web-std/blob' /** @param {number} size */ export async function randomBytes(size) { @@ -21,36 +18,15 @@ export async function randomBytes(size) { } /** @param {number} size */ -export async function randomCAR(size) { +export async function randomBlob(size) { const bytes = await randomBytes(size) const hash = await sha256.digest(bytes) - const root = CID.create(1, raw.code, hash) + const cid = CID.create(1, raw.code, hash) - const { writer, out } = CarWriter.create(root) - writer.put({ cid: root, bytes }) - writer.close() - - const chunks = [] - for await (const chunk of out) { - chunks.push(chunk) + return { + cid, + bytes, } - const blob = new Blob(chunks) - const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - - return Object.assign(blob, { cid, roots: [root], bytes }) -} - -/** - * @param {number} length - * @param {number} size - */ -export async function randomCARs(length, size) { - return ( - await Promise.all(Array.from({ length }).map(() => randomCAR(size))) - ).map((car) => ({ - link: car.cid, - size: car.size, - })) } /** @@ -58,20 +34,20 @@ export async function randomCARs(length, size) { * @param {number} size */ export async function randomCargo(length, size) { - const cars = await Promise.all( - Array.from({ length }).map(() => randomCAR(size)) + const blobs = await Promise.all( + Array.from({ length }).map(() => randomBlob(size)) ) - return cars.map((car) => { - const piece = Piece.fromPayload(car.bytes) + return blobs.map((blob) => { + const piece = Piece.fromPayload(blob.bytes) return { link: piece.link, height: piece.height, root: piece.root, - content: car.cid, + content: blob.cid, padding: piece.padding, - bytes: car.bytes, + bytes: blob.bytes, } }) } diff --git a/packages/filecoin-client/package.json b/packages/filecoin-client/package.json index a7e803e77..ff330222d 100644 --- a/packages/filecoin-client/package.json +++ b/packages/filecoin-client/package.json @@ -61,7 +61,6 @@ "@web3-storage/capabilities": "workspace:^" }, "devDependencies": { - "@ipld/car": "^5.1.1", "@ipld/dag-json": "^10.1.4", "@types/assert": "^1.5.6", "@types/mocha": "^10.0.1", diff --git a/packages/filecoin-client/test/helpers/car.js b/packages/filecoin-client/test/helpers/car.js deleted file mode 100644 index cc529313d..000000000 --- a/packages/filecoin-client/test/helpers/car.js +++ /dev/null @@ -1,22 +0,0 @@ -import { CarWriter } from '@ipld/car' -import * as CAR from '@ucanto/transport/car' -import { toBlock } from './block.js' - -/** - * @param {Uint8Array} bytes - */ -export async function toCAR(bytes) { - const block = await toBlock(bytes) - const { writer, out } = CarWriter.create(block.cid) - writer.put(block) - writer.close() - - const chunks = [] - for await (const chunk of out) { - chunks.push(chunk) - } - const blob = new Blob(chunks) - const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - - return Object.assign(blob, { cid, roots: [block.cid], bytes }) -} diff --git a/packages/filecoin-client/test/helpers/random.js b/packages/filecoin-client/test/helpers/random.js index 8aa31dd8a..1f9775c3f 100644 --- a/packages/filecoin-client/test/helpers/random.js +++ b/packages/filecoin-client/test/helpers/random.js @@ -1,6 +1,7 @@ import { Aggregate, Piece } from '@web3-storage/data-segment' - -import { toCAR } from './car.js' +import { sha256 } from 'multiformats/hashes/sha2' +import * as raw from 'multiformats/codecs/raw' +import { CID } from 'multiformats' /** @param {number} size */ export async function randomBytes(size) { @@ -9,6 +10,7 @@ export async function randomBytes(size) { const chunk = new Uint8Array(Math.min(size, 65_536)) if (!globalThis.crypto) { try { + // @ts-expect-error no types for webcrypto const { webcrypto } = await import('node:crypto') webcrypto.getRandomValues(chunk) } catch (error) { @@ -27,22 +29,15 @@ export async function randomBytes(size) { } /** @param {number} size */ -export async function randomCAR(size) { +export async function randomBlob(size) { const bytes = await randomBytes(size) - return toCAR(bytes) -} + const hash = await sha256.digest(bytes) + const cid = CID.create(1, raw.code, hash) -/** - * @param {number} length - * @param {number} size - */ -export async function randomCARs(length, size) { - return ( - await Promise.all(Array.from({ length }).map(() => randomCAR(size))) - ).map((car) => ({ - link: car.cid, - size: car.size, - })) + return { + cid, + bytes, + } } /** @@ -50,19 +45,19 @@ export async function randomCARs(length, size) { * @param {number} size */ export async function randomCargo(length, size) { - const cars = await Promise.all( - Array.from({ length }).map(() => randomCAR(size)) + const blobs = await Promise.all( + Array.from({ length }).map(() => randomBlob(size)) ) - return cars.map((car) => { - const piece = Piece.fromPayload(car.bytes) + return blobs.map((blob) => { + const piece = Piece.fromPayload(blob.bytes) return { link: piece.link, height: piece.height, root: piece.root, padding: piece.padding, - content: car.cid, + content: blob.cid, } }) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1ac87d4b..aa7e80f81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -273,9 +273,9 @@ importers: specifier: ^6.0.0 version: 6.0.0 devDependencies: - '@ipld/car': - specifier: ^5.1.1 - version: 5.2.4 + '@types/assert': + specifier: ^1.5.10 + version: 1.5.10 '@types/mocha': specifier: ^10.0.1 version: 10.0.4 @@ -328,9 +328,6 @@ importers: specifier: workspace:^ version: link:../capabilities devDependencies: - '@ipld/car': - specifier: ^5.1.1 - version: 5.2.4 '@ipld/dag-json': specifier: ^10.1.4 version: 10.1.5 @@ -3820,6 +3817,10 @@ packages: '@types/estree': 1.0.5 dev: true + /@types/assert@1.5.10: + resolution: {integrity: sha512-qEO+AUgYab7GVbeDDgUNCU3o0aZUoIMpNAe+w5LDbRxfxQX7vQAdDgwj1AroX+i8KaV56FWg0srXlSZROnsrIQ==} + dev: true + /@types/assert@1.5.9: resolution: {integrity: sha512-uQ+/hp0DbfrhnJFgiqHlsrLTGgr8HZGssV7vYN8XunlvyJkzArcyBy/9e5Rey2T7lpL7jVuoNI/8CWCDCwISgg==} dev: true