From fd74cbb9a59c10f1688a5994dd2cdc6722be00be Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 18 Jun 2024 11:59:26 +0100 Subject: [PATCH] feat!: allow invocation configuration to be generated on demand (#1507) Since the blob protocol landed, it's become harder to obtain a delegation to upload a specific CAR. It was tricky before because of sharding, but assuming you created a CAR below the shard threshold, its easy enough to obtain a `store/add` delegation tied to the CAR CID. With the blob protocol, we now generate an index, and `blob/add` that as well. It means we can't just create a CAR and delegate `can: blob/add, nb: { link, size }` for the CAR, because we also need to delegate `blob/add` for the index, but we don't know the index CID or it's size up front... It's ok if the delegation is not restricted to the specific CAR, but for certain use cases it's necessary to delegate just what is needed. This PR alters the client to allow invocation configuration to be generated on demand, when the proofs are required. This is a backwards compatible change. So, for example instead of calling: `uploadFile({ issuer, with, proofs }, file)` you now call: ```js const configure = caps => { const proofs = [] for (const { can, nb } of caps) { // decide if want to delegate and create delegation proofs.push(...) } if (!proofs.length) throw new Error('no proofs available') return { issuer, with, proofs } } uploadFile(configure, file) ``` ...and `configure` will be called for each proof that is needed (`blob/add`, `index/add` and `upload/add`). It means also that we no longer have to recommend folks to create a CAR, get the CID and then upload it, they can just use `uploadFile` and pass in the invocation config function to obtain the correct delegations when required. --- packages/upload-client/README.md | 59 ++++- packages/upload-client/package.json | 12 +- .../src/{blob.js => blob/add.js} | 200 +++------------- packages/upload-client/src/blob/get.js | 60 +++++ packages/upload-client/src/blob/index.js | 4 + packages/upload-client/src/blob/list.js | 60 +++++ packages/upload-client/src/blob/remove.js | 60 +++++ packages/upload-client/src/index.js | 76 ++++-- .../src/{dag-index.js => index/add.js} | 24 +- packages/upload-client/src/index/index.js | 1 + packages/upload-client/src/types.ts | 21 +- packages/upload-client/src/upload.js | 222 ------------------ packages/upload-client/src/upload/add.js | 77 ++++++ packages/upload-client/src/upload/get.js | 73 ++++++ packages/upload-client/src/upload/index.js | 4 + packages/upload-client/src/upload/list.js | 62 +++++ packages/upload-client/src/upload/remove.js | 60 +++++ packages/upload-client/test/blob.test.js | 38 +-- packages/upload-client/test/dag-index.test.js | 2 +- packages/upload-client/test/helpers/utils.js | 2 +- packages/upload-client/test/index.test.js | 158 +++++++++++++ packages/upload-client/test/upload.test.js | 2 +- packages/w3up-client/src/capability/blob.js | 5 +- .../w3up-client/test/capability/blob.test.js | 18 +- .../w3up-client/test/capability/index.test.js | 6 +- 25 files changed, 837 insertions(+), 469 deletions(-) rename packages/upload-client/src/{blob.js => blob/add.js} (62%) create mode 100644 packages/upload-client/src/blob/get.js create mode 100644 packages/upload-client/src/blob/index.js create mode 100644 packages/upload-client/src/blob/list.js create mode 100644 packages/upload-client/src/blob/remove.js rename packages/upload-client/src/{dag-index.js => index/add.js} (70%) create mode 100644 packages/upload-client/src/index/index.js delete mode 100644 packages/upload-client/src/upload.js create mode 100644 packages/upload-client/src/upload/add.js create mode 100644 packages/upload-client/src/upload/get.js create mode 100644 packages/upload-client/src/upload/index.js create mode 100644 packages/upload-client/src/upload/list.js create mode 100644 packages/upload-client/src/upload/remove.js diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index f75a10b74..2ad1ad35d 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -180,6 +180,7 @@ await Upload.add(conf, rootCID, carCIDs) - [`CARMetadata`](#carmetadata) - [`DirectoryEntryLinkCallback`](#directoryentrylinkcallback) - [`InvocationConfig`](#invocationconfig) + - [`InvocationConfigurator`](#invocationconfigurator) - [`ShardStoredCallback`](#shardstoredcallback) - [Contributing](#contributing) - [License](#license) @@ -190,7 +191,7 @@ await Upload.add(conf, rootCID, carCIDs) ```ts function uploadDirectory( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, files: File[], options: { retries?: number @@ -207,13 +208,13 @@ Uploads a directory of files to the service and returns the root data CID for th Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig), [`ShardStoredCallback`](#shardstoredcallback) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator), [`ShardStoredCallback`](#shardstoredcallback) ### `uploadFile` ```ts function uploadFile( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, file: Blob, options: { retries?: number @@ -229,13 +230,13 @@ Uploads a file to the service and returns the root data CID for the generated DA Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator) ### `uploadCAR` ```ts function uploadCAR( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, car: Blob, options: { retries?: number @@ -252,7 +253,7 @@ Uploads a CAR file to the service. The difference between this function and [Blo Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig), [`ShardStoredCallback`](#shardstoredcallback) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator), [`ShardStoredCallback`](#shardstoredcallback) ### `Blob.add` @@ -512,6 +513,52 @@ This is the configuration for the UCAN invocation. It's values can be obtained f - The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. - The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. +### `InvocationConfigurator` + +A function that generates [invocation configuration](#invocationconfig) for the requested capabilities. The intention is for the client to be able to [request, on demand, delegated capabilities from an application server](https://github.com/storacha-network/w3up-examples/tree/main/delegated-upload). + +```ts +interface InvocationConfigurator { + (caps: CapabilityQuery[]): Await +} + +interface CapabilityQuery { + can: ServiceAbility + nb?: unknown +} + +// "space/blob/add", "space/index/add" etc. +type ServiceAbility = string +``` + +The function may be called multiple times with different requested capabilities. + +Example: + +```js +import { Agent } from '@web3-storage/access' +import * as Space from '@web3-storage/access/space' + +const agent = await Agent.create() +const space = await Space.generate({ name: 'myspace' }) + +const configure = async (caps) => ({ + issuer: agent.issuer, + with: space.did(), + proofs: [ + // delegate from the space to the agent the requested capabilities + await Delegation.delegate({ + issuer: space.signer, + audience: agent.did(), + capabilities: caps.map(c => ({ can: c.can, with: space.did(), nb: c.nb })), + expiration: Math.floor(Date.now() / 1000) + (60 * 60) // 1h in seconds + }) + ] +}) + +await uploadFile(configure, new Blob(['Hello World!'])) +``` + ### `ShardStoredCallback` A function called after a DAG shard has been successfully stored by the service: diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 63b2aac82..81be97f90 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -31,11 +31,12 @@ }, "exports": { ".": "./dist/src/index.js", - "./blob": "./dist/src/blob.js", + "./blob": "./dist/src/blob/index.js", "./car": "./dist/src/car.js", "./fetch-with-upload-progress": "./dist/src/fetch-with-upload-progress.js", + "./index": "./dist/src/index/index.js", "./sharding": "./dist/src/sharding.js", - "./upload": "./dist/src/upload.js", + "./upload": "./dist/src/upload/index.js", "./store": "./dist/src/store.js", "./unixfs": "./dist/src/unixfs.js", "./types": "./dist/src/types.js" @@ -43,16 +44,19 @@ "typesVersions": { "*": { "blob": [ - "dist/src/blob.d.ts" + "dist/src/blob/index.d.ts" ], "car": [ "dist/src/car.d.ts" ], + "index": [ + "dist/src/index/index.d.ts" + ], "sharding": [ "dist/src/sharding.d.ts" ], "upload": [ - "dist/src/upload.d.ts" + "dist/src/upload/index.d.ts" ], "store": [ "dist/src/store.d.ts" diff --git a/packages/upload-client/src/blob.js b/packages/upload-client/src/blob/add.js similarity index 62% rename from packages/upload-client/src/blob.js rename to packages/upload-client/src/blob/add.js index b9adda959..493c4a4f3 100644 --- a/packages/upload-client/src/blob.js +++ b/packages/upload-client/src/blob/add.js @@ -1,4 +1,3 @@ -import { sha256 } from 'multiformats/hashes/sha2' import { ed25519 } from '@ucanto/principal' import { conclude } from '@web3-storage/capabilities/ucan' import * as UCAN from '@web3-storage/capabilities/ucan' @@ -8,20 +7,17 @@ import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as HTTPCapabilities from '@web3-storage/capabilities/http' import { SpaceDID } from '@web3-storage/capabilities/utils' import retry, { AbortError } from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' -import { poll } from './receipts.js' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' +import { poll } from '../receipts.js' /** * @param {string} url - * @param {import('./types.js').ProgressFn} handler + * @param {import('../types.js').ProgressFn} handler */ function createUploadProgressHandler(url, handler) { - /** - * - * @param {import('./types.js').ProgressStatus} status - */ - function onUploadProgress({ total, loaded, lengthComputable }) { + /** @param {import('../types.js').ProgressStatus} status */ + const onUploadProgress = ({ total, loaded, lengthComputable }) => { return handler({ total, loaded, lengthComputable, url }) } return onUploadProgress @@ -74,7 +70,7 @@ function parseBlobAddReceiptNext(receipt) { // Decode receipts available const nextReceipts = concludefxs.map((fx) => getConcludeReceipt(fx)) - /** @type {import('@ucanto/interface').Receipt | undefined} */ + /** @type {import('@ucanto/interface').Receipt | undefined} */ // @ts-expect-error types unknown for next const allocateReceipt = nextReceipts.find((receipt) => receipt.ran.link().equals(allocateTask.cid) @@ -85,7 +81,7 @@ function parseBlobAddReceiptNext(receipt) { receipt.ran.link().equals(putTask.cid) ) - /** @type {import('@ucanto/interface').Receipt | undefined} */ + /** @type {import('@ucanto/interface').Receipt | undefined} */ // @ts-expect-error types unknown for next const acceptReceipt = nextReceipts.find((receipt) => receipt.ran.link().equals(acceptTask.cid) @@ -115,7 +111,7 @@ function parseBlobAddReceiptNext(receipt) { // FIXME this code has been copied over from upload-api /** * @param {import('@ucanto/interface').Signer} id - * @param {import('@ucanto/interface').Verifier} serviceDid + * @param {import('@ucanto/interface').Principal} serviceDid * @param {import('@ucanto/interface').Receipt} receipt */ export function createConcludeInvocation(id, serviceDid, receipt) { @@ -152,7 +148,7 @@ export function createConcludeInvocation(id, serviceDid, receipt) { * * Required delegated capability proofs: `blob/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration + * @param {import('../types.js').InvocationConfig} conf Configuration * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. * * The `issuer` is the signing authority that is issuing the UCAN @@ -165,19 +161,20 @@ export function createConcludeInvocation(id, serviceDid, receipt) { * has the capability to perform the action. * * The issuer needs the `blob/add` delegated capability. + * @param {import('multiformats').MultihashDigest} digest * @param {Blob|Uint8Array} data Blob data. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} */ export async function add( { issuer, with: resource, proofs, audience }, + digest, data, options = {} ) { /* c8 ignore next 2 */ const bytes = data instanceof Uint8Array ? data : new Uint8Array(await data.arrayBuffer()) - const multihash = await sha256.digest(bytes) const size = bytes.length /* c8 ignore next */ const conn = options.connection ?? connection @@ -190,12 +187,7 @@ export async function add( /* c8 ignore next */ audience: audience ?? servicePrincipal, with: SpaceDID.from(resource), - nb: { - blob: { - digest: multihash.bytes, - size, - }, - }, + nb: input(digest, size), proofs, nonce: options.nonce, }) @@ -293,8 +285,8 @@ export async function add( }) const httpPutConcludeInvocation = createConcludeInvocation( issuer, - // @ts-expect-error object of type unknown - audience, + /* c8 ignore next */ + audience ?? servicePrincipal, httpPutReceipt ) const ucanConclude = await httpPutConcludeInvocation.execute(conn) @@ -324,155 +316,21 @@ export async function add( blocks, }) - return { - multihash, - site, - } + return { site } } -/** - * List Blobs stored in the space. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/list` delegated capability. - * @param {import('./types.js').ListRequestOptions} [options] - * @returns {Promise} - */ -export async function list( - { issuer, with: resource, proofs, audience }, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.list - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - proofs, - nb: { - cursor: options.cursor, - size: options.size, - }, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.list.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Remove a stored Blob file by digest. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/remove` delegated capability. - * @param {import('multiformats').MultihashDigest} multihash of the blob - * @param {import('./types.js').RequestOptions} [options] - */ -export async function remove( - { issuer, with: resource, proofs, audience }, - multihash, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.remove - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { - digest: multihash.bytes, - }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.remove.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out -} +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.add.can /** - * Gets a stored Blob file by digest. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * Returns required input to the invocation. * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/get/0/1` delegated capability. - * @param {import('multiformats').MultihashDigest} multihash of the blob - * @param {import('./types.js').RequestOptions} [options] + * @param {import('multiformats').MultihashDigest} digest + * @param {number} size */ -export async function get( - { issuer, with: resource, proofs, audience }, - multihash, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { - digest: multihash.bytes, - }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.get.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out -} +export const input = (digest, size) => ({ + blob: { + digest: digest.bytes, + size, + }, +}) diff --git a/packages/upload-client/src/blob/get.js b/packages/upload-client/src/blob/get.js new file mode 100644 index 000000000..5287922a1 --- /dev/null +++ b/packages/upload-client/src/blob/get.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Gets a stored Blob file by digest. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/get/0/1` delegated capability. + * @param {import('multiformats').MultihashDigest} multihash of the blob + * @param {import('../types.js').RequestOptions} [options] + */ +export async function get( + { issuer, with: resource, proofs, audience }, + multihash, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.get + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(multihash), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.get.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.get.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').MultihashDigest} digest + */ +export const input = (digest) => ({ digest: digest.bytes }) diff --git a/packages/upload-client/src/blob/index.js b/packages/upload-client/src/blob/index.js new file mode 100644 index 000000000..f278e2c20 --- /dev/null +++ b/packages/upload-client/src/blob/index.js @@ -0,0 +1,4 @@ +export { add } from './add.js' +export { get } from './get.js' +export { list } from './list.js' +export { remove } from './remove.js' diff --git a/packages/upload-client/src/blob/list.js b/packages/upload-client/src/blob/list.js new file mode 100644 index 000000000..eaef942d3 --- /dev/null +++ b/packages/upload-client/src/blob/list.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * List Blobs stored in the space. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/list` delegated capability. + * @param {import('../types.js').ListRequestOptions} [options] + * @returns {Promise} + */ +export async function list( + { issuer, with: resource, proofs, audience }, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.list + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + proofs, + nb: input(options.cursor, options.size), + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.list.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.list.can + +/** + * Returns required input to the invocation. + * + * @param {string} [cursor] + * @param {number} [size] + */ +export const input = (cursor, size) => ({ cursor, size }) diff --git a/packages/upload-client/src/blob/remove.js b/packages/upload-client/src/blob/remove.js new file mode 100644 index 000000000..6ecb4db10 --- /dev/null +++ b/packages/upload-client/src/blob/remove.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Remove a stored Blob file by digest. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/remove` delegated capability. + * @param {import('multiformats').MultihashDigest} multihash of the blob + * @param {import('../types.js').RequestOptions} [options] + */ +export async function remove( + { issuer, with: resource, proofs, audience }, + multihash, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.remove + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(multihash), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.remove.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.remove.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').MultihashDigest} digest + */ +export const input = (digest) => ({ digest: digest.bytes }) diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 77aa3f666..f8dbbd35d 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -2,10 +2,14 @@ import * as PieceHasher from '@web3-storage/data-segment/multihash' import { Storefront } from '@web3-storage/filecoin-client' import * as Link from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' import * as Store from './store.js' -import * as Blob from './blob.js' -import * as Index from './dag-index.js' -import * as Upload from './upload.js' +import * as Blob from './blob/index.js' +import * as BlobAdd from './blob/add.js' +import * as Index from './index/index.js' +import * as IndexAdd from './index/add.js' +import * as Upload from './upload/index.js' +import * as UploadAdd from './upload/add.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' @@ -23,8 +27,9 @@ export * as Receipt from './receipts.js' * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object. * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -56,8 +61,9 @@ export async function uploadFile(conf, file, options = {}) { * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -97,8 +103,9 @@ export async function uploadDirectory(conf, files, options = {}) { * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -120,7 +127,7 @@ export async function uploadCAR(conf, car, options = {}) { } /** - * @param {import('./types.js').InvocationConfig} conf + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf * @param {ReadableStream} blocks * @param {import('./types.js').UploadOptions} [options] * @returns {Promise} @@ -130,6 +137,8 @@ async function uploadBlockStream( blocks, { pieceHasher = PieceHasher, ...options } = {} ) { + /** @type {import('./types.js').InvocationConfigurator} */ + const configure = typeof conf === 'function' ? conf : () => conf /** @type {Array>} */ const shardIndexes = [] /** @type {import('./types.js').CARLink[]} */ @@ -144,16 +153,23 @@ async function uploadBlockStream( new TransformStream({ async transform(car, controller) { const bytes = new Uint8Array(await car.arrayBuffer()) + const digest = await sha256.digest(bytes) + const conf = await configure([ + { + can: BlobAdd.ability, + nb: BlobAdd.input(digest, bytes.length), + }, + ]) // Invoke blob/add and write bytes to write target - const { multihash } = await Blob.add(conf, bytes, options) - // Should this be raw instead? - const cid = Link.create(CAR.code, multihash) + await Blob.add(conf, digest, bytes, options) + const cid = Link.create(CAR.code, digest) + let piece if (pieceHasher) { const multihashDigest = await pieceHasher.digest(bytes) /** @type {import('@web3-storage/capabilities/types').PieceLink} */ piece = Link.create(raw.code, multihashDigest) - const content = Link.create(raw.code, multihash) + const content = Link.create(raw.code, digest) // Invoke filecoin/offer for data const result = await Storefront.filecoinOffer( @@ -206,14 +222,36 @@ async function uploadBlockStream( throw new Error('failed to archive DAG index', { cause: indexBytes.error }) } - // Store the index in the space - const { multihash } = await Blob.add(conf, indexBytes.ok, options) - const indexLink = Link.create(CAR.code, multihash) + const indexDigest = await sha256.digest(indexBytes.ok) + const indexLink = Link.create(CAR.code, indexDigest) + const [blobAddConf, indexAddConf, uploadAddConf] = await Promise.all([ + configure([ + { + can: BlobAdd.ability, + nb: BlobAdd.input(indexDigest, indexBytes.ok.length), + }, + ]), + configure([ + { + can: IndexAdd.ability, + nb: IndexAdd.input(indexLink), + }, + ]), + configure([ + { + can: UploadAdd.ability, + nb: UploadAdd.input(root, shards), + }, + ]), + ]) + + // Store the index in the space + await Blob.add(blobAddConf, indexDigest, indexBytes.ok, options) // Register the index with the service - await Index.add(conf, indexLink, options) + await Index.add(indexAddConf, indexLink, options) // Register an upload with the service - await Upload.add(conf, root, shards, options) + await Upload.add(uploadAddConf, root, shards, options) return root } diff --git a/packages/upload-client/src/dag-index.js b/packages/upload-client/src/index/add.js similarity index 70% rename from packages/upload-client/src/dag-index.js rename to packages/upload-client/src/index/add.js index 572d8c046..662040b43 100644 --- a/packages/upload-client/src/dag-index.js +++ b/packages/upload-client/src/index/add.js @@ -1,8 +1,8 @@ import * as IndexCapabilities from '@web3-storage/capabilities/index' import { SpaceDID } from '@web3-storage/capabilities/utils' import retry from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' /** * Register an "index" with the service. The issuer needs the `index/add` @@ -10,7 +10,7 @@ import { REQUEST_RETRIES } from './constants.js' * * Required delegated capability proofs: `index/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration + * @param {import('../types.js').InvocationConfig} conf Configuration * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. * * The `issuer` is the signing authority that is issuing the UCAN @@ -23,9 +23,9 @@ import { REQUEST_RETRIES } from './constants.js' * has the capability to perform the action. * * The issuer needs the `index/add` delegated capability. - * @param {import('./types.js').CARLink} index Index to store. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} + * @param {import('../types.js').CARLink} index Index to store. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} */ export async function add( { issuer, with: resource, proofs, audience }, @@ -42,7 +42,7 @@ export async function add( /* c8 ignore next */ audience: audience ?? servicePrincipal, with: SpaceDID.from(resource), - nb: { index }, + nb: input(index), proofs, }) .execute(conn) @@ -61,3 +61,13 @@ export async function add( return result.out.ok } + +/** Returns the ability used by an invocation. */ +export const ability = IndexCapabilities.add.can + +/** + * Returns required input to the invocation. + * + * @param {import('../types.js').CARLink} index + */ +export const input = (index) => ({ index }) diff --git a/packages/upload-client/src/index/index.js b/packages/upload-client/src/index/index.js new file mode 100644 index 000000000..6448ab66f --- /dev/null +++ b/packages/upload-client/src/index/index.js @@ -0,0 +1 @@ +export { add } from './add.js' diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 31cfe2a60..408e069df 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -2,13 +2,7 @@ import type { FetchOptions as IpfsUtilsFetchOptions, ProgressStatus as XHRProgressStatus, } from 'ipfs-utils/src/types.js' -import { - MultihashDigest, - Link, - UnknownLink, - Version, - MultihashHasher, -} from 'multiformats' +import { Link, UnknownLink, Version, MultihashHasher } from 'multiformats' import { Block } from '@ipld/unixfs' import { ServiceMethod, @@ -20,6 +14,7 @@ import { Failure, Delegation, Capabilities, + Await, } from '@ucanto/interface' import { UCANConclude, @@ -74,6 +69,7 @@ import { UsageReport, UsageReportSuccess, UsageReportFailure, + ServiceAbility, } from '@web3-storage/capabilities/types' import { StorefrontService } from '@web3-storage/filecoin-client/storefront' import { code as pieceHashCode } from '@web3-storage/data-segment/multihash' @@ -213,6 +209,16 @@ export interface InvocationConfig { proofs: Proof[] } +export interface CapabilityQuery { + can: ServiceAbility + nb?: unknown +} + +/** Generates invocation configuration for the requested capabilities. */ +export interface InvocationConfigurator { + (caps: CapabilityQuery[]): Await +} + export interface UnixFSEncodeResult { /** * Root CID for the DAG. @@ -396,6 +402,5 @@ export interface FileLike extends BlobLike { } export interface BlobAddOk { - multihash: MultihashDigest site: Delegation } diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js deleted file mode 100644 index 7322c8c72..000000000 --- a/packages/upload-client/src/upload.js +++ /dev/null @@ -1,222 +0,0 @@ -import * as UploadCapabilities from '@web3-storage/capabilities/upload' -import { SpaceDID } from '@web3-storage/capabilities/utils' -import retry from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' - -/** - * Register an "upload" with the service. The issuer needs the `upload/add` - * delegated capability. - * - * Required delegated capability proofs: `upload/add` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/add` delegated capability. - * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. - * @param {import('./types.js').CARLink[]} shards CIDs of CAR files that contain the DAG. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function add( - { issuer, with: resource, proofs, audience }, - root, - shards, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await UploadCapabilities.add - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root, shards }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.add.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Get details of an "upload". - * - * Required delegated capability proofs: `upload/get` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/get` delegated capability. - * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function get( - { issuer, with: resource, proofs, audience }, - root, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await UploadCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.get.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * List uploads created by the issuer. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/list` delegated capability. - * @param {import('./types.js').ListRequestOptions} [options] - * @returns {Promise} - */ -export async function list( - { issuer, with: resource, proofs, audience }, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - const result = await UploadCapabilities.list - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - proofs, - nb: { - cursor: options.cursor, - size: options.size, - pre: options.pre, - }, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.list.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Remove an upload by root data CID. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/remove` delegated capability. - * @param {import('multiformats').UnknownLink} root Root data CID to remove. - * @param {import('./types.js').RequestOptions} [options] - */ -export async function remove( - { issuer, with: resource, proofs, audience }, - root, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await UploadCapabilities.remove - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.remove.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} diff --git a/packages/upload-client/src/upload/add.js b/packages/upload-client/src/upload/add.js new file mode 100644 index 000000000..f526ba653 --- /dev/null +++ b/packages/upload-client/src/upload/add.js @@ -0,0 +1,77 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import retry from 'p-retry' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' + +/** + * Register an "upload" with the service. The issuer needs the `upload/add` + * delegated capability. + * + * Required delegated capability proofs: `upload/add` + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/add` delegated capability. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('../types.js').CARLink[]} shards CIDs of CAR files that contain the DAG. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} + */ +export async function add( + { issuer, with: resource, proofs, audience }, + root, + shards, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await retry( + async () => { + return await UploadCapabilities.add + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root, shards), + proofs, + nonce: options.nonce, + }) + .execute(conn) + }, + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } + ) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.add.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.add.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats/link').UnknownLink} root + * @param {import('../types.js').CARLink[]} shards + */ +export const input = (root, shards) => ({ root, shards }) diff --git a/packages/upload-client/src/upload/get.js b/packages/upload-client/src/upload/get.js new file mode 100644 index 000000000..42433f87e --- /dev/null +++ b/packages/upload-client/src/upload/get.js @@ -0,0 +1,73 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import retry from 'p-retry' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' + +/** + * Get details of an "upload". + * + * Required delegated capability proofs: `upload/get` + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/get` delegated capability. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} + */ +export async function get( + { issuer, with: resource, proofs, audience }, + root, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await retry( + async () => { + return await UploadCapabilities.get + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root), + proofs, + nonce: options.nonce, + }) + .execute(conn) + }, + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } + ) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.get.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.get.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats/link').UnknownLink} root + */ +export const input = (root) => ({ root }) diff --git a/packages/upload-client/src/upload/index.js b/packages/upload-client/src/upload/index.js new file mode 100644 index 000000000..f278e2c20 --- /dev/null +++ b/packages/upload-client/src/upload/index.js @@ -0,0 +1,4 @@ +export { add } from './add.js' +export { get } from './get.js' +export { list } from './list.js' +export { remove } from './remove.js' diff --git a/packages/upload-client/src/upload/list.js b/packages/upload-client/src/upload/list.js new file mode 100644 index 000000000..0075b416a --- /dev/null +++ b/packages/upload-client/src/upload/list.js @@ -0,0 +1,62 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * List uploads created by the issuer. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/list` delegated capability. + * @param {import('../types.js').ListRequestOptions} [options] + * @returns {Promise} + */ +export async function list( + { issuer, with: resource, proofs, audience }, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const result = await UploadCapabilities.list + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + proofs, + nb: input(options.cursor, options.size, options.pre), + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.list.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.list.can + +/** + * Returns required input to the invocation. + * + * @param {string} [cursor] + * @param {number} [size] + * @param {boolean} [pre] + */ +export const input = (cursor, size, pre) => ({ cursor, size, pre }) diff --git a/packages/upload-client/src/upload/remove.js b/packages/upload-client/src/upload/remove.js new file mode 100644 index 000000000..488edaa91 --- /dev/null +++ b/packages/upload-client/src/upload/remove.js @@ -0,0 +1,60 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Remove an upload by root data CID. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/remove` delegated capability. + * @param {import('multiformats').UnknownLink} root Root data CID to remove. + * @param {import('../types.js').RequestOptions} [options] + */ +export async function remove( + { issuer, with: resource, proofs, audience }, + root, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await UploadCapabilities.remove + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.remove.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.remove.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').UnknownLink} root + */ +export const input = (root) => ({ root }) diff --git a/packages/upload-client/test/blob.test.js b/packages/upload-client/test/blob.test.js index 71ec04eae..327aa8a09 100644 --- a/packages/upload-client/test/blob.test.js +++ b/packages/upload-client/test/blob.test.js @@ -7,7 +7,7 @@ import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as UCAN from '@web3-storage/capabilities/ucan' import * as BlobCapabilities from '@web3-storage/capabilities/blob' -import * as Blob from '../src/blob.js' +import * as Blob from '../src/blob/index.js' import { serviceSigner } from './fixtures.js' import { randomBytes } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -79,8 +79,9 @@ describe('Blob.add', () => { /** @type {import('../src/types.js').ProgressStatus[]} */ const progress = [] - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -106,14 +107,12 @@ describe('Blob.add', () => { 128 ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - // make sure it can also work without fetchWithUploadProgress /** @type {import('../src/types.js').ProgressStatus[]} */ let progressWithoutUploadProgress = [] - const { multihash: multihashWithoutUploadProgress } = await Blob.add( + await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -123,7 +122,6 @@ describe('Blob.add', () => { receiptsEndpoint, } ) - assert.deepEqual(multihashWithoutUploadProgress, bytesHash) assert.equal( progressWithoutUploadProgress.reduce( (max, { loaded }) => Math.max(max, loaded), @@ -137,6 +135,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -181,6 +180,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -194,6 +194,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -238,6 +239,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -255,6 +257,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -299,6 +302,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -356,8 +360,9 @@ describe('Blob.add', () => { channel: server, }) - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -365,9 +370,6 @@ describe('Blob.add', () => { } ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - assert(site) assert.equal(site.capabilities[0].can, Assert.location.can) // we're not verifying this as it's a mocked value @@ -421,8 +423,9 @@ describe('Blob.add', () => { channel: server, }) - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -430,9 +433,6 @@ describe('Blob.add', () => { } ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - assert(site) assert.equal(site.capabilities[0].can, Assert.location.can) // we're not verifying this as it's a mocked value @@ -444,6 +444,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -488,6 +489,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -501,6 +503,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -545,6 +548,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -558,6 +562,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const service = mockService({ ucan: { @@ -605,6 +610,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -619,6 +625,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -659,6 +666,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), diff --git a/packages/upload-client/test/dag-index.test.js b/packages/upload-client/test/dag-index.test.js index fc62d24b6..14e2cd07e 100644 --- a/packages/upload-client/test/dag-index.test.js +++ b/packages/upload-client/test/dag-index.test.js @@ -4,7 +4,7 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as IndexCapabilities from '@web3-storage/capabilities/index' -import * as Index from '../src/dag-index.js' +import * as Index from '../src/index/index.js' import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' diff --git a/packages/upload-client/test/helpers/utils.js b/packages/upload-client/test/helpers/utils.js index b35f0a171..e6ac53c87 100644 --- a/packages/upload-client/test/helpers/utils.js +++ b/packages/upload-client/test/helpers/utils.js @@ -6,7 +6,7 @@ import * as Server from '@ucanto/server' import * as HTTP from '@web3-storage/capabilities/http' import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' import { W3sBlob } from '@web3-storage/capabilities' -import { createConcludeInvocation } from '../../../upload-client/src/blob.js' +import { createConcludeInvocation } from '../../src/blob/add.js' import { randomCAR } from './random.js' export const validateAuthorization = () => ({ ok: {} }) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 4c3b77911..7dfb60abe 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -1047,4 +1047,162 @@ describe('uploadCAR', () => { 'bafkzcibcoibrsisrq3nrfmsxvynduf4kkf7qy33ip65w7ttfk7guyqod5w5mmei' ) }) + + it('generates invocation configuration on demand', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() // The "user" that will ask the service to accept the upload + const bytes = await randomBytes(128) + const file = new Blob([bytes]) + const expectedCar = await toCAR(bytes) + const piece = Piece.fromPayload(bytes).link + + /** @type {import('../src/types.js').CARLink|undefined} */ + let carCID + + const service = mockService({ + ucan: { + conclude: provide(UCAN.conclude, () => { + return { ok: { time: Date.now() } } + }), + }, + space: { + blob: { + add: provide( + BlobCapabilities.add, + // @ts-ignore Argument of type + async function ({ invocation, capability }) { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + assert.equal(capability.can, BlobCapabilities.add.can) + assert.equal(capability.with, space.did()) + return setupBlobAddSuccessResponse( + { issuer: space, audience: agent, with: space }, + invocation + ) + } + ), + }, + index: { + add: Server.provideAdvanced({ + capability: IndexCapabilities.add, + handler: async ({ capability }) => { + assert(capability.nb.index) + return Server.ok({}) + }, + }), + }, + }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, + upload: { + add: provide(UploadCapabilities.add, ({ invocation }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.add.can) + assert.equal(invCap.with, space.did()) + assert.equal(invCap.nb?.shards?.length, 1) + assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString()) + return { + ok: { + root: expectedCar.roots[0], + shards: [expectedCar.cid], + }, + } + }), + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + codec: CAR.inbound, + validateAuthorization, + }) + const connection = Client.connect({ + id: serviceSigner, + codec: CAR.outbound, + channel: server, + }) + const dataCID = await uploadFile( + async (caps) => { + const proofs = [] + for (const { can, nb } of caps) { + if (can === BlobCapabilities.add.can) { + proofs.push( + await BlobCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').BlobAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } else if (can === IndexCapabilities.add.can) { + proofs.push( + await IndexCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').IndexAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } else if (can === UploadCapabilities.add.can) { + proofs.push( + await UploadCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').UploadAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } + } + return { + issuer: agent, + with: space.did(), + proofs, + audience: serviceSigner, + } + }, + file, + { + connection, + onShardStored: (meta) => { + carCID = meta.cid + }, + receiptsEndpoint, + } + ) + + assert(service.space.blob.add.called) + assert.equal(service.space.blob.add.callCount, 2) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + assert(service.space.index.add.called) + assert.equal(service.space.index.add.callCount, 1) + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) + + assert.equal(carCID?.toString(), expectedCar.cid.toString()) + assert.equal(dataCID.toString(), expectedCar.roots[0].toString()) + }) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index f29430945..7cd5a4b0e 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -5,7 +5,7 @@ import { provide } from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as UploadCapabilities from '@web3-storage/capabilities/upload' -import * as Upload from '../src/upload.js' +import * as Upload from '../src/upload/index.js' import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' diff --git a/packages/w3up-client/src/capability/blob.js b/packages/w3up-client/src/capability/blob.js index 487af1978..450fcfa4d 100644 --- a/packages/w3up-client/src/capability/blob.js +++ b/packages/w3up-client/src/capability/blob.js @@ -1,5 +1,6 @@ import { Blob } from '@web3-storage/upload-client' import { Blob as BlobCapabilities } from '@web3-storage/capabilities' +import { sha256 } from 'multiformats/hashes/sha2' import { Base } from '../base.js' /** @@ -18,7 +19,9 @@ export class BlobClient extends Base { async add(blob, options = {}) { const conf = await this._invocationConfig([BlobCapabilities.add.can]) options.connection = this._serviceConf.upload - return Blob.add(conf, blob, options) + const bytes = new Uint8Array(await blob.arrayBuffer()) + const digest = await sha256.digest(bytes) + return { digest, ...(await Blob.add(conf, digest, bytes, options)) } } /** diff --git a/packages/w3up-client/test/capability/blob.test.js b/packages/w3up-client/test/capability/blob.test.js index a713f4756..1baacf1ff 100644 --- a/packages/w3up-client/test/capability/blob.test.js +++ b/packages/w3up-client/test/capability/blob.test.js @@ -33,16 +33,16 @@ export const BlobClient = Test.withContext({ const bytes = await randomBytes(128) const bytesHash = await sha256.digest(bytes) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) // TODO we should check blobsStorage as well - assert.deepEqual(await allocationsStorage.exists(space.did(), multihash), { + assert.deepEqual(await allocationsStorage.exists(space.did(), digest), { ok: true, }) - assert.deepEqual(multihash.bytes, bytesHash.bytes) + assert.deepEqual(digest.bytes, bytesHash.bytes) }, 'should list stored blobs': async ( assert, @@ -71,10 +71,10 @@ export const BlobClient = Test.withContext({ const bytes = await randomBytes(128) const bytesHash = await sha256.digest(bytes) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - assert.deepEqual(multihash.bytes, bytesHash.bytes) + assert.deepEqual(digest.bytes, bytesHash.bytes) const { results: [entry], @@ -109,11 +109,11 @@ export const BlobClient = Test.withContext({ }) const bytes = await randomBytes(128) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - const result = await alice.capability.blob.remove(multihash) + const result = await alice.capability.blob.remove(digest) assert.ok(result.ok) }, 'should get a stored blob': async ( @@ -142,11 +142,11 @@ export const BlobClient = Test.withContext({ }) const bytes = await randomBytes(128) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - const result = await alice.capability.blob.get(multihash) + const result = await alice.capability.blob.get(digest) assert.ok(result.ok) }, }) diff --git a/packages/w3up-client/test/capability/index.test.js b/packages/w3up-client/test/capability/index.test.js index 6a812330f..40196272c 100644 --- a/packages/w3up-client/test/capability/index.test.js +++ b/packages/w3up-client/test/capability/index.test.js @@ -29,16 +29,14 @@ export const IndexClient = Test.withContext({ const index = ShardedDAGIndex.create(car.cid) const indexBytes = Result.unwrap(await index.archive()) - const { multihash } = await alice.capability.blob.add( + const { digest } = await alice.capability.blob.add( new Blob([indexBytes]), { receiptsEndpoint, } ) - assert.ok( - await alice.capability.index.add(Link.create(CAR.code, multihash)) - ) + assert.ok(await alice.capability.index.add(Link.create(CAR.code, digest))) }, }, })