diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md index f75a10b74..2ad1ad35d 100644 --- a/packages/upload-client/README.md +++ b/packages/upload-client/README.md @@ -180,6 +180,7 @@ await Upload.add(conf, rootCID, carCIDs) - [`CARMetadata`](#carmetadata) - [`DirectoryEntryLinkCallback`](#directoryentrylinkcallback) - [`InvocationConfig`](#invocationconfig) + - [`InvocationConfigurator`](#invocationconfigurator) - [`ShardStoredCallback`](#shardstoredcallback) - [Contributing](#contributing) - [License](#license) @@ -190,7 +191,7 @@ await Upload.add(conf, rootCID, carCIDs) ```ts function uploadDirectory( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, files: File[], options: { retries?: number @@ -207,13 +208,13 @@ Uploads a directory of files to the service and returns the root data CID for th Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig), [`ShardStoredCallback`](#shardstoredcallback) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator), [`ShardStoredCallback`](#shardstoredcallback) ### `uploadFile` ```ts function uploadFile( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, file: Blob, options: { retries?: number @@ -229,13 +230,13 @@ Uploads a file to the service and returns the root data CID for the generated DA Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator) ### `uploadCAR` ```ts function uploadCAR( - conf: InvocationConfig, + conf: InvocationConfig | InvocationConfigurator, car: Blob, options: { retries?: number @@ -252,7 +253,7 @@ Uploads a CAR file to the service. The difference between this function and [Blo Required delegated capability proofs: `blob/add`, `index/add`, `upload/add`, `filecoin/offer` -More information: [`InvocationConfig`](#invocationconfig), [`ShardStoredCallback`](#shardstoredcallback) +More information: [`InvocationConfig`](#invocationconfig), [`InvocationConfigurator`](#invocationconfigurator), [`ShardStoredCallback`](#shardstoredcallback) ### `Blob.add` @@ -512,6 +513,52 @@ This is the configuration for the UCAN invocation. It's values can be obtained f - The `issuer` is the signing authority that is issuing the UCAN invocation(s). It is typically the user _agent_. - The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. +### `InvocationConfigurator` + +A function that generates [invocation configuration](#invocationconfig) for the requested capabilities. The intention is for the client to be able to [request, on demand, delegated capabilities from an application server](https://github.com/storacha-network/w3up-examples/tree/main/delegated-upload). + +```ts +interface InvocationConfigurator { + (caps: CapabilityQuery[]): Await +} + +interface CapabilityQuery { + can: ServiceAbility + nb?: unknown +} + +// "space/blob/add", "space/index/add" etc. +type ServiceAbility = string +``` + +The function may be called multiple times with different requested capabilities. + +Example: + +```js +import { Agent } from '@web3-storage/access' +import * as Space from '@web3-storage/access/space' + +const agent = await Agent.create() +const space = await Space.generate({ name: 'myspace' }) + +const configure = async (caps) => ({ + issuer: agent.issuer, + with: space.did(), + proofs: [ + // delegate from the space to the agent the requested capabilities + await Delegation.delegate({ + issuer: space.signer, + audience: agent.did(), + capabilities: caps.map(c => ({ can: c.can, with: space.did(), nb: c.nb })), + expiration: Math.floor(Date.now() / 1000) + (60 * 60) // 1h in seconds + }) + ] +}) + +await uploadFile(configure, new Blob(['Hello World!'])) +``` + ### `ShardStoredCallback` A function called after a DAG shard has been successfully stored by the service: diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 63b2aac82..81be97f90 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -31,11 +31,12 @@ }, "exports": { ".": "./dist/src/index.js", - "./blob": "./dist/src/blob.js", + "./blob": "./dist/src/blob/index.js", "./car": "./dist/src/car.js", "./fetch-with-upload-progress": "./dist/src/fetch-with-upload-progress.js", + "./index": "./dist/src/index/index.js", "./sharding": "./dist/src/sharding.js", - "./upload": "./dist/src/upload.js", + "./upload": "./dist/src/upload/index.js", "./store": "./dist/src/store.js", "./unixfs": "./dist/src/unixfs.js", "./types": "./dist/src/types.js" @@ -43,16 +44,19 @@ "typesVersions": { "*": { "blob": [ - "dist/src/blob.d.ts" + "dist/src/blob/index.d.ts" ], "car": [ "dist/src/car.d.ts" ], + "index": [ + "dist/src/index/index.d.ts" + ], "sharding": [ "dist/src/sharding.d.ts" ], "upload": [ - "dist/src/upload.d.ts" + "dist/src/upload/index.d.ts" ], "store": [ "dist/src/store.d.ts" diff --git a/packages/upload-client/src/blob.js b/packages/upload-client/src/blob/add.js similarity index 62% rename from packages/upload-client/src/blob.js rename to packages/upload-client/src/blob/add.js index b9adda959..493c4a4f3 100644 --- a/packages/upload-client/src/blob.js +++ b/packages/upload-client/src/blob/add.js @@ -1,4 +1,3 @@ -import { sha256 } from 'multiformats/hashes/sha2' import { ed25519 } from '@ucanto/principal' import { conclude } from '@web3-storage/capabilities/ucan' import * as UCAN from '@web3-storage/capabilities/ucan' @@ -8,20 +7,17 @@ import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as HTTPCapabilities from '@web3-storage/capabilities/http' import { SpaceDID } from '@web3-storage/capabilities/utils' import retry, { AbortError } from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' -import { poll } from './receipts.js' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' +import { poll } from '../receipts.js' /** * @param {string} url - * @param {import('./types.js').ProgressFn} handler + * @param {import('../types.js').ProgressFn} handler */ function createUploadProgressHandler(url, handler) { - /** - * - * @param {import('./types.js').ProgressStatus} status - */ - function onUploadProgress({ total, loaded, lengthComputable }) { + /** @param {import('../types.js').ProgressStatus} status */ + const onUploadProgress = ({ total, loaded, lengthComputable }) => { return handler({ total, loaded, lengthComputable, url }) } return onUploadProgress @@ -74,7 +70,7 @@ function parseBlobAddReceiptNext(receipt) { // Decode receipts available const nextReceipts = concludefxs.map((fx) => getConcludeReceipt(fx)) - /** @type {import('@ucanto/interface').Receipt | undefined} */ + /** @type {import('@ucanto/interface').Receipt | undefined} */ // @ts-expect-error types unknown for next const allocateReceipt = nextReceipts.find((receipt) => receipt.ran.link().equals(allocateTask.cid) @@ -85,7 +81,7 @@ function parseBlobAddReceiptNext(receipt) { receipt.ran.link().equals(putTask.cid) ) - /** @type {import('@ucanto/interface').Receipt | undefined} */ + /** @type {import('@ucanto/interface').Receipt | undefined} */ // @ts-expect-error types unknown for next const acceptReceipt = nextReceipts.find((receipt) => receipt.ran.link().equals(acceptTask.cid) @@ -115,7 +111,7 @@ function parseBlobAddReceiptNext(receipt) { // FIXME this code has been copied over from upload-api /** * @param {import('@ucanto/interface').Signer} id - * @param {import('@ucanto/interface').Verifier} serviceDid + * @param {import('@ucanto/interface').Principal} serviceDid * @param {import('@ucanto/interface').Receipt} receipt */ export function createConcludeInvocation(id, serviceDid, receipt) { @@ -152,7 +148,7 @@ export function createConcludeInvocation(id, serviceDid, receipt) { * * Required delegated capability proofs: `blob/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration + * @param {import('../types.js').InvocationConfig} conf Configuration * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. * * The `issuer` is the signing authority that is issuing the UCAN @@ -165,19 +161,20 @@ export function createConcludeInvocation(id, serviceDid, receipt) { * has the capability to perform the action. * * The issuer needs the `blob/add` delegated capability. + * @param {import('multiformats').MultihashDigest} digest * @param {Blob|Uint8Array} data Blob data. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} */ export async function add( { issuer, with: resource, proofs, audience }, + digest, data, options = {} ) { /* c8 ignore next 2 */ const bytes = data instanceof Uint8Array ? data : new Uint8Array(await data.arrayBuffer()) - const multihash = await sha256.digest(bytes) const size = bytes.length /* c8 ignore next */ const conn = options.connection ?? connection @@ -190,12 +187,7 @@ export async function add( /* c8 ignore next */ audience: audience ?? servicePrincipal, with: SpaceDID.from(resource), - nb: { - blob: { - digest: multihash.bytes, - size, - }, - }, + nb: input(digest, size), proofs, nonce: options.nonce, }) @@ -293,8 +285,8 @@ export async function add( }) const httpPutConcludeInvocation = createConcludeInvocation( issuer, - // @ts-expect-error object of type unknown - audience, + /* c8 ignore next */ + audience ?? servicePrincipal, httpPutReceipt ) const ucanConclude = await httpPutConcludeInvocation.execute(conn) @@ -324,155 +316,21 @@ export async function add( blocks, }) - return { - multihash, - site, - } + return { site } } -/** - * List Blobs stored in the space. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/list` delegated capability. - * @param {import('./types.js').ListRequestOptions} [options] - * @returns {Promise} - */ -export async function list( - { issuer, with: resource, proofs, audience }, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.list - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - proofs, - nb: { - cursor: options.cursor, - size: options.size, - }, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.list.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Remove a stored Blob file by digest. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/remove` delegated capability. - * @param {import('multiformats').MultihashDigest} multihash of the blob - * @param {import('./types.js').RequestOptions} [options] - */ -export async function remove( - { issuer, with: resource, proofs, audience }, - multihash, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.remove - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { - digest: multihash.bytes, - }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.remove.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out -} +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.add.can /** - * Gets a stored Blob file by digest. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * Returns required input to the invocation. * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `blob/get/0/1` delegated capability. - * @param {import('multiformats').MultihashDigest} multihash of the blob - * @param {import('./types.js').RequestOptions} [options] + * @param {import('multiformats').MultihashDigest} digest + * @param {number} size */ -export async function get( - { issuer, with: resource, proofs, audience }, - multihash, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await BlobCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { - digest: multihash.bytes, - }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${BlobCapabilities.get.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out -} +export const input = (digest, size) => ({ + blob: { + digest: digest.bytes, + size, + }, +}) diff --git a/packages/upload-client/src/blob/get.js b/packages/upload-client/src/blob/get.js new file mode 100644 index 000000000..5287922a1 --- /dev/null +++ b/packages/upload-client/src/blob/get.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Gets a stored Blob file by digest. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/get/0/1` delegated capability. + * @param {import('multiformats').MultihashDigest} multihash of the blob + * @param {import('../types.js').RequestOptions} [options] + */ +export async function get( + { issuer, with: resource, proofs, audience }, + multihash, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.get + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(multihash), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.get.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.get.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').MultihashDigest} digest + */ +export const input = (digest) => ({ digest: digest.bytes }) diff --git a/packages/upload-client/src/blob/index.js b/packages/upload-client/src/blob/index.js new file mode 100644 index 000000000..f278e2c20 --- /dev/null +++ b/packages/upload-client/src/blob/index.js @@ -0,0 +1,4 @@ +export { add } from './add.js' +export { get } from './get.js' +export { list } from './list.js' +export { remove } from './remove.js' diff --git a/packages/upload-client/src/blob/list.js b/packages/upload-client/src/blob/list.js new file mode 100644 index 000000000..eaef942d3 --- /dev/null +++ b/packages/upload-client/src/blob/list.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * List Blobs stored in the space. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/list` delegated capability. + * @param {import('../types.js').ListRequestOptions} [options] + * @returns {Promise} + */ +export async function list( + { issuer, with: resource, proofs, audience }, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.list + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + proofs, + nb: input(options.cursor, options.size), + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.list.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.list.can + +/** + * Returns required input to the invocation. + * + * @param {string} [cursor] + * @param {number} [size] + */ +export const input = (cursor, size) => ({ cursor, size }) diff --git a/packages/upload-client/src/blob/remove.js b/packages/upload-client/src/blob/remove.js new file mode 100644 index 000000000..6ecb4db10 --- /dev/null +++ b/packages/upload-client/src/blob/remove.js @@ -0,0 +1,60 @@ +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Remove a stored Blob file by digest. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/remove` delegated capability. + * @param {import('multiformats').MultihashDigest} multihash of the blob + * @param {import('../types.js').RequestOptions} [options] + */ +export async function remove( + { issuer, with: resource, proofs, audience }, + multihash, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.remove + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(multihash), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.remove.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out +} + +/** Returns the ability used by an invocation. */ +export const ability = BlobCapabilities.remove.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').MultihashDigest} digest + */ +export const input = (digest) => ({ digest: digest.bytes }) diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 77aa3f666..f8dbbd35d 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -2,10 +2,14 @@ import * as PieceHasher from '@web3-storage/data-segment/multihash' import { Storefront } from '@web3-storage/filecoin-client' import * as Link from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' import * as Store from './store.js' -import * as Blob from './blob.js' -import * as Index from './dag-index.js' -import * as Upload from './upload.js' +import * as Blob from './blob/index.js' +import * as BlobAdd from './blob/add.js' +import * as Index from './index/index.js' +import * as IndexAdd from './index/add.js' +import * as Upload from './upload/index.js' +import * as UploadAdd from './upload/add.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' @@ -23,8 +27,9 @@ export * as Receipt from './receipts.js' * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object. * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -56,8 +61,9 @@ export async function uploadFile(conf, file, options = {}) { * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -97,8 +103,9 @@ export async function uploadDirectory(conf, files, options = {}) { * Required delegated capability proofs: `blob/add`, `index/add`, * `filecoin/offer`, `upload/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`, or a + * function that generates this object * * The `issuer` is the signing authority that is issuing the UCAN * invocation(s). It is typically the user _agent_. @@ -120,7 +127,7 @@ export async function uploadCAR(conf, car, options = {}) { } /** - * @param {import('./types.js').InvocationConfig} conf + * @param {import('./types.js').InvocationConfig|import('./types.js').InvocationConfigurator} conf * @param {ReadableStream} blocks * @param {import('./types.js').UploadOptions} [options] * @returns {Promise} @@ -130,6 +137,8 @@ async function uploadBlockStream( blocks, { pieceHasher = PieceHasher, ...options } = {} ) { + /** @type {import('./types.js').InvocationConfigurator} */ + const configure = typeof conf === 'function' ? conf : () => conf /** @type {Array>} */ const shardIndexes = [] /** @type {import('./types.js').CARLink[]} */ @@ -144,16 +153,23 @@ async function uploadBlockStream( new TransformStream({ async transform(car, controller) { const bytes = new Uint8Array(await car.arrayBuffer()) + const digest = await sha256.digest(bytes) + const conf = await configure([ + { + can: BlobAdd.ability, + nb: BlobAdd.input(digest, bytes.length), + }, + ]) // Invoke blob/add and write bytes to write target - const { multihash } = await Blob.add(conf, bytes, options) - // Should this be raw instead? - const cid = Link.create(CAR.code, multihash) + await Blob.add(conf, digest, bytes, options) + const cid = Link.create(CAR.code, digest) + let piece if (pieceHasher) { const multihashDigest = await pieceHasher.digest(bytes) /** @type {import('@web3-storage/capabilities/types').PieceLink} */ piece = Link.create(raw.code, multihashDigest) - const content = Link.create(raw.code, multihash) + const content = Link.create(raw.code, digest) // Invoke filecoin/offer for data const result = await Storefront.filecoinOffer( @@ -206,14 +222,36 @@ async function uploadBlockStream( throw new Error('failed to archive DAG index', { cause: indexBytes.error }) } - // Store the index in the space - const { multihash } = await Blob.add(conf, indexBytes.ok, options) - const indexLink = Link.create(CAR.code, multihash) + const indexDigest = await sha256.digest(indexBytes.ok) + const indexLink = Link.create(CAR.code, indexDigest) + const [blobAddConf, indexAddConf, uploadAddConf] = await Promise.all([ + configure([ + { + can: BlobAdd.ability, + nb: BlobAdd.input(indexDigest, indexBytes.ok.length), + }, + ]), + configure([ + { + can: IndexAdd.ability, + nb: IndexAdd.input(indexLink), + }, + ]), + configure([ + { + can: UploadAdd.ability, + nb: UploadAdd.input(root, shards), + }, + ]), + ]) + + // Store the index in the space + await Blob.add(blobAddConf, indexDigest, indexBytes.ok, options) // Register the index with the service - await Index.add(conf, indexLink, options) + await Index.add(indexAddConf, indexLink, options) // Register an upload with the service - await Upload.add(conf, root, shards, options) + await Upload.add(uploadAddConf, root, shards, options) return root } diff --git a/packages/upload-client/src/dag-index.js b/packages/upload-client/src/index/add.js similarity index 70% rename from packages/upload-client/src/dag-index.js rename to packages/upload-client/src/index/add.js index 572d8c046..662040b43 100644 --- a/packages/upload-client/src/dag-index.js +++ b/packages/upload-client/src/index/add.js @@ -1,8 +1,8 @@ import * as IndexCapabilities from '@web3-storage/capabilities/index' import { SpaceDID } from '@web3-storage/capabilities/utils' import retry from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' /** * Register an "index" with the service. The issuer needs the `index/add` @@ -10,7 +10,7 @@ import { REQUEST_RETRIES } from './constants.js' * * Required delegated capability proofs: `index/add` * - * @param {import('./types.js').InvocationConfig} conf Configuration + * @param {import('../types.js').InvocationConfig} conf Configuration * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. * * The `issuer` is the signing authority that is issuing the UCAN @@ -23,9 +23,9 @@ import { REQUEST_RETRIES } from './constants.js' * has the capability to perform the action. * * The issuer needs the `index/add` delegated capability. - * @param {import('./types.js').CARLink} index Index to store. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} + * @param {import('../types.js').CARLink} index Index to store. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} */ export async function add( { issuer, with: resource, proofs, audience }, @@ -42,7 +42,7 @@ export async function add( /* c8 ignore next */ audience: audience ?? servicePrincipal, with: SpaceDID.from(resource), - nb: { index }, + nb: input(index), proofs, }) .execute(conn) @@ -61,3 +61,13 @@ export async function add( return result.out.ok } + +/** Returns the ability used by an invocation. */ +export const ability = IndexCapabilities.add.can + +/** + * Returns required input to the invocation. + * + * @param {import('../types.js').CARLink} index + */ +export const input = (index) => ({ index }) diff --git a/packages/upload-client/src/index/index.js b/packages/upload-client/src/index/index.js new file mode 100644 index 000000000..6448ab66f --- /dev/null +++ b/packages/upload-client/src/index/index.js @@ -0,0 +1 @@ +export { add } from './add.js' diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 31cfe2a60..408e069df 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -2,13 +2,7 @@ import type { FetchOptions as IpfsUtilsFetchOptions, ProgressStatus as XHRProgressStatus, } from 'ipfs-utils/src/types.js' -import { - MultihashDigest, - Link, - UnknownLink, - Version, - MultihashHasher, -} from 'multiformats' +import { Link, UnknownLink, Version, MultihashHasher } from 'multiformats' import { Block } from '@ipld/unixfs' import { ServiceMethod, @@ -20,6 +14,7 @@ import { Failure, Delegation, Capabilities, + Await, } from '@ucanto/interface' import { UCANConclude, @@ -74,6 +69,7 @@ import { UsageReport, UsageReportSuccess, UsageReportFailure, + ServiceAbility, } from '@web3-storage/capabilities/types' import { StorefrontService } from '@web3-storage/filecoin-client/storefront' import { code as pieceHashCode } from '@web3-storage/data-segment/multihash' @@ -213,6 +209,16 @@ export interface InvocationConfig { proofs: Proof[] } +export interface CapabilityQuery { + can: ServiceAbility + nb?: unknown +} + +/** Generates invocation configuration for the requested capabilities. */ +export interface InvocationConfigurator { + (caps: CapabilityQuery[]): Await +} + export interface UnixFSEncodeResult { /** * Root CID for the DAG. @@ -396,6 +402,5 @@ export interface FileLike extends BlobLike { } export interface BlobAddOk { - multihash: MultihashDigest site: Delegation } diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js deleted file mode 100644 index 7322c8c72..000000000 --- a/packages/upload-client/src/upload.js +++ /dev/null @@ -1,222 +0,0 @@ -import * as UploadCapabilities from '@web3-storage/capabilities/upload' -import { SpaceDID } from '@web3-storage/capabilities/utils' -import retry from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' - -/** - * Register an "upload" with the service. The issuer needs the `upload/add` - * delegated capability. - * - * Required delegated capability proofs: `upload/add` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/add` delegated capability. - * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. - * @param {import('./types.js').CARLink[]} shards CIDs of CAR files that contain the DAG. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function add( - { issuer, with: resource, proofs, audience }, - root, - shards, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await UploadCapabilities.add - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root, shards }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.add.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Get details of an "upload". - * - * Required delegated capability proofs: `upload/get` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/get` delegated capability. - * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function get( - { issuer, with: resource, proofs, audience }, - root, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await UploadCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.get.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * List uploads created by the issuer. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/list` delegated capability. - * @param {import('./types.js').ListRequestOptions} [options] - * @returns {Promise} - */ -export async function list( - { issuer, with: resource, proofs, audience }, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - const result = await UploadCapabilities.list - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - proofs, - nb: { - cursor: options.cursor, - size: options.size, - pre: options.pre, - }, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.list.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Remove an upload by root data CID. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `upload/remove` delegated capability. - * @param {import('multiformats').UnknownLink} root Root data CID to remove. - * @param {import('./types.js').RequestOptions} [options] - */ -export async function remove( - { issuer, with: resource, proofs, audience }, - root, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await UploadCapabilities.remove - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { root }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${UploadCapabilities.remove.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} diff --git a/packages/upload-client/src/upload/add.js b/packages/upload-client/src/upload/add.js new file mode 100644 index 000000000..f526ba653 --- /dev/null +++ b/packages/upload-client/src/upload/add.js @@ -0,0 +1,77 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import retry from 'p-retry' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' + +/** + * Register an "upload" with the service. The issuer needs the `upload/add` + * delegated capability. + * + * Required delegated capability proofs: `upload/add` + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/add` delegated capability. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('../types.js').CARLink[]} shards CIDs of CAR files that contain the DAG. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} + */ +export async function add( + { issuer, with: resource, proofs, audience }, + root, + shards, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await retry( + async () => { + return await UploadCapabilities.add + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root, shards), + proofs, + nonce: options.nonce, + }) + .execute(conn) + }, + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } + ) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.add.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.add.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats/link').UnknownLink} root + * @param {import('../types.js').CARLink[]} shards + */ +export const input = (root, shards) => ({ root, shards }) diff --git a/packages/upload-client/src/upload/get.js b/packages/upload-client/src/upload/get.js new file mode 100644 index 000000000..42433f87e --- /dev/null +++ b/packages/upload-client/src/upload/get.js @@ -0,0 +1,73 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import retry from 'p-retry' +import { servicePrincipal, connection } from '../service.js' +import { REQUEST_RETRIES } from '../constants.js' + +/** + * Get details of an "upload". + * + * Required delegated capability proofs: `upload/get` + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/get` delegated capability. + * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored. + * @param {import('../types.js').RequestOptions} [options] + * @returns {Promise} + */ +export async function get( + { issuer, with: resource, proofs, audience }, + root, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await retry( + async () => { + return await UploadCapabilities.get + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root), + proofs, + nonce: options.nonce, + }) + .execute(conn) + }, + { + onFailedAttempt: console.warn, + retries: options.retries ?? REQUEST_RETRIES, + } + ) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.get.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.get.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats/link').UnknownLink} root + */ +export const input = (root) => ({ root }) diff --git a/packages/upload-client/src/upload/index.js b/packages/upload-client/src/upload/index.js new file mode 100644 index 000000000..f278e2c20 --- /dev/null +++ b/packages/upload-client/src/upload/index.js @@ -0,0 +1,4 @@ +export { add } from './add.js' +export { get } from './get.js' +export { list } from './list.js' +export { remove } from './remove.js' diff --git a/packages/upload-client/src/upload/list.js b/packages/upload-client/src/upload/list.js new file mode 100644 index 000000000..0075b416a --- /dev/null +++ b/packages/upload-client/src/upload/list.js @@ -0,0 +1,62 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * List uploads created by the issuer. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/list` delegated capability. + * @param {import('../types.js').ListRequestOptions} [options] + * @returns {Promise} + */ +export async function list( + { issuer, with: resource, proofs, audience }, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const result = await UploadCapabilities.list + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + proofs, + nb: input(options.cursor, options.size, options.pre), + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.list.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.list.can + +/** + * Returns required input to the invocation. + * + * @param {string} [cursor] + * @param {number} [size] + * @param {boolean} [pre] + */ +export const input = (cursor, size, pre) => ({ cursor, size, pre }) diff --git a/packages/upload-client/src/upload/remove.js b/packages/upload-client/src/upload/remove.js new file mode 100644 index 000000000..488edaa91 --- /dev/null +++ b/packages/upload-client/src/upload/remove.js @@ -0,0 +1,60 @@ +import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import { SpaceDID } from '@web3-storage/capabilities/utils' +import { servicePrincipal, connection } from '../service.js' + +/** + * Remove an upload by root data CID. + * + * @param {import('../types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `upload/remove` delegated capability. + * @param {import('multiformats').UnknownLink} root Root data CID to remove. + * @param {import('../types.js').RequestOptions} [options] + */ +export async function remove( + { issuer, with: resource, proofs, audience }, + root, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await UploadCapabilities.remove + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: input(root), + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${UploadCapabilities.remove.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out.ok +} + +/** Returns the ability used by an invocation. */ +export const ability = UploadCapabilities.remove.can + +/** + * Returns required input to the invocation. + * + * @param {import('multiformats').UnknownLink} root + */ +export const input = (root) => ({ root }) diff --git a/packages/upload-client/test/blob.test.js b/packages/upload-client/test/blob.test.js index 71ec04eae..327aa8a09 100644 --- a/packages/upload-client/test/blob.test.js +++ b/packages/upload-client/test/blob.test.js @@ -7,7 +7,7 @@ import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as UCAN from '@web3-storage/capabilities/ucan' import * as BlobCapabilities from '@web3-storage/capabilities/blob' -import * as Blob from '../src/blob.js' +import * as Blob from '../src/blob/index.js' import { serviceSigner } from './fixtures.js' import { randomBytes } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -79,8 +79,9 @@ describe('Blob.add', () => { /** @type {import('../src/types.js').ProgressStatus[]} */ const progress = [] - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -106,14 +107,12 @@ describe('Blob.add', () => { 128 ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - // make sure it can also work without fetchWithUploadProgress /** @type {import('../src/types.js').ProgressStatus[]} */ let progressWithoutUploadProgress = [] - const { multihash: multihashWithoutUploadProgress } = await Blob.add( + await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -123,7 +122,6 @@ describe('Blob.add', () => { receiptsEndpoint, } ) - assert.deepEqual(multihashWithoutUploadProgress, bytesHash) assert.equal( progressWithoutUploadProgress.reduce( (max, { loaded }) => Math.max(max, loaded), @@ -137,6 +135,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -181,6 +180,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -194,6 +194,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -238,6 +239,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -255,6 +257,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -299,6 +302,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -356,8 +360,9 @@ describe('Blob.add', () => { channel: server, }) - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -365,9 +370,6 @@ describe('Blob.add', () => { } ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - assert(site) assert.equal(site.capabilities[0].can, Assert.location.can) // we're not verifying this as it's a mocked value @@ -421,8 +423,9 @@ describe('Blob.add', () => { channel: server, }) - const { site, multihash } = await Blob.add( + const { site } = await Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -430,9 +433,6 @@ describe('Blob.add', () => { } ) - assert(multihash) - assert.deepEqual(multihash, bytesHash) - assert(site) assert.equal(site.capabilities[0].can, Assert.location.can) // we're not verifying this as it's a mocked value @@ -444,6 +444,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -488,6 +489,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -501,6 +503,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -545,6 +548,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), @@ -558,6 +562,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const service = mockService({ ucan: { @@ -605,6 +610,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection, @@ -619,6 +625,7 @@ describe('Blob.add', () => { const space = await Signer.generate() const agent = await Signer.generate() const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) const proofs = [ await BlobCapabilities.add.delegate({ @@ -659,6 +666,7 @@ describe('Blob.add', () => { await assert.rejects( Blob.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, bytes, { connection } ), diff --git a/packages/upload-client/test/dag-index.test.js b/packages/upload-client/test/dag-index.test.js index fc62d24b6..14e2cd07e 100644 --- a/packages/upload-client/test/dag-index.test.js +++ b/packages/upload-client/test/dag-index.test.js @@ -4,7 +4,7 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as IndexCapabilities from '@web3-storage/capabilities/index' -import * as Index from '../src/dag-index.js' +import * as Index from '../src/index/index.js' import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' diff --git a/packages/upload-client/test/helpers/utils.js b/packages/upload-client/test/helpers/utils.js index b35f0a171..e6ac53c87 100644 --- a/packages/upload-client/test/helpers/utils.js +++ b/packages/upload-client/test/helpers/utils.js @@ -6,7 +6,7 @@ import * as Server from '@ucanto/server' import * as HTTP from '@web3-storage/capabilities/http' import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' import { W3sBlob } from '@web3-storage/capabilities' -import { createConcludeInvocation } from '../../../upload-client/src/blob.js' +import { createConcludeInvocation } from '../../src/blob/add.js' import { randomCAR } from './random.js' export const validateAuthorization = () => ({ ok: {} }) diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 4c3b77911..7dfb60abe 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -1047,4 +1047,162 @@ describe('uploadCAR', () => { 'bafkzcibcoibrsisrq3nrfmsxvynduf4kkf7qy33ip65w7ttfk7guyqod5w5mmei' ) }) + + it('generates invocation configuration on demand', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() // The "user" that will ask the service to accept the upload + const bytes = await randomBytes(128) + const file = new Blob([bytes]) + const expectedCar = await toCAR(bytes) + const piece = Piece.fromPayload(bytes).link + + /** @type {import('../src/types.js').CARLink|undefined} */ + let carCID + + const service = mockService({ + ucan: { + conclude: provide(UCAN.conclude, () => { + return { ok: { time: Date.now() } } + }), + }, + space: { + blob: { + add: provide( + BlobCapabilities.add, + // @ts-ignore Argument of type + async function ({ invocation, capability }) { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + assert.equal(capability.can, BlobCapabilities.add.can) + assert.equal(capability.with, space.did()) + return setupBlobAddSuccessResponse( + { issuer: space, audience: agent, with: space }, + invocation + ) + } + ), + }, + index: { + add: Server.provideAdvanced({ + capability: IndexCapabilities.add, + handler: async ({ capability }) => { + assert(capability.nb.index) + return Server.ok({}) + }, + }), + }, + }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, + upload: { + add: provide(UploadCapabilities.add, ({ invocation }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.add.can) + assert.equal(invCap.with, space.did()) + assert.equal(invCap.nb?.shards?.length, 1) + assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString()) + return { + ok: { + root: expectedCar.roots[0], + shards: [expectedCar.cid], + }, + } + }), + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + codec: CAR.inbound, + validateAuthorization, + }) + const connection = Client.connect({ + id: serviceSigner, + codec: CAR.outbound, + channel: server, + }) + const dataCID = await uploadFile( + async (caps) => { + const proofs = [] + for (const { can, nb } of caps) { + if (can === BlobCapabilities.add.can) { + proofs.push( + await BlobCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').BlobAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } else if (can === IndexCapabilities.add.can) { + proofs.push( + await IndexCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').IndexAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } else if (can === UploadCapabilities.add.can) { + proofs.push( + await UploadCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + nb: /** @type {import('@web3-storage/capabilities/types').UploadAdd['nb']} */ ( + nb + ), + expiration: Infinity, + }) + ) + } + } + return { + issuer: agent, + with: space.did(), + proofs, + audience: serviceSigner, + } + }, + file, + { + connection, + onShardStored: (meta) => { + carCID = meta.cid + }, + receiptsEndpoint, + } + ) + + assert(service.space.blob.add.called) + assert.equal(service.space.blob.add.callCount, 2) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + assert(service.space.index.add.called) + assert.equal(service.space.index.add.callCount, 1) + assert(service.upload.add.called) + assert.equal(service.upload.add.callCount, 1) + + assert.equal(carCID?.toString(), expectedCar.cid.toString()) + assert.equal(dataCID.toString(), expectedCar.roots[0].toString()) + }) }) diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index f29430945..7cd5a4b0e 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -5,7 +5,7 @@ import { provide } from '@ucanto/server' import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as UploadCapabilities from '@web3-storage/capabilities/upload' -import * as Upload from '../src/upload.js' +import * as Upload from '../src/upload/index.js' import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' diff --git a/packages/w3up-client/src/capability/blob.js b/packages/w3up-client/src/capability/blob.js index 487af1978..450fcfa4d 100644 --- a/packages/w3up-client/src/capability/blob.js +++ b/packages/w3up-client/src/capability/blob.js @@ -1,5 +1,6 @@ import { Blob } from '@web3-storage/upload-client' import { Blob as BlobCapabilities } from '@web3-storage/capabilities' +import { sha256 } from 'multiformats/hashes/sha2' import { Base } from '../base.js' /** @@ -18,7 +19,9 @@ export class BlobClient extends Base { async add(blob, options = {}) { const conf = await this._invocationConfig([BlobCapabilities.add.can]) options.connection = this._serviceConf.upload - return Blob.add(conf, blob, options) + const bytes = new Uint8Array(await blob.arrayBuffer()) + const digest = await sha256.digest(bytes) + return { digest, ...(await Blob.add(conf, digest, bytes, options)) } } /** diff --git a/packages/w3up-client/test/capability/blob.test.js b/packages/w3up-client/test/capability/blob.test.js index a713f4756..1baacf1ff 100644 --- a/packages/w3up-client/test/capability/blob.test.js +++ b/packages/w3up-client/test/capability/blob.test.js @@ -33,16 +33,16 @@ export const BlobClient = Test.withContext({ const bytes = await randomBytes(128) const bytesHash = await sha256.digest(bytes) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) // TODO we should check blobsStorage as well - assert.deepEqual(await allocationsStorage.exists(space.did(), multihash), { + assert.deepEqual(await allocationsStorage.exists(space.did(), digest), { ok: true, }) - assert.deepEqual(multihash.bytes, bytesHash.bytes) + assert.deepEqual(digest.bytes, bytesHash.bytes) }, 'should list stored blobs': async ( assert, @@ -71,10 +71,10 @@ export const BlobClient = Test.withContext({ const bytes = await randomBytes(128) const bytesHash = await sha256.digest(bytes) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - assert.deepEqual(multihash.bytes, bytesHash.bytes) + assert.deepEqual(digest.bytes, bytesHash.bytes) const { results: [entry], @@ -109,11 +109,11 @@ export const BlobClient = Test.withContext({ }) const bytes = await randomBytes(128) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - const result = await alice.capability.blob.remove(multihash) + const result = await alice.capability.blob.remove(digest) assert.ok(result.ok) }, 'should get a stored blob': async ( @@ -142,11 +142,11 @@ export const BlobClient = Test.withContext({ }) const bytes = await randomBytes(128) - const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + const { digest } = await alice.capability.blob.add(new Blob([bytes]), { receiptsEndpoint, }) - const result = await alice.capability.blob.get(multihash) + const result = await alice.capability.blob.get(digest) assert.ok(result.ok) }, }) diff --git a/packages/w3up-client/test/capability/index.test.js b/packages/w3up-client/test/capability/index.test.js index 6a812330f..40196272c 100644 --- a/packages/w3up-client/test/capability/index.test.js +++ b/packages/w3up-client/test/capability/index.test.js @@ -29,16 +29,14 @@ export const IndexClient = Test.withContext({ const index = ShardedDAGIndex.create(car.cid) const indexBytes = Result.unwrap(await index.archive()) - const { multihash } = await alice.capability.blob.add( + const { digest } = await alice.capability.blob.add( new Blob([indexBytes]), { receiptsEndpoint, } ) - assert.ok( - await alice.capability.index.add(Link.create(CAR.code, multihash)) - ) + assert.ok(await alice.capability.index.add(Link.create(CAR.code, digest))) }, }, })