From cd5dab6643c47e14245ca215649594375a19aa7b Mon Sep 17 00:00:00 2001 From: dantb Date: Wed, 19 Jun 2024 12:04:06 +0200 Subject: [PATCH] Delegation journey documentation (#5025) * Delegation journey documentation * Fix reference * Add request/response examples and edit --------- Co-authored-by: Simon --- .../assets/files/delegate-create-post.json | 40 +++++ .../api/assets/files/delegate-create-post.sh | 8 + .../assets/files/delegate-validate-post.json | 5 + .../assets/files/delegate-validate-post.sh | 15 ++ .../delta/api/assets/files/register-post.json | 34 +++++ .../delta/api/assets/files/register-post.sh | 15 ++ .../main/paradox/docs/delta/api/files-api.md | 140 ++++++++++++++++-- 7 files changed, 242 insertions(+), 15 deletions(-) create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.sh create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.sh create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/register-post.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/register-post.sh diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.json new file mode 100644 index 0000000000..9846620dad --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.json @@ -0,0 +1,40 @@ +{ + "@context" : [ + "https://bluebrain.github.io/nexus/contexts/files.json", + "https://bluebrain.github.io/nexus/contexts/metadata.json" + ], + "@id" : "http://delta:8080/v1/resources/fcp5xcw67hittxm/loeu5kgu48i8tw7/_/1205b629-2a73-4891-abff-140a64e9c391", + "@type" : "File", + "description" : "a description of the file", + "name" : "My File", + "_bytes" : 29625, + "_constrainedBy" : "https://bluebrain.github.io/nexus/schemas/files.json", + "_createdAt" : "2024-06-14T12:53:38.516222Z", + "_createdBy" : "http://delta:8080/v1/realms/test-pmcfwjobxkuvebro/users/fwhbqlnnhqncpsqk", + "_deprecated" : false, + "_digest" : { + "_algorithm" : "SHA-256", + "_value" : "05bf442810213b9e5fecd5242eefeff1f3d207913861c96658c75ccf58997e87" + }, + "_filename" : "myfile.png", + "_incoming" : "http://delta:8080/v1/files/fcp5xcw67hittxm/loeu5kgu48i8tw7/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Ffcp5xcw67hittxm%2Floeu5kgu48i8tw7%2F_%2F1205b629-2a73-4891-abff-140a64e9c391/incoming", + "_keywords" : { + "key1" : "value1", + "key2" : "value2" + }, + "_location" : "myprefix/fcp5xcw67hittxm/loeu5kgu48i8tw7/files/f/f/9/5/3/4/c/2/ngbeydtjyibvcanj", + "_mediaType" : "image/png", + "_origin" : "External", + "_outgoing" : "http://delta:8080/v1/files/fcp5xcw67hittxm/loeu5kgu48i8tw7/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Ffcp5xcw67hittxm%2Floeu5kgu48i8tw7%2F_%2F1205b629-2a73-4891-abff-140a64e9c391/outgoing", + "_project" : "http://delta:8080/v1/projects/fcp5xcw67hittxm/loeu5kgu48i8tw7", + "_rev" : 1, + "_self" : "http://delta:8080/v1/files/fcp5xcw67hittxm/loeu5kgu48i8tw7/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Ffcp5xcw67hittxm%2Floeu5kgu48i8tw7%2F_%2F1205b629-2a73-4891-abff-140a64e9c391", + "_storage" : { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/mys3storage", + "@type" : "S3Storage", + "_rev" : 3 + }, + "_updatedAt" : "2024-06-14T12:53:38.516222Z", + "_updatedBy" : "http://delta:8080/v1/realms/test-pmcfwjobxkuvebro/users/fwhbqlnnhqncpsqk", + "_uuid" : "c96a229c-8d1d-46a5-8f47-08555fd98cb3" +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.sh b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.sh new file mode 100644 index 0000000000..17f3770be1 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-create-post.sh @@ -0,0 +1,8 @@ +curl -X POST \ + -H "Content-Type: application/json" \ + "http://localhost:8080/v1/delegate/files/myorg/myproject?storage=mys3storage" -d \ + '{ + "protected" : "eyJleHAiOjE3MTg2Mjg1NDUsImFsZyI6IlJTMjU2In0", + "payload" : "eyJidWNrZXQiOiJqNDc4M2NjNWFkY2F1a3UiLCJwYXRoIjoibXlwcmVmaXgvYTE0bTYxZWgzajVuNzVjLzhrbXlpb2QyMzM1MnhpZC9maWxlcy9iLzIvOS80LzMvYy9jLzYvaG5qeXR3b2x0ZnVyc2JlaCIsIm1lZGlhVHlwZSI6ImltYWdlL2RhbiIsIm1ldGFkYXRhIjp7ImRlc2NyaXB0aW9uIjoicnByZndzdWpveGJpaXllaSIsImtleXdvcmRzIjp7ImtvZXZvd3Bsc3Jld2NlZGIiOiJ2dnNlaW9wa3Z5Y3d1dG1oIn0sIm5hbWUiOiJmbW94dHh3a3JvdWhtZG90In0sImlkIjoiaHR0cDovL2RlbHRhOjgwODAvdjEvcmVzb3VyY2VzL2ExNG02MWVoM2o1bjc1Yy84a215aW9kMjMzNTJ4aWQvXy9hNGE5ODE2Mi1jYTNmLTQ2YmUtYWUxMC00MzJjYTZmOTdjNGYifQ", + "signature" : "htQLymfQIrks7MejErb4v3mnhQT2W6iPZdkd7LVBPJ0Ksybj8XG8dbTJH5pjZJF7-HXi848R14tquZ6iSeXpEqFGiZYge8obPQRLpJA0qbc9Mmhlq-CTbIdsy5OFpdzcDadSj6_k_kzuU2PR-Fli9GtH-34z2d4C9dWsBmnUo_IA3dvSFCF_PaQuajo7cJYa_0yc4VVGKG-xYi9yV4ylD5D2cxMUDFun78NOKDD_2upF-kuf9t5E-NjCl0DffkelbqYuH6nMop2zmwfu-cwHnChaDwKM7HLJGLomD5duU5sq-mVsunnMy58NgzMecLGDbER-27zk7w0TwxkXTKfxWg" + }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.json new file mode 100644 index 0000000000..7ab32b0f00 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.json @@ -0,0 +1,5 @@ +{ + "protected" : "eyJleHAiOjE3MTg2Mjg1NDUsImFsZyI6IlJTMjU2In0", + "payload" : "eyJidWNrZXQiOiJqNDc4M2NjNWFkY2F1a3UiLCJwYXRoIjoibXlwcmVmaXgvYTE0bTYxZWgzajVuNzVjLzhrbXlpb2QyMzM1MnhpZC9maWxlcy9iLzIvOS80LzMvYy9jLzYvaG5qeXR3b2x0ZnVyc2JlaCIsIm1lZGlhVHlwZSI6ImltYWdlL2RhbiIsIm1ldGFkYXRhIjp7ImRlc2NyaXB0aW9uIjoicnByZndzdWpveGJpaXllaSIsImtleXdvcmRzIjp7ImtvZXZvd3Bsc3Jld2NlZGIiOiJ2dnNlaW9wa3Z5Y3d1dG1oIn0sIm5hbWUiOiJmbW94dHh3a3JvdWhtZG90In0sImlkIjoiaHR0cDovL2RlbHRhOjgwODAvdjEvcmVzb3VyY2VzL2ExNG02MWVoM2o1bjc1Yy84a215aW9kMjMzNTJ4aWQvXy9hNGE5ODE2Mi1jYTNmLTQ2YmUtYWUxMC00MzJjYTZmOTdjNGYifQ", + "signature" : "htQLymfQIrks7MejErb4v3mnhQT2W6iPZdkd7LVBPJ0Ksybj8XG8dbTJH5pjZJF7-HXi848R14tquZ6iSeXpEqFGiZYge8obPQRLpJA0qbc9Mmhlq-CTbIdsy5OFpdzcDadSj6_k_kzuU2PR-Fli9GtH-34z2d4C9dWsBmnUo_IA3dvSFCF_PaQuajo7cJYa_0yc4VVGKG-xYi9yV4ylD5D2cxMUDFun78NOKDD_2upF-kuf9t5E-NjCl0DffkelbqYuH6nMop2zmwfu-cwHnChaDwKM7HLJGLomD5duU5sq-mVsunnMy58NgzMecLGDbER-27zk7w0TwxkXTKfxWg" +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.sh b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.sh new file mode 100644 index 0000000000..a4c5156a07 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/delegate-validate-post.sh @@ -0,0 +1,15 @@ +curl -X POST \ + -H "Content-Type: application/json" \ + "http://localhost:8080/v1/delegate/files/myorg/myproject/validate?storage=mys3storage" -d \ + '{ + "filename": "myfile.png", + "mediaType": "image/png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } + }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/register-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/register-post.json new file mode 100644 index 0000000000..76a8db3850 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/register-post.json @@ -0,0 +1,34 @@ +{ + "@context" : [ + "https://bluebrain.github.io/nexus/contexts/files.json", + "https://bluebrain.github.io/nexus/contexts/metadata.json" + ], + "@id" : "http://delta:8080/v1/resources/hkrxvoxdyiiev1p/03vctrp70usfehq/_/iwa41vwspwke6nx", + "@type" : "File", + "_bytes" : 29625, + "_constrainedBy" : "https://bluebrain.github.io/nexus/schemas/files.json", + "_createdAt" : "2024-06-14T12:44:36.525177Z", + "_createdBy" : "http://delta:8080/v1/realms/test-vuaplrsvrbkpkhca/users/byfxikrrdlmmvsvv", + "_deprecated" : false, + "_digest" : { + "_algorithm" : "SHA-256", + "_value" : "05bf442810213b9e5fecd5242eefeff1f3d207913861c96658c75ccf58997e87" + }, + "_filename" : "myfile.png", + "_incoming" : "http://delta:8080/v1/files/hkrxvoxdyiiev1p/03vctrp70usfehq/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Fhkrxvoxdyiiev1p%2F03vctrp70usfehq%2F_%2Fiwa41vwspwke6nx/incoming", + "_location" : "relative/path/to/myfile.png", + "_mediaType" : "image/png", + "_origin" : "External", + "_outgoing" : "http://delta:8080/v1/files/hkrxvoxdyiiev1p/03vctrp70usfehq/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Fhkrxvoxdyiiev1p%2F03vctrp70usfehq%2F_%2Fiwa41vwspwke6nx/outgoing", + "_project" : "http://delta:8080/v1/projects/hkrxvoxdyiiev1p/03vctrp70usfehq", + "_rev" : 1, + "_self" : "http://delta:8080/v1/files/hkrxvoxdyiiev1p/03vctrp70usfehq/http:%2F%2Fdelta:8080%2Fv1%2Fresources%2Fhkrxvoxdyiiev1p%2F03vctrp70usfehq%2F_%2Fiwa41vwspwke6nx", + "_storage" : { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/mys3storage", + "@type" : "S3Storage", + "_rev" : 3 + }, + "_updatedAt" : "2024-06-14T12:44:36.525177Z", + "_updatedBy" : "http://delta:8080/v1/realms/test-vuaplrsvrbkpkhca/users/byfxikrrdlmmvsvv", + "_uuid" : "79695062-ecbb-42dc-a62d-61c2c02be129" +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/register-post.sh b/docs/src/main/paradox/docs/delta/api/assets/files/register-post.sh new file mode 100644 index 0000000000..4174eb1e72 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/register-post.sh @@ -0,0 +1,15 @@ +curl -X POST \ + -H "Content-Type: application/json" \ + "http://localhost:8080/v1/files/myorg/myproject/register/myfile?storage=mys3storage" -d \ + '{ + "path": "relative/path/to/myfile.png", + "mediaType": "image/png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } + }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index da322fc47e..44bdd99479 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -32,6 +32,15 @@ When using the endpoints described on this page, the responses will contain glob - `_incoming`: address to query to obtain the @ref:[list of incoming links](resources-api.md#list-incoming-links) - `_outgoing`: address to query to obtain the @ref:[list of outgoing links](resources-api.md#list-outgoing-links) +## Custom file metadata + +When creating file resources, users can optionally provide custom metadata to be indexed and therefore searchable. + +This takes the form of a `metadata` field containing a JSON Object with one or more of the following fields: + +- `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. +- `description`: a string that describes the file. It will be indexed in the full-text search. +- `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. ## Indexing @@ -52,10 +61,7 @@ This part can contain the following disposition parameters: **Headers:** -- `x-nxs-file-metadata`: an optional JSON object containing one or more of the following fields: - - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. - - `description`: a string that describes the file. It will be indexed in the full-text search. - - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. +- `x-nxs-file-metadata`: an optional JSON object containing one or more of fields described in @ref:[custom file metadata](#custom-file-metadata). - `x-nxs-file-content-length`: the size of the uploaded file: - mandatory to upload to a S3 storage - ignored for other types of storage @@ -127,10 +133,7 @@ POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. - `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. -- `{metadata}`: JSON Object - optional object containing one or more of the following fields: - - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. - - `description`: a string that describes the file. It will be indexed in the full-text search. - - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. +- `{metadata}`: JSON Object - Optional, see @ref:[custom file metadata](#custom-file-metadata). **Example** @@ -167,10 +170,7 @@ When not specified, the default storage of the project is used. - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. - `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. -- `{metadata}`: JSON Object - optional object containing one or more of the following fields: - - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. - - `description`: a string that describes the file. It will be indexed in the full-text search. - - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. +- `{metadata}`: JSON Object - Optional, see @ref:[custom file metadata](#custom-file-metadata). **Example** @@ -437,11 +437,121 @@ Request Response : @@snip [listed.json](assets/files/listed.json) -## Delegation & Registration -To support files stored in the cloud, delta allows users to register files already uploaded to S3. This is useful primarily for large files where uploading directly through delta is inefficient and expensive. +## Delegation & Registration (S3 only) +To support files stored in the cloud, Delta allows users to register files already uploaded to S3. This is useful primarily for large files where uploading directly through Delta using HTTP is inefficient and expensive. + +There are two use cases: registering an already uploaded file by specifying its path, and asking Delta to generate a path in its standard format. + +### Register external file + +This endpoint accepts a path and creates a new file resource based on an existing S3 file. + +``` +POST /v1/files/{org_label}/{project_label}/register/{file_id}?storage={storageId}&tag={tagName} + { + "path": "{path}", + "mediaType": "{mediaType}", + "metadata": {metadata} + } +``` + +... where + +- `{path}`: String - the relative path to the file from the root of S3. +- `{mediaType}`: String - Optional @link:[MIME](https://en.wikipedia.org/wiki/MIME){ open=new } specifying the file type. If omitted this will be inferred by S3. +- `{metadata}`: JSON Object - Optional, see @ref:[custom file metadata](#custom-file-metadata). + +**Example** + +Request +: @@snip [register-post.sh](assets/files/register-post.sh) + +Response +: @@snip [register-post.json](assets/files/register-post.json) + +### Delegating file uploads + +Users can use delegation for files that cannot be uploaded through Delta (e.g. large files). Here Delta will provide bucket and path details for the upload. Users are then expected to upload the file using other methods, and call back to Delta to register this file with the same metadata that was initially validated. The three steps are outlined in detail below. + +#### 1. Validate and generate path for file delegation + +Delta accepts and validates the following payload. + +``` +POST /v1/delegate/files/{org_label}/{project_label}/validate?storage={storageId} + { + "filename": "{filename}", + "mediaType": "{mediaType}", + "metadata": {metadata} + } +``` + +... where + +- `{filename}`: String - mandatory name given to the file within the generated path. +- `{mediaType}`: String - Optional @link:[MIME](https://en.wikipedia.org/wiki/MIME){ open=new } specifying the file type. If omitted this will be inferred by S3. +- `{metadata}`: JSON Object - Optional, see @ref:[custom file metadata](#custom-file-metadata). + +It then generates the following details for the file: + +```json + { + "bucket": "", + "id": "", + "path": "", + "mediaType": "", + "metadata": {} + } +``` + +The user is expected to upload their file to `path` within `bucket`. The `id` is reserved for when the file resource is created. `mediaType` and `metadata` are what the user specified in the request. + +This payload is then signed using the [flattened JWS serialization format](https://datatracker.ietf.org/doc/html/rfc7515#section-7.2.2) to ensure that Delta has validated and generated the correct data. This same payload will be passed when creating the file resource. + +```json + { + "payload": "", + "protected": "", + "signature": "" + } +``` + +The `payload` field can be base64 decoded to access the generated file details. Note that `protected` contains an expiry field `exp` with the datetime at which this signature will expire (in epoch seconds). To view this can also be base64 decoded. + +**Example** -There are two use cases: registering an already uploaded file by specifying its path, and asking Delta to generate a path with its standard format. +Request +: @@snip [delegate-validate-post.sh](assets/files/delegate-validate-post.sh) +Response +: @@snip [delegate-validate-post.json](assets/files/delegate-validate-post.json) + +#### 2. Upload file to S3 + +Using the bucket and path from the previous step, the file should be uploaded to S3 by whatever means are appropriate. The only restriction is that this must be finished before the expiry datetime of the signed payload. + +#### 3. Create delegated file resource + +Once the file has been uploaded to S3 at the specified path, the file resource can be created. The payload can be passed back exactly as it was returned in the previous step. + +``` +POST /v1/delegate/files/{org_label}/{project_label}?storage={storageId} + { + "payload": "", + "protected": "", + "signature": "" + } +``` + +Delta will verify that the signature matches the payload and that the expiry date is not passed. Then the file will be registered as a resource. The usual file resource response will be returned with all the standard metadata and file location details. + +**Example** + +Request +: @@snip [delegate-create-post.sh](assets/files/delegate-create-post.sh) + +Response +: @@snip [delegate-create-post.json](assets/files/delegate-create-post.json) ## Server Sent Events