feat: add s3 backend

ckanext/files/assets/ts/files--modules.ts

@@ -61,12 +61,14 @@ ckan.module("files--resource-select", function ($) {
 ckan.module("files--auto-upload", function ($) {
     return {
         options: {
+            adapter: "Standard",
             spinner: null,
             action: null,
             successEvent: "files-file-created",
             errorEvent: "files-file-failed",
             eventTarget: null,
             copyIdInto: null,
+            requestParams: null,
         },
         queue: null,
 
@@ -90,14 +92,24 @@ ckan.module("files--auto-upload", function ($) {
                 .closest("form")
                 .find("input[type=submit],button[type=submit]");
             this.idTarget = $(this.options.copyIdInto);
+
+            this.uploader = this.sandbox.files.makeUploader(
+                this.options.adapter,
+                {
+                    uploadAction: this.options.action,
+                },
+            );
+
+            this.uploader.addEventListener("progress", console.log);
         },
 
         upload(...files: File[]) {
             files.forEach(async (file) => {
                 this.queue.add(file);
                 this.refreshFormState();
                 const options: ckan.CKANEXT_FILES.UploadOptions = {
-                    uploaderArgs: [{ uploadAction: this.options.action }],
+                    uploader: this.uploader,
+                    requestParams: {...this.options.requestParams, multipart: this.uploader instanceof ckan.CKANEXT_FILES.adapters.Multipart},
                 };
 
                 this.sandbox.files

ckanext/files/assets/ts/files--s3.ts

@@ -0,0 +1,65 @@
+namespace ckan {
+  export namespace CKANEXT_FILES {
+    export namespace adapters {
+      export type S3UploadInfo = UploadInfo & {
+        storage_data: StorageData & { upload_url: string };
+      };
+
+      export class S3Multipart extends Multipart {
+        async __x_uploadChunk(
+          info: S3UploadInfo,
+          part: Blob,
+          start: number,
+        ): Promise<UploadInfo> {
+          if (!part.size) {
+            throw new Error("0-length chunks are not allowed");
+          }
+
+            debugger;
+
+          const request = new XMLHttpRequest();
+
+          request.open("PUT", info.storage_data.upload_url);
+          request.send(part);
+
+          const resp: any = await new Promise((done, fail) => {
+            request.addEventListener("load", (event) => done(request));
+          });
+          let uploaded;
+
+          if ([200, 201].includes(resp.status)) {
+            uploaded = info.size;
+          } else {
+            throw new Error(await resp.responseText);
+          }
+
+          if (!Number.isInteger(uploaded)) {
+            throw new Error(`Invalid uploaded size ${uploaded}`);
+          }
+
+          return new Promise((done, fail) => {
+            this.sandbox.client.call(
+              "POST",
+              "files_multipart_update",
+              {
+                id: info.id,
+                uploaded,
+                  etag: resp.getResponseHeader("ETag")
+              },
+              (data: any) => {
+                done(data.result);
+              },
+              (resp: any) => {
+                fail(
+                  typeof resp.responseJSON === "string"
+                    ? resp.responseText
+                    : resp.responseJSON.error,
+                );
+              },
+            );
+          });
+        }
+      }
+    }
+  }
+}

ckanext/files/assets/ts/files--shared.ts

@@ -99,7 +99,9 @@ namespace ckan {
                 }
                 dispatchMultipartId(file: File, id: string) {
                     this.dispatchEvent(
-                        new CustomEvent("multipartid", { detail: { file, id } }),
+                        new CustomEvent("multipartid", {
+                            detail: { file, id },
+                        }),
                     );
                 }
                 dispatchProgress(file: File, loaded: number, total: number) {
@@ -205,8 +207,10 @@ namespace ckan {
             }
 
             export class Multipart extends Base {
-                static defaultSettings = { chunkSize: 1024 * 1024 * 5 };
-                protected initializeAction = "files_multipart_start";
+                static defaultSettings = {
+                    chunkSize: 1024 * 1024 * 5,
+                    uploadAction: "files_multipart_start",
+                };
 
                 private _active = new Set<File>();
 
@@ -271,6 +275,13 @@ namespace ckan {
                             info,
                             file.slice(start, start + this.settings.chunkSize),
                             start,
+                            {
+                                progressData: {
+                                    file,
+                                    uploaded: info.storage_data.uploaded,
+                                    size: file.size,
+                                },
+                            },
                         );
 
                         const uploaded = info.storage_data.uploaded;
@@ -295,21 +306,25 @@ namespace ckan {
                         return;
                     }
                     this.dispatchFinish(file, info);
-                    return info
+                    return info;
                 }
 
-                _initializeUpload(file: File, params: {[key: string]: any}): Promise<UploadInfo> {
+                _initializeUpload(
+                    file: File,
+                    params: { [key: string]: any },
+                ): Promise<UploadInfo> {
                     return new Promise((done, fail) =>
                         this.sandbox.client.call(
                             "POST",
-                            this.initializeAction,
+                            this.settings.uploadAction,
                             Object.assign(
                                 {},
                                 {
                                     storage: this.settings.storage,
                                     name: file.name,
                                     size: file.size,
-                                    content_type: file.type,
+                                    content_type:
+                                        file.type || "application/octet-stream",
                                 },
                                 params,
                             ),
@@ -351,13 +366,29 @@ namespace ckan {
                     info: UploadInfo,
                     part: Blob,
                     start: number,
+                    extras: any = {},
                 ): Promise<UploadInfo> {
                     if (!part.size) {
                         throw new Error("0-length chunks are not allowed");
                     }
                     const request = new XMLHttpRequest();
 
                     const result = new Promise<UploadInfo>((done, fail) => {
+                        if (extras["progressData"]) {
+                            const { file, uploaded, size } =
+                                extras["progressData"];
+                            request.upload.addEventListener(
+                                "progress",
+                                (event) => {
+                                    this.dispatchProgress(
+                                        file,
+                                        uploaded + event.loaded,
+                                        size,
+                                    );
+                                },
+                            );
+                        }
+
                         request.addEventListener("load", (event) => {
                             const result = JSON.parse(request.responseText);
                             if (result.success) {

ckanext/files/assets/webassets.yml

@@ -5,6 +5,7 @@ files-js:
     - scripts/files--shared.js
     - scripts/files--modules.js
     - scripts/files--google-cloud-storage-uploader.js
+    - scripts/files--s3.js
 
   extra:
     preload:

ckanext/files/helpers.py

@@ -60,7 +60,9 @@ def files_content_type_icon(
     return None
 
 
-def files_get_file(file_id: str) -> shared.File | None:
+def files_get_file(file_id: str | None) -> shared.File | None:
+    if not file_id:
+        return None
     return model.Session.get(shared.File, file_id)
 
 

ckanext/files/logic/action.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import copy
-
 from typing import Any, cast
 
 import sqlalchemy as sa
@@ -712,10 +711,16 @@ def files_multipart_complete(
         data.update(fileobj.plugin_data)
         fileobj.plugin_data = data
 
-    sess.query(Owner).where(
-        Owner.item_type == "multipart",
-        Owner.item_id == multipart.id,
-    ).update({"item_id": fileobj.id, "item_type": "file"})
+    if owner_info := multipart.owner_info:
+        sess.add(
+            Owner(
+                item_id=fileobj.id,
+                item_type="file",
+                owner_id=owner_info.owner_id,
+                owner_type=owner_info.owner_type,
+            )
+        )
+
     sess.add(fileobj)
     sess.delete(multipart)
     sess.commit()
@@ -906,7 +911,9 @@ def files_resource_upload(
     and always uses resources storage.
 
     New file is not attached to resource. You need to call
-    `files_transfer_ownership` manually, when resource created.
+    `files_transfer_ownership` manually, when resource created. Or you can use
+    `files_transfer_ownership("resource","id")` validator to do it
+    automatically.
 
     Args:
         name (str): human-readable name of the file.
@@ -916,6 +923,7 @@ def files_resource_upload(
 
     Returns:
         dictionary with file details
+
     """
     tk.check_access("files_resource_upload", context, data_dict)
     storage_name = shared.config.resources_storage()
@@ -925,7 +933,9 @@ def files_resource_upload(
         )
 
     # TODO: pull cache from the context
-    return tk.get_action("files_file_create")(
+    return tk.get_action(
+        "files_multipart_start" if data_dict["multipart"] else "files_file_create"
+    )(
         Context(context, ignore_auth=True),
         dict(data_dict, storage=storage_name),
     )

ckanext/files/logic/auth.py

@@ -290,7 +290,12 @@ def files_resource_upload(context: Context, data_dict: dict[str, Any]) -> AuthRe
     except shared.exc.UnknownStorageError:
         return {"success": False}
 
-    return {"success": True}
+    if data_dict.get("resource_id"):
+        return authz.is_authorized(
+            "resource_update", context, {"id": data_dict["resource_id"]}
+        )
+
+    return authz.is_authorized("resource_create", context, data_dict)
 
 
 @tk.auth_disallow_anonymous_access

ckanext/files/logic/schema.py

@@ -131,11 +131,13 @@ def multipart_update(not_empty: Validator, unicode_safe: Validator) -> Schema:
 
 
 @validator_args
-def multipart_complete(not_empty: Validator, unicode_safe: Validator, boolean_validator: Validator) -> Schema:
+def multipart_complete(
+    not_empty: Validator, unicode_safe: Validator, boolean_validator: Validator
+) -> Schema:
     return {
         "id": [not_empty, unicode_safe],
         "keep_storage_data": [boolean_validator],
-        "keep_plugin_data": [boolean_validator]
+        "keep_plugin_data": [boolean_validator],
     }
 
 
@@ -194,11 +196,18 @@ def file_unpin(
 
 
 @validator_args
-def resource_upload(ignore: Validator) -> Schema:
-    schema = file_create()
-    schema["storage"] = [ignore]
-    schema["__extras"] = [ignore]
-    return schema
+def resource_upload(
+    keep_extras: Validator,
+    unicode_safe: Validator,
+    boolean_validator: Validator,
+    ignore_missing: Validator,
+) -> Schema:
+    return {
+        "multipart": [boolean_validator],
+        "resource_id": [ignore_missing, unicode_safe],
+        "package_id": [ignore_missing, unicode_safe],
+        "__extras": [keep_extras],
+    }
 
 
 @validator_args