feat: S3Compatible storage adapter

src/Storage/Device/S3.php

@@ -136,6 +136,11 @@ class S3 extends Device
      */
     protected array $amzHeaders;
 
+    /**
+     * @var bool
+     */
+    protected bool $vhost;
+
     /**
      * S3 Constructor
      *
@@ -155,6 +160,7 @@ public function __construct(string $root, string $accessKey, string $secretKey,
         $this->root = $root;
         $this->acl = $acl;
         $this->amzHeaders = [];
+        $this->vhost = true;
 
         $host = match ($region) {
             self::CN_NORTH_1, self::CN_NORTH_4, self::CN_NORTHWEST_1 => $bucket.'.s3.'.$region.'.amazonaws.cn',
@@ -196,6 +202,14 @@ public function getRoot(): string
         return $this->root;
     }
 
+    /**
+     * @return string
+     */
+    public function getBucket(): string
+    {
+        return $this->bucket;
+    }
+
     /**
      * @param  string  $filename
      * @param  string|null  $prefix
@@ -438,6 +452,17 @@ public function delete(string $path, bool $recursive = false): bool
         return true;
     }
 
+    private function makeUri(): string
+    {
+        $base = '/';
+
+        if ($this->vhost) {
+            return $base;
+        } else {
+            return $base.$this->getBucket();
+        }
+    }
+
     /**
      * Get list of objects in the given path.
      *
@@ -448,16 +473,19 @@ public function delete(string $path, bool $recursive = false): bool
      */
     private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '')
     {
-        $uri = '/';
-        $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */
+        $uri = $this->makeUri();
         $this->headers['content-type'] = 'text/plain';
         $this->headers['content-md5'] = \base64_encode(md5('', true));
 
         $parameters = [
             'list-type' => 2,
-            'prefix' => $prefix,
             'max-keys' => $maxKeys,
         ];
+
+        if ($this->vhost) {
+            $parameters['prefix'] = \ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */
+        }
+
         if (! empty($continuationToken)) {
             $parameters['continuation-token'] = $continuationToken;
         }
@@ -477,8 +505,8 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken =
     public function deletePath(string $path): bool
     {
         $path = $this->getRoot().'/'.$path;
+        $uri = $this->makeUri();
 
-        $uri = '/';
         $continuationToken = '';
         do {
             $objects = $this->listObjects($path, continuationToken: $continuationToken);

src/Storage/Device/S3Compatible.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace Utopia\Storage\Device;
+
+use Utopia\Storage\Storage;
+
+/**
+ * Adapter for S3-Compatible storage providers
+ *
+ * Endpoint URL is explicitly defined by the caller due to providers allowing
+ * path-style or vhost-style endpoints as well as extra parameters such as
+ * client ID.
+ */
+class S3Compatible extends S3
+{
+    /**
+     * @var string
+     */
+    protected string $endpoint;
+
+    /**
+     * S3Compatible Constructor
+     *
+     * @param  string  $endpoint
+     * @param  string  $root
+     * @param  string  $accessKey
+     * @param  string  $secretKey
+     * @param  string  $bucket
+     * @param  bool  $vhost
+     * @param  string  $region
+     * @param  string  $acl
+     */
+    public function __construct(string $endpoint, string $root, string $accessKey, string $secretKey, string $bucket, bool $vhost, string $region = self::US_EAST_1, string $acl = self::ACL_PRIVATE)
+    {
+        if (! $vhost) {
+            $root = $bucket.$root;
+        }
+        parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl);
+
+        $this->endpoint = $endpoint;
+        $this->vhost = $vhost;
+
+        /**
+         * Workaround to prevent having to do a refactor of
+         * the S3 class along with adapter addition. Class only
+         * supports https for the time being.
+         *
+         * There are multiple endpoint styles dependent upon the provider
+         * used. Examples are:
+         * <scheme>://<endpoint.url>/<bucket>
+         * <scheme>://<clientid.endpoint.url>/<bucket>
+         * <scheme>://<bucketvhost.endpoint.url>
+         */
+        $endpoint = \preg_replace('/^https?:\/\//i', '', $endpoint);
+        $this->headers['host'] = $endpoint;
+    }
+
+    /**
+     * @return string
+     */
+    public function getName(): string
+    {
+        return 'S3-Compatible Storage';
+    }
+
+    /**
+     * @return string
+     */
+    public function getType(): string
+    {
+        return Storage::DEVICE_S3COMPATIBLE;
+    }
+
+    /**
+     * @return string
+     */
+    public function getDescription(): string
+    {
+        return 'Generic Connector For S3-Compatible Storage Providers';
+    }
+}

src/Storage/Storage.php

@@ -21,6 +21,8 @@ class Storage
 
     const DEVICE_LINODE = 'linode';
 
+    const DEVICE_S3COMPATIBLE = 's3compatible';
+
     /**
      * Devices.
      *

tests/Storage/Device/S3CompatibleTest.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Utopia\Tests\Storage\Device;
+
+use Utopia\Storage\Device\S3Compatible;
+use Utopia\Tests\Storage\S3Base;
+
+class S3CompatibleTest extends S3Base
+{
+    protected function init(): void
+    {
+        $root = '/root';
+        $endpoint = $_SERVER['S3COMPATIBLE_ENDPOINT'] ?? '';
+        $vhost = $_SERVER['S3COMPATIBLE_VHOST'] === 'true';
+        $this->vhost = $vhost;
+        $key = $_SERVER['S3COMPATIBLE_ACCESS_KEY'] ?? '';
+        $secret = $_SERVER['S3COMPATIBLE_SECRET'] ?? '';
+        $bucket = $_SERVER['S3COMPATIBLE_BUCKET'] ?? 'appwrite-test-bucket';
+        $region = $_SERVER['S3COMPATIBLE_REGION'] ?? '';
+
+        if ($vhost) {
+            $this->root = $root;
+        } else {
+            $this->root = $bucket.$root;
+        }
+
+        $this->object = new S3Compatible($endpoint, $root, $key, $secret, $bucket, $vhost, $region);
+    }
+
+    /**
+     * @return string
+     */
+    public function getAdapterName(): string
+    {
+        return $this->object->getName();
+    }
+
+    /**
+     * @return string
+     */
+    public function getAdapterType(): string
+    {
+        return $this->object->getType();
+    }
+
+    /**
+     * @return string
+     */
+    public function getAdapterDescription(): string
+    {
+        return $this->object->getDescription();
+    }
+}

tests/Storage/S3Base.php

@@ -29,6 +29,11 @@ abstract protected function getAdapterDescription(): string;
      */
     protected $root = '/root';
 
+    /**
+     * @var bool
+     */
+    protected $vhost = true;
+
     public function setUp(): void
     {
         $this->init();
@@ -138,12 +143,18 @@ public function testXMLUpload()
 
     public function testDeletePath()
     {
+        if ($this->vhost) {
+            $dpParam = 'bucket';
+        } else {
+            $dpParam = '';
+        }
+
         // Test Single Object
         $path = $this->object->getPath('text-for-delete-path.txt');
         $path = str_ireplace($this->object->getRoot(), $this->object->getRoot().DIRECTORY_SEPARATOR.'bucket', $path);
         $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain'));
         $this->assertEquals(true, $this->object->exists($path));
-        $this->assertEquals(true, $this->object->deletePath('bucket'));
+        $this->assertEquals(true, $this->object->deletePath($dpParam));
         $this->assertEquals(false, $this->object->exists($path));
 
         // Test Multiple Objects
@@ -157,7 +168,7 @@ public function testDeletePath()
         $this->assertEquals(true, $this->object->write($path2, 'Hello World', 'text/plain'));
         $this->assertEquals(true, $this->object->exists($path2));
 
-        $this->assertEquals(true, $this->object->deletePath('bucket'));
+        $this->assertEquals(true, $this->object->deletePath($dpParam));
         $this->assertEquals(false, $this->object->exists($path));
         $this->assertEquals(false, $this->object->exists($path2));
     }