Merge pull request #41914 from nextcloud/s3-copy-size-limit

only do a multipart s3 copy when above the regular copy limit
nextcloud · Dec 29, 2023 · 77cec80 · 77cec80
2 parents cdc2723 + e405437
commit 77cec80
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 11 deletions.
diff --git a/apps/files_external/tests/Storage/Amazons3MultiPartTest.php b/apps/files_external/tests/Storage/Amazons3MultiPartTest.php
@@ -0,0 +1,68 @@
+<?php
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @author Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\files_external\tests\Storage;
+
+use OCA\Files_External\Lib\Storage\AmazonS3;
+
+/**
+ * Class Amazons3Test
+ *
+ * @group DB
+ *
+ * @package OCA\Files_External\Tests\Storage
+ */
+class Amazons3MultiPartTest extends \Test\Files\Storage\Storage {
+	private $config;
+	/** @var AmazonS3 */
+	protected $instance;
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$this->config = include('files_external/tests/config.amazons3.php');
+		if (! is_array($this->config) or ! $this->config['run']) {
+			$this->markTestSkipped('AmazonS3 backend not configured');
+		}
+		$this->instance = new AmazonS3($this->config + [
+			'putSizeLimit' => 1,
+			'copySizeLimit' => 1,
+		]);
+	}
+
+	protected function tearDown(): void {
+		if ($this->instance) {
+			$this->instance->rmdir('');
+		}
+
+		parent::tearDown();
+	}
+
+	public function testStat() {
+		$this->markTestSkipped('S3 doesn\'t update the parents folder mtime');
+	}
+
+	public function testHashInFileName() {
+		$this->markTestSkipped('Localstack has a bug with hashes in filename');
+	}
+}
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -71,6 +71,11 @@ trait S3ConnectionTrait {
 	/** @var int */
 	private $putSizeLimit;
 
+	/** @var int */
+	private $copySizeLimit;
+
+	private bool $useMultipartCopy = true;
+
 	protected $test;
 
 	protected function parseParams($params) {
@@ -87,6 +92,8 @@ protected function parseParams($params) {
 		$this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
 		$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
 		$this->putSizeLimit = $params['putSizeLimit'] ?? 104857600;
+		$this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000;
+		$this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true);
 		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
 		$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
 		if (!isset($params['port']) || $params['port'] === '') {

diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -196,16 +196,24 @@ public function copyObject($from, $to, array $options = []) {
 			'Key' => $from,
 		] + $this->getSSECParameters());
 
-		$copy = new MultipartCopy($this->getConnection(), [
-			"source_bucket" => $this->getBucket(),
-			"source_key" => $from
-		], array_merge([
-			"bucket" => $this->getBucket(),
-			"key" => $to,
-			"acl" => "private",
-			"params" => $this->getSSECParameters() + $this->getSSECParameters(true),
-			"source_metadata" => $sourceMetadata
-		], $options));
-		$copy->copy();
+		$size = (int)($sourceMetadata->get('Size') ?? $sourceMetadata->get('ContentLength'));
+
+		if ($this->useMultipartCopy && $size > $this->copySizeLimit) {
+			$copy = new MultipartCopy($this->getConnection(), [
+				"source_bucket" => $this->getBucket(),
+				"source_key" => $from
+			], array_merge([
+				"bucket" => $this->getBucket(),
+				"key" => $to,
+				"acl" => "private",
+				"params" => $this->getSSECParameters() + $this->getSSECParameters(true),
+				"source_metadata" => $sourceMetadata
+			], $options));
+			$copy->copy();
+		} else {
+			$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([
+				'params' => $this->getSSECParameters() + $this->getSSECParameters(true)
+			], $options));
+		}
 	}
 }