diff --git a/.github/workflows/sftp.yml b/.github/workflows/sftp.yml new file mode 100644 index 0000000000000..939372a53d20d --- /dev/null +++ b/.github/workflows/sftp.yml @@ -0,0 +1,75 @@ +name: SFTP unit tests +on: + push: + branches: + - master + - stable* + paths: + - 'apps/files_external/**' + pull_request: + paths: + - 'apps/files_external/**' + +env: + APP_NAME: files_external + +jobs: + sftp-tests: + runs-on: ubuntu-latest + + if: ${{ github.repository_owner != 'nextcloud-gmbh' }} + + strategy: + # do not stop on another job's failure + fail-fast: false + matrix: + php-versions: ['8.0'] + sftpd: ['openssh'] + + name: php${{ matrix.php-versions }}-${{ matrix.sftpd }} + + steps: + - name: Checkout server + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up sftpd + run: | + sudo mkdir /tmp/sftp + sudo chown -R 0777 /tmp/sftp + if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 + with: + php-version: ${{ matrix.php-versions }} + tools: phpunit:9 + extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + run: | + mkdir data + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable --force ${{ env.APP_NAME }} + php -S localhost:8080 & + - name: PHPUnit + run: | + echo " true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/${{ env.APP_NAME }}/tests/config.sftp.php + phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/SftpTest.php + - name: sftpd logs + if: always() + run: | + ls -l /tmp/sftp + docker logs sftp + + sftp-summary: + runs-on: ubuntu-latest + needs: sftp-tests + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.sftp-tests.result != 'success' }}; then exit 1; fi diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index 62184c733f284..aa2396ec6786a 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -497,7 +497,7 @@ public function fopen($path, $mode) { try { return $this->readObject($path); - } catch (S3Exception $e) { + } catch (\Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, diff --git a/apps/files_external/lib/Lib/Storage/SFTP.php b/apps/files_external/lib/Lib/Storage/SFTP.php index e46f60d0be47b..80b1472ce6664 100644 --- a/apps/files_external/lib/Lib/Storage/SFTP.php +++ b/apps/files_external/lib/Lib/Storage/SFTP.php @@ -36,15 +36,21 @@ */ namespace OCA\Files_External\Lib\Storage; +use Icewind\Streams\CountWrapper; use Icewind\Streams\IteratorDirectory; use Icewind\Streams\RetryWrapper; +use OC\Files\Filesystem; +use OC\Files\Storage\Common; +use OCP\Constants; +use OCP\Files\FileInfo; +use OCP\Files\IMimeTypeDetector; use phpseclib\Net\SFTP\Stream; /** * Uses phpseclib's Net\SFTP class and the Net\SFTP\Stream stream wrapper to * provide access to SFTP servers. */ -class SFTP extends \OC\Files\Storage\Common { +class SFTP extends Common { private $host; private $user; private $root; @@ -56,6 +62,9 @@ class SFTP extends \OC\Files\Storage\Common { * @var \phpseclib\Net\SFTP */ protected $client; + private IMimeTypeDetector $mimeTypeDetector; + + const COPY_CHUNK_SIZE = 8 * 1024 * 1024; /** * @param string $host protocol://server:port @@ -111,6 +120,7 @@ public function __construct($params) { $this->root = '/' . ltrim($this->root, '/'); $this->root = rtrim($this->root, '/') . '/'; + $this->mimeTypeDetector = \OC::$server->get(IMimeTypeDetector::class); } /** @@ -370,20 +380,24 @@ public function unlink($path) { public function fopen($path, $mode) { try { $absPath = $this->absPath($path); + $connection = $this->getConnection(); switch ($mode) { case 'r': case 'rb': - if (!$this->file_exists($path)) { + $stat = $this->stat($path); + if (!$stat) { return false; } SFTPReadStream::register(); - $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]); + $context = stream_context_create(['sftp' => ['session' => $connection, 'size' => $stat['size']]]); $handle = fopen('sftpread://' . trim($absPath, '/'), 'r', false, $context); return RetryWrapper::wrap($handle); case 'w': case 'wb': SFTPWriteStream::register(); - $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]); + // the SFTPWriteStream doesn't go through the "normal" methods so it doesn't clear the stat cache. + $connection->_remove_from_stat_cache($absPath); + $context = stream_context_create(['sftp' => ['session' => $connection]]); return fopen('sftpwrite://' . trim($absPath, '/'), 'w', false, $context); case 'a': case 'ab': @@ -395,7 +409,7 @@ public function fopen($path, $mode) { case 'x+': case 'c': case 'c+': - $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]); + $context = stream_context_create(['sftp' => ['session' => $connection]]); $handle = fopen($this->constructUrl($path), $mode, false, $context); return RetryWrapper::wrap($handle); } @@ -450,14 +464,14 @@ public function rename($source, $target) { } /** - * {@inheritdoc} + * @return array{mtime: int, size: int, ctime: int}|false */ public function stat($path) { try { $stat = $this->getConnection()->stat($this->absPath($path)); - $mtime = $stat ? $stat['mtime'] : -1; - $size = $stat ? $stat['size'] : 0; + $mtime = $stat ? (int)$stat['mtime'] : -1; + $size = $stat ? (int)$stat['size'] : 0; return ['mtime' => $mtime, 'size' => $size, 'ctime' => -1]; } catch (\Exception $e) { @@ -476,4 +490,99 @@ public function constructUrl($path) { $url = 'sftp://' . urlencode($this->user) . '@' . $this->host . ':' . $this->port . $this->root . $path; return $url; } + + public function file_put_contents($path, $data) { + /** @psalm-suppress InternalMethod */ + $result = $this->getConnection()->put($this->absPath($path), $data); + if ($result) { + return strlen($data); + } else { + return false; + } + } + + public function writeStream(string $path, $stream, int $size = null): int { + if ($size === null) { + $stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size) { + $size = $writtenSize; + }); + if (!is_resource($stream)) { + throw new \Exception("Failed to wrap stream"); + } + } + /** @psalm-suppress InternalMethod */ + $result = $this->getConnection()->put($this->absPath($path), $stream); + fclose($stream); + if ($result) { + return $size; + } else { + throw new \Exception("Failed to write steam to sftp storage"); + } + } + + public function copy($source, $target) { + if ($this->is_dir($source) || $this->is_dir($target)) { + return parent::copy($source, $target); + } else { + $absSource = $this->absPath($source); + $absTarget = $this->absPath($target); + + $connection = $this->getConnection(); + $size = $connection->size($absSource); + if ($size === false) { + return false; + } + for ($i = 0; $i < $size; $i += self::COPY_CHUNK_SIZE) { + /** @psalm-suppress InvalidArgument */ + $chunk = $connection->get($absSource, false, $i, self::COPY_CHUNK_SIZE); + if ($chunk === false) { + return false; + } + /** @psalm-suppress InternalMethod */ + if (!$connection->put($absTarget, $chunk, \phpseclib\Net\SFTP::SOURCE_STRING, $i)) { + return false; + } + } + return true; + } + } + + public function getPermissions($path) { + $stat = $this->getConnection()->stat($this->absPath($path)); + if (!$stat) { + return 0; + } + if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) { + return Constants::PERMISSION_ALL; + } else { + return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE; + } + } + + public function getMetaData($path) { + $stat = $this->getConnection()->stat($this->absPath($path)); + if (!$stat) { + return null; + } + + if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) { + $stat['permissions'] = Constants::PERMISSION_ALL; + } else { + $stat['permissions'] = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE; + } + + if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) { + $stat['size'] = -1; + $stat['mimetype'] = FileInfo::MIMETYPE_FOLDER; + } else { + $stat['mimetype'] = $this->mimeTypeDetector->detectPath($path); + } + + $stat['etag'] = $this->getETag($path); + $stat['storage_mtime'] = $stat['mtime']; + $stat['name'] = basename($path); + + $keys = ['size', 'mtime', 'mimetype', 'etag', 'storage_mtime', 'permissions', 'name']; + return array_intersect_key($stat, array_flip($keys)); + } } diff --git a/apps/files_external/lib/Lib/Storage/SFTPReadStream.php b/apps/files_external/lib/Lib/Storage/SFTPReadStream.php index c4749b154538e..7a98c6b2a6d59 100644 --- a/apps/files_external/lib/Lib/Storage/SFTPReadStream.php +++ b/apps/files_external/lib/Lib/Storage/SFTPReadStream.php @@ -49,6 +49,8 @@ class SFTPReadStream implements File { private $eof = false; private $buffer = ''; + private bool $pendingRead = false; + private int $size = 0; public static function register($protocol = 'sftpread') { if (in_array($protocol, stream_get_wrappers(), true)) { @@ -75,6 +77,9 @@ protected function loadContext($name) { } else { throw new \BadMethodCallException('Invalid context, session not set'); } + if (isset($context['size'])) { + $this->size = $context['size']; + } return $context; } @@ -118,7 +123,25 @@ public function stream_open($path, $mode, $options, &$opened_path) { } public function stream_seek($offset, $whence = SEEK_SET) { - return false; + switch ($whence) { + case SEEK_SET: + $this->seekTo($offset); + break; + case SEEK_CUR: + $this->seekTo($this->readPosition + $offset); + break; + case SEEK_END: + $this->seekTo($this->size + $offset); + break; + } + return true; + } + + private function seekTo(int $offset): void { + $this->internalPosition = $offset; + $this->readPosition = $offset; + $this->buffer = ''; + $this->request_chunk(256 * 1024); } public function stream_tell() { @@ -142,11 +165,17 @@ public function stream_read($count) { } private function request_chunk($size) { + if ($this->pendingRead) { + $this->sftp->_get_sftp_packet(); + } + $packet = pack('Na*N3', strlen($this->handle), $this->handle, $this->internalPosition / 4294967296, $this->internalPosition, $size); + $this->pendingRead = true; return $this->sftp->_send_sftp_packet(NET_SFTP_READ, $packet); } private function read_chunk() { + $this->pendingRead = false; $response = $this->sftp->_get_sftp_packet(); switch ($this->sftp->packet_type) { @@ -195,6 +224,10 @@ public function stream_eof() { } public function stream_close() { + // we still have a read request incoming that needs to be handled before we can close + if ($this->pendingRead) { + $this->sftp->_get_sftp_packet(); + } if (!$this->sftp->_close_handle($this->handle)) { return false; }