Merge pull request #5939 from nextcloud/feature/chunk-c2

Implement support for server chunked file upload V2 API

src/common/syncjournaldb.cpp

@@ -1648,7 +1648,7 @@ SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString &file)
 
         if (query->next().hasData) {
             bool ok = true;
-            res._chunk = query->intValue(0);
+            res._chunkUploadV1 = query->intValue(0);
             res._transferid = query->int64Value(1);
             res._errorCount = query->intValue(2);
             res._size = query->int64Value(3);
@@ -1678,7 +1678,7 @@ void SyncJournalDb::setUploadInfo(const QString &file, const SyncJournalDb::Uplo
         }
 
         query->bindValue(1, file);
-        query->bindValue(2, i._chunk);
+        query->bindValue(2, i._chunkUploadV1);
         query->bindValue(3, i._transferid);
         query->bindValue(4, i._errorCount);
         query->bindValue(5, i._size);
@@ -2714,13 +2714,8 @@ bool operator==(const SyncJournalDb::DownloadInfo &lhs,
 bool operator==(const SyncJournalDb::UploadInfo &lhs,
     const SyncJournalDb::UploadInfo &rhs)
 {
-    return lhs._errorCount == rhs._errorCount
-        && lhs._chunk == rhs._chunk
-        && lhs._modtime == rhs._modtime
-        && lhs._valid == rhs._valid
-        && lhs._size == rhs._size
-        && lhs._transferid == rhs._transferid
-        && lhs._contentChecksum == rhs._contentChecksum;
+    return lhs._errorCount == rhs._errorCount && lhs._chunkUploadV1 == rhs._chunkUploadV1 && lhs._modtime == rhs._modtime && lhs._valid == rhs._valid
+        && lhs._size == rhs._size && lhs._transferid == rhs._transferid && lhs._contentChecksum == rhs._contentChecksum;
 }
 
 QDebug& operator<<(QDebug &stream, const SyncJournalFileRecord::EncryptionStatus status)

src/common/syncjournaldb.h

@@ -115,7 +115,7 @@ class OCSYNC_EXPORT SyncJournalDb : public QObject
     };
     struct UploadInfo
     {
-        int _chunk = 0;
+        int _chunkUploadV1 = 0;
         uint _transferid = 0;
         qint64 _size = 0;
         qint64 _modtime = 0;

src/gui/folder.cpp

@@ -1036,9 +1036,10 @@ SyncOptions Folder::initializeSyncOptions() const
     opt._vfs = _vfs;
     opt._parallelNetworkJobs = _accountState->account()->isHttp2Supported() ? 20 : 6;
 
-    opt._initialChunkSize = cfgFile.chunkSize();
-    opt._minChunkSize = cfgFile.minChunkSize();
-    opt._maxChunkSize = cfgFile.maxChunkSize();
+    // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller
+    opt.setMinChunkSize(cfgFile.minChunkSize());
+    opt.setMaxChunkSize(cfgFile.maxChunkSize());
+    opt._initialChunkSize = ::qBound(opt.minChunkSize(), cfgFile.chunkSize(), opt.maxChunkSize());
     opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration();
 
     opt.fillFromEnvironmentVariables();

src/libsync/bulkpropagatorjob.cpp

@@ -140,7 +140,7 @@ void BulkPropagatorJob::doStartUpload(SyncFileItemPtr item,
     // in reconcile (issue #5106)
     SyncJournalDb::UploadInfo pi;
     pi._valid = true;
-    pi._chunk = 0;
+    pi._chunkUploadV1 = 0;
     pi._transferid = 0; // We set a null transfer id because it is not chunked.
     pi._modtime = item->_modtime;
     pi._errorCount = 0;

src/libsync/configfile.cpp

@@ -246,19 +246,19 @@ int ConfigFile::timeout() const
 qint64 ConfigFile::chunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(chunkSizeC), 10 * 1000 * 1000).toLongLong(); // default to 10 MB
+    return settings.value(QLatin1String(chunkSizeC), 10LL * 1000LL * 1000LL).toLongLong(); // default to 10 MB
 }
 
 qint64 ConfigFile::maxChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(maxChunkSizeC), 1000 * 1000 * 1000).toLongLong(); // default to 1000 MB
+    return settings.value(QLatin1String(maxChunkSizeC), 5LL * 1000LL * 1000LL * 1000LL).toLongLong(); // default to 5000 MB
 }
 
 qint64 ConfigFile::minChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(minChunkSizeC), 1000 * 1000).toLongLong(); // default to 1 MB
+    return settings.value(QLatin1String(minChunkSizeC), 5LL * 1000LL * 1000LL).toLongLong(); // default to 5 MB
 }
 
 chrono::milliseconds ConfigFile::targetChunkUploadDuration() const

src/libsync/owncloudpropagator.cpp

@@ -1046,7 +1046,8 @@ bool OwncloudPropagator::isDelayedUploadItem(const SyncFileItemPtr &item) const
         return true;
     };
 
-    return account()->capabilities().bulkUpload() && !_scheduleDelayedTasks && !item->isEncrypted() && _syncOptions._minChunkSize > item->_size && !isInBulkUploadBlackList(item->_file) && !checkFileShouldBeEncrypted(item);
+    return account()->capabilities().bulkUpload() && !_scheduleDelayedTasks && !item->isEncrypted() && _syncOptions.minChunkSize() > item->_size
+        && !isInBulkUploadBlackList(item->_file) && !checkFileShouldBeEncrypted(item);
 }
 
 void OwncloudPropagator::setScheduleDelayedTasks(bool active)

src/libsync/propagateupload.h

@@ -378,27 +378,6 @@ private slots:
 class PropagateUploadFileNG : public PropagateUploadFileCommon
 {
     Q_OBJECT
-private:
-    qint64 _sent = 0; /// amount of data (bytes) that was already sent
-    uint _transferId = 0; /// transfer id (part of the url)
-    int _currentChunk = 0; /// Id of the next chunk that will be sent
-    qint64 _currentChunkSize = 0; /// current chunk size
-    bool _removeJobError = false; /// If not null, there was an error removing the job
-
-    // Map chunk number with its size  from the PROPFIND on resume.
-    // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.)
-    struct ServerChunkInfo
-    {
-        qint64 size = 0LL;
-        QString originalName;
-    };
-    QMap<qint64, ServerChunkInfo> _serverChunks;
-
-    /**
-     * Return the URL of a chunk.
-     * If chunk == -1, returns the URL of the parent folder containing the chunks
-     */
-    QUrl chunkUrl(int chunk = -1);
 
 public:
     PropagateUploadFileNG(OwncloudPropagator *propagator, const SyncFileItemPtr &item)
@@ -408,11 +387,9 @@ class PropagateUploadFileNG : public PropagateUploadFileCommon
 
     void doStartUpload() override;
 
-private:
-    void startNewUpload();
-    void startNextChunk();
 public slots:
     void abort(OCC::PropagateUploadFileNG::AbortType abortType) override;
+
 private slots:
     void slotPropfindFinished();
     void slotPropfindFinishedWithError();
@@ -422,5 +399,28 @@ private slots:
     void slotPutFinished();
     void slotMoveJobFinished();
     void slotUploadProgress(qint64, qint64);
+
+private:
+    // Map chunk number with its size  from the PROPFIND on resume.
+    // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.)
+    struct ServerChunkInfo {
+        qint64 size = 0LL;
+        QString originalName;
+    };
+
+    [[nodiscard]] QUrl chunkUploadFolderUrl() const;
+    [[nodiscard]] QUrl chunkUrl(const int chunk) const;
+
+    void startNewUpload();
+    void startNextChunk();
+    void finishUpload();
+
+    QMap<qint64, ServerChunkInfo> _serverChunks;
+
+    qint64 _sent = 0; /// amount of data (bytes) that was already sent
+    uint _transferId = 0; /// transfer id (part of the url)
+    int _currentChunk = 1; /// Id of the next chunk that will be sent
+    qint64 _currentChunkSize = 0; /// current chunk size
+    bool _removeJobError = false; /// If not null, there was an error removing the job
 };
 }