Prepare the infrastructure for the resuming interrupted uploads

crsib · Feb 22, 2024 · a4c8862 · a4c8862
1 parent 791f1da
commit a4c8862
Show file tree

Hide file tree

Showing 8 changed files with 503 additions and 31 deletions.
diff --git a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp
diff --git a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h
@@ -12,6 +12,7 @@
 
 #include <optional>
 #include <string>
+#include <vector>
 
 #include "sqlite/SafeConnection.h"
 
@@ -23,20 +24,54 @@ struct DBProjectData final
 {
    std::string ProjectId;
    std::string SnapshotId;
-   int64_t SavesCount = 0;
+   int64_t SavesCount       = 0;
    int64_t LastAudioPreview = 0;
    std::string LocalPath;
    int64_t LastModified = 0;
-   int64_t LastRead = 0;
+   int64_t LastRead     = 0;
 
    enum SyncStatusType
    {
-      SyncStatusSynced = 0,
-      SyncStatusUploading = 1,
+      SyncStatusSynced      = 0,
+      SyncStatusUploading   = 1,
       SyncStatusDownloading = 2,
    } SyncStatus = {};
 };
 
+struct PendingSnapshotData final
+{
+   std::string ProjectId;
+   std::string SnapshotId;
+   std::string ConfirmUrl;
+};
+
+struct PendingProjectBlobData final
+{
+   std::string ProjectId;
+   std::string SnapshotId;
+
+   std::string UploadUrl;
+   std::string ConfirmUrl;
+   std::string FailUrl;
+
+   std::vector<uint8_t> BlobData;
+};
+
+struct PendingProjectBlockData final
+{
+   std::string ProjectId;
+   std::string SnapshotId;
+
+   std::string UploadUrl;
+   std::string ConfirmUrl;
+   std::string FailUrl;
+
+   int64_t BlockId {};
+   int BlockSampleFormat {};
+
+   std::string BlockHash;
+};
+
 class CloudProjectsDatabase final
 {
    CloudProjectsDatabase();
@@ -50,27 +85,55 @@ class CloudProjectsDatabase final
    sqlite::SafeConnection::Lock GetConnection();
    const sqlite::SafeConnection::Lock GetConnection() const;
 
-   std::optional<DBProjectData> GetProjectData(const std::string_view& projectId) const;
-   std::optional<DBProjectData> GetProjectDataForPath(const std::string& projectPath) const;
-   bool MarkProjectAsSynced(const std::string_view& projectId, const std::string_view& snapshotId);
+   std::optional<DBProjectData>
+   GetProjectData(std::string_view projectId) const;
+   std::optional<DBProjectData>
+   GetProjectDataForPath(const std::string& projectPath) const;
+   bool
+   MarkProjectAsSynced(std::string_view projectId, std::string_view snapshotId);
 
-   void UpdateProjectBlockList(const std::string_view& projectId, const SampleBlockIDSet& blockSet);
+   void UpdateProjectBlockList(
+      std::string_view projectId, const SampleBlockIDSet& blockSet);
 
-   std::optional<std::string> GetBlockHash(const std::string_view& projectId, int64_t blockId) const;
+   std::optional<std::string>
+   GetBlockHash(std::string_view projectId, int64_t blockId) const;
 
    void UpdateBlockHashes(
-      const std::string_view& projectId,
+      std::string_view projectId,
       const std::vector<std::pair<int64_t, std::string>>& hashes);
 
    bool UpdateProjectData(const DBProjectData& projectData);
 
    std::string GetProjectUserSlug(std::string_view projectId);
    void SetProjectUserSlug(std::string_view projectId, std::string_view slug);
 
+   bool IsProjectBlockLocked(std::string_view projectId, int64_t blockId) const;
+
+   void AddPendingSnapshot(const PendingSnapshotData& snapshotData);
+   void RemovePendingSnapshot(
+      std::string_view projectId, std::string_view snapshotId);
+   std::vector<PendingSnapshotData>
+   GetPendingSnapshots(std::string_view projectId) const;
+
+   void AddPendingProjectBlob(const PendingProjectBlobData& blobData);
+   void RemovePendingProjectBlob(
+      std::string_view projectId, std::string_view snapshotId);
+   std::optional<PendingProjectBlobData> GetPendingProjectBlob(
+      std::string_view projectId, std::string_view snapshotId) const;
+
+   void AddPendingProjectBlocks(
+      const std::vector<PendingProjectBlockData>& blockData);
+   void RemovePendingProjectBlock(
+      std::string_view projectId, std::string_view snapshotId, int64_t blockId);
+   void RemovePendingProjectBlocks(
+      std::string_view projectId, std::string_view snapshotId);
+   std::vector<PendingProjectBlockData> GetPendingProjectBlocks(
+      std::string_view projectId, std::string_view snapshotId);
+
 private:
-   std::optional<DBProjectData> DoGetProjectData(sqlite::RunResult result) const;
+   std::optional<DBProjectData>
+   DoGetProjectData(sqlite::RunResult result) const;
    bool OpenConnection();
    std::shared_ptr<sqlite::SafeConnection> mConnection;
-
 };
 } // namespace cloud::audiocom::sync
diff --git a/libraries/lib-cloud-audiocom/sync/DataUploader.cpp b/libraries/lib-cloud-audiocom/sync/DataUploader.cpp
@@ -181,7 +181,6 @@ struct DataUploader::Response final
       Request request { Target.FailUrl };
 
       NetworkResponse = NetworkManager::GetInstance().doPost(request, nullptr, 0);
-      CancelContext->OnCancelled(NetworkResponse);
 
       NetworkResponse->setRequestFinishedCallback(
          [this](auto)
@@ -198,6 +197,7 @@ struct DataUploader::Response final
             // Ignore other errors, server will collect garbage
             // and delete the file eventually
          });
+      CancelContext->OnCancelled(NetworkResponse);
    }
 
    void CleanUp()

diff --git a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp
@@ -32,10 +32,6 @@
 #include "WaveClip.h"
 #include "WaveTrack.h"
 
-#include "TransactionScope.h"
-
-#include "CodeConversions.h"
-
 #include "IResponse.h"
 #include "NetworkManager.h"
 #include "Request.h"
@@ -211,7 +207,9 @@ LocalProjectSnapshot::LocalProjectSnapshot(
     , mServiceConfig { config }
     , mOAuthService { oauthService }
     , mProjectName { std::move(name) }
-    , mCancellationContext { audacity::concurrency::CancellationContext::Create() }
+    , mCancellationContext {
+       audacity::concurrency::CancellationContext::Create()
+    }
 {
 }
 
@@ -490,18 +488,30 @@ void LocalProjectSnapshot::OnSnapshotCreated(
    if (mCancelled.load(std::memory_order_acquire))
       return;
 
+   StorePendingSnapshot(response, projectData);
+
    DataUploader::Get().Upload(
       mCancellationContext, mServiceConfig, response.SyncState.FileUrls,
       projectData.ProjectSnapshot,
       [this](ResponseResult result)
       {
+         auto& db = CloudProjectsDatabase::Get();
+
+         const auto projectId  = mCreateSnapshotResponse->Project.Id;
+         const auto snapshotId = mCreateSnapshotResponse->Snapshot.Id;
+
          if (result.Code != ResponseResultCode::Success)
          {
+            db.RemovePendingSnapshot(projectId, snapshotId);
+            db.RemovePendingProjectBlob(projectId, snapshotId);
+            db.RemovePendingProjectBlocks(projectId, snapshotId);
+
             DataUploadFailed(result);
             return;
          }
 
          mProjectCloudExtension.OnProjectDataUploaded(*this);
+         db.RemovePendingProjectBlob(projectId, snapshotId);
 
          if (mProjectBlocksLock->MissingBlocks.empty())
          {
@@ -517,6 +527,11 @@ void LocalProjectSnapshot::OnSnapshotCreated(
                const auto handledBlocks =
                   result.UploadedBlocks + result.FailedBlocks;
 
+               if (uploadResult.Code != ResponseResultCode::ConnectionFailed)
+                  CloudProjectsDatabase::Get().RemovePendingProjectBlock(
+                     mCreateSnapshotResponse->Project.Id,
+                     mCreateSnapshotResponse->Snapshot.Id, block.Id);
+
                mProjectCloudExtension.OnBlockUploaded(
                   *this, block.Hash,
                   uploadResult.Code == ResponseResultCode::Success);
@@ -536,6 +551,38 @@ void LocalProjectSnapshot::OnSnapshotCreated(
       });
 }
 
+void LocalProjectSnapshot::StorePendingSnapshot(
+   const CreateSnapshotResponse& response, const ProjectUploadData& projectData)
+{
+   CloudProjectsDatabase::Get().AddPendingSnapshot(
+      { response.Project.Id, response.Snapshot.Id,
+        mServiceConfig.GetSnapshotSyncUrl(
+           mProjectCloudExtension.GetCloudProjectId(),
+           mProjectCloudExtension.GetSnapshotId()) });
+
+   CloudProjectsDatabase::Get().AddPendingProjectBlob(
+      { response.Project.Id, response.Snapshot.Id,
+        response.SyncState.FileUrls.UploadUrl,
+        response.SyncState.FileUrls.SuccessUrl,
+        response.SyncState.FileUrls.FailUrl, projectData.ProjectSnapshot });
+
+   if (mProjectBlocksLock->MissingBlocks.empty())
+      return;
+
+   std::vector<PendingProjectBlockData> pendingBlocks;
+   pendingBlocks.reserve(mProjectBlocksLock->MissingBlocks.size());
+
+   for (const auto& block : mProjectBlocksLock->MissingBlocks)
+   {
+      pendingBlocks.push_back(PendingProjectBlockData {
+         response.Project.Id, response.Snapshot.Id, block.BlockUrls.UploadUrl,
+         block.BlockUrls.SuccessUrl, block.BlockUrls.FailUrl, block.Block.Id,
+         static_cast<int>(block.Block.Format), block.Block.Hash });
+   }
+
+   CloudProjectsDatabase::Get().AddPendingProjectBlocks(pendingBlocks);
+}
+
 void LocalProjectSnapshot::MarkSnapshotSynced(int64_t blocksCount)
 {
    using namespace audacity::network_manager;
@@ -557,6 +604,10 @@ void LocalProjectSnapshot::MarkSnapshotSynced(int64_t blocksCount)
    response->setRequestFinishedCallback(
       [this, response, blocksCount](auto)
       {
+         CloudProjectsDatabase::Get().RemovePendingSnapshot(
+            mCreateSnapshotResponse->Project.Id,
+            mCreateSnapshotResponse->Snapshot.Id);
+
          if (response->getError() != NetworkError::NoError)
          {
             UploadFailed(DeduceUploadError(*response));

diff --git a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h
@@ -82,6 +82,8 @@ class CLOUD_AUDIOCOM_API LocalProjectSnapshot final :
 
    void
    OnSnapshotCreated(const CreateSnapshotResponse& response, bool newProject);
+   void StorePendingSnapshot(
+      const CreateSnapshotResponse& response, const ProjectUploadData& data);
    void MarkSnapshotSynced(int64_t blocksCount);
 
 

diff --git a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp
@@ -388,7 +388,18 @@ void ProjectCloudExtension::OnUpdateSaved(const ProjectSerializer& serializer)
    mUploadQueue.back()->Data.ProjectSnapshot = std::move(data);
 
    if (mUploadQueue.back()->Operation)
+   {
       mUploadQueue.back()->Operation->SetUploadData(mUploadQueue.back()->Data);
+
+      auto dbData = CloudProjectsDatabase::Get().GetProjectData(mProjectId);
+
+      if (dbData)
+      {
+         dbData->LocalPath =
+            audacity::ToUTF8(ProjectFileIO::Get(mProject).GetFileName());
+         CloudProjectsDatabase::Get().UpdateProjectData(*dbData);
+      }
+   }
    else
       Publish({ ProjectSyncStatus::Failed }, false);
 }
@@ -656,6 +667,16 @@ std::string ProjectCloudExtension::GetCloudProjectPage() const
    return GetServiceConfig().GetProjectPageUrl(userSlug, projectId);
 }
 
+bool ProjectCloudExtension::IsBlockLocked(int64_t blockID) const
+{
+   const auto projectId = GetCloudProjectId();
+
+   if (projectId.empty())
+      return false;
+
+   return CloudProjectsDatabase::Get().IsProjectBlockLocked(projectId, blockID);
+}
+
 bool CloudStatusChangedMessage::IsSyncing() const noexcept
 {
    return Status == ProjectSyncStatus::Syncing;

diff --git a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h
@@ -121,6 +121,8 @@ class CLOUD_AUDIOCOM_API ProjectCloudExtension final : public ClientData::Base
 
    std::string GetCloudProjectPage() const;
 
+   bool IsBlockLocked(int64_t blockID) const;
+
 private:
    struct UploadQueueElement;
    struct CloudStatusChangedNotifier;

diff --git a/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp b/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp
@@ -195,7 +195,7 @@ class IOExtension final : public ProjectFileIOExtension
    bool
    IsBlockLocked(const AudacityProject& project, int64_t blockId) const override
    {
-      return false;
+      return ProjectCloudExtension::Get(project).IsBlockLocked(blockId);
    }
 };
-Original file line number
+Diff line change
@@ Expand Up / @@ -195,7 +195,7 @@ class IOExtension final : public ProjectFileIOExtension @@
        bool
        IsBlockLocked(const AudacityProject& project, int64_t blockId) const override
        {
-          return false;
+          return ProjectCloudExtension::Get(project).IsBlockLocked(blockId);
        }
     };
@@ Expand Down @@