diff --git a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp index 3bfa77c813a3..468ae7b7b24b 100644 --- a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp +++ b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.cpp @@ -36,13 +36,6 @@ CREATE TABLE IF NOT EXISTS projects CREATE INDEX IF NOT EXISTS local_path_index ON projects (local_path); -CREATE TABLE IF NOT EXISTS pending_blocks -( - project_id INTEGER, - block_id INTEGER, - PRIMARY KEY (project_id, block_id) -); - CREATE TABLE IF NOT EXISTS block_hashes ( project_id INTEGER, @@ -53,6 +46,42 @@ CREATE TABLE IF NOT EXISTS block_hashes CREATE INDEX IF NOT EXISTS block_hashes_index ON block_hashes (hash); +CREATE TABLE IF NOT EXISTS pending_snapshots +( + project_id TEXT, + snapshot_id TEXT, + confirm_url TEXT, + PRIMARY KEY (project_id, snapshot_id) +); + +CREATE TABLE IF NOT EXISTS pending_project_blobs +( + project_id TEXT, + snapshot_id TEXT, + + upload_url TEXT, + confirm_url TEXT, + fail_url TEXT, + + blob BLOB, + PRIMARY KEY (project_id, snapshot_id) +); + +CREATE TABLE IF NOT EXISTS pending_project_blocks +( + project_id TEXT, + snapshot_id TEXT, + + upload_url TEXT, + confirm_url TEXT, + fail_url TEXT, + + block_id INTEGER, + block_sample_format INTEGER, + block_hash TEXT, + PRIMARY KEY (project_id, snapshot_id, block_id) +); + CREATE TABLE IF NOT EXISTS project_users ( project_id INTEGER, @@ -93,7 +122,7 @@ const sqlite::SafeConnection::Lock CloudProjectsDatabase::GetConnection() const } std::optional -CloudProjectsDatabase::GetProjectData(const std::string_view& projectId) const +CloudProjectsDatabase::GetProjectData(std::string_view projectId) const { auto connection = GetConnection(); @@ -127,7 +156,7 @@ std::optional CloudProjectsDatabase::GetProjectDataForPath( } bool CloudProjectsDatabase::MarkProjectAsSynced( - const std::string_view& projectId, const std::string_view& snapshotId) + std::string_view projectId, std::string_view snapshotId) { auto connection = GetConnection(); @@ -153,7 +182,7 @@ bool CloudProjectsDatabase::MarkProjectAsSynced( } void CloudProjectsDatabase::UpdateProjectBlockList( - const std::string_view& projectId, const SampleBlockIDSet& blockSet) + std::string_view projectId, const SampleBlockIDSet& blockSet) { auto connection = GetConnection(); @@ -176,7 +205,7 @@ void CloudProjectsDatabase::UpdateProjectBlockList( } std::optional CloudProjectsDatabase::GetBlockHash( - const std::string_view& projectId, int64_t blockId) const + std::string_view projectId, int64_t blockId) const { auto connection = GetConnection(); @@ -205,7 +234,7 @@ std::optional CloudProjectsDatabase::GetBlockHash( } void CloudProjectsDatabase::UpdateBlockHashes( - const std::string_view& projectId, + std::string_view projectId, const std::vector>& hashes) { auto connection = GetConnection(); @@ -297,6 +326,310 @@ void CloudProjectsDatabase::SetProjectUserSlug( statement->Prepare(projectId, slug).Run(); } +bool CloudProjectsDatabase::IsProjectBlockLocked( + std::string_view projectId, int64_t blockId) const +{ + auto connection = GetConnection(); + + if (!connection) + return false; + + auto statement = connection->CreateStatement( + "SELECT 1 FROM pending_project_blocks WHERE project_id = ? AND block_id = ? LIMIT 1"); + + if (!statement) + return false; + + auto result = statement->Prepare(projectId, blockId).Run(); + + for (auto row : result) + return true; + + return false; +} + +void CloudProjectsDatabase::AddPendingSnapshot( + const PendingSnapshotData& snapshotData) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "INSERT OR REPLACE INTO pending_snapshots (project_id, snapshot_id, confirm_url) VALUES (?, ?, ?)"); + + if (!statement) + return; + + statement + ->Prepare( + snapshotData.ProjectId, snapshotData.SnapshotId, + snapshotData.ConfirmUrl) + .Run(); +} + +void CloudProjectsDatabase::RemovePendingSnapshot( + std::string_view projectId, std::string_view snapshotId) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "DELETE FROM pending_snapshots WHERE project_id = ? AND snapshot_id = ?"); + + if (!statement) + return; + + statement->Prepare(projectId, snapshotId).Run(); +} + +std::vector +CloudProjectsDatabase::GetPendingSnapshots(std::string_view projectId) const +{ + auto connection = GetConnection(); + + if (!connection) + return {}; + + auto statement = connection->CreateStatement( + "SELECT project_id, snapshot_id, confirm_url FROM pending_snapshots WHERE project_id = ?"); + + if (!statement) + return {}; + + auto result = statement->Prepare(projectId).Run(); + + std::vector snapshots; + + for (auto row : result) + { + PendingSnapshotData data; + + if (!row.Get(0, data.ProjectId)) + return {}; + + if (!row.Get(1, data.SnapshotId)) + return {}; + + if (!row.Get(2, data.ConfirmUrl)) + return {}; + + snapshots.push_back(data); + } + + return snapshots; +} + +void CloudProjectsDatabase::AddPendingProjectBlob( + const PendingProjectBlobData& blobData) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "INSERT OR REPLACE INTO pending_project_blobs (project_id, snapshot_id, upload_url, confirm_url, fail_url, blob) VALUES (?, ?, ?, ?, ?, ?)"); + + if (!statement) + return; + + statement->Prepare() + .Bind(1, blobData.ProjectId) + .Bind(2, blobData.SnapshotId) + .Bind(3, blobData.UploadUrl) + .Bind(4, blobData.ConfirmUrl) + .Bind(5, blobData.FailUrl) + .Bind(6, blobData.BlobData.data(), blobData.BlobData.size(), false) + .Run(); +} + +void CloudProjectsDatabase::RemovePendingProjectBlob( + std::string_view projectId, std::string_view snapshotId) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "DELETE FROM pending_project_blobs WHERE project_id = ? AND snapshot_id = ?"); + + if (!statement) + return; + + statement->Prepare(projectId, snapshotId).Run(); +} + +std::optional +CloudProjectsDatabase::GetPendingProjectBlob( + std::string_view projectId, std::string_view snapshotId) const +{ + auto connection = GetConnection(); + + if (!connection) + return {}; + + auto statement = connection->CreateStatement( + "SELECT project_id, snapshot_id, upload_url, confirm_url, fail_url, blob FROM pending_project_blobs WHERE project_id = ? AND snapshot_id = ?"); + + if (!statement) + return {}; + + auto result = statement->Prepare(projectId, snapshotId).Run(); + + for (auto row : result) + { + PendingProjectBlobData data; + + if (!row.Get(1, data.ProjectId)) + return {}; + + if (!row.Get(2, data.SnapshotId)) + return {}; + + if (!row.Get(3, data.UploadUrl)) + return {}; + + if (!row.Get(4, data.ConfirmUrl)) + return {}; + + if (!row.Get(5, data.FailUrl)) + return {}; + + const auto size = row.GetColumnBytes(6); + data.BlobData.resize(size); + + if (size != row.ReadData(6, data.BlobData.data(), size)) + return {}; + + return data; + } + + return {}; +} + +void CloudProjectsDatabase::AddPendingProjectBlocks( + const std::vector& blockData) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto tx = connection->BeginTransaction("AddPendingProjectBlocks"); + + auto statement = connection->CreateStatement( + "INSERT OR REPLACE INTO pending_project_blocks (project_id, snapshot_id, upload_url, confirm_url, fail_url, block_id, block_sample_format, block_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + + if (!statement) + return; + + for (const auto& data : blockData) + statement->Prepare() + .Bind(1, data.ProjectId) + .Bind(2, data.SnapshotId) + .Bind(3, data.UploadUrl) + .Bind(4, data.ConfirmUrl) + .Bind(5, data.FailUrl) + .Bind(6, data.BlockId) + .Bind(7, data.BlockSampleFormat) + .Bind(8, data.BlockHash) + .Run(); + + tx.Commit(); +} + +void CloudProjectsDatabase::RemovePendingProjectBlock( + std::string_view projectId, std::string_view snapshotId, int64_t blockId) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "DELETE FROM pending_project_blocks WHERE project_id = ? AND snapshot_id = ? AND block_id = ?"); + + if (!statement) + return; + + statement->Prepare(projectId, snapshotId, blockId).Run(); +} + +void CloudProjectsDatabase::RemovePendingProjectBlocks( + std::string_view projectId, std::string_view snapshotId) +{ + auto connection = GetConnection(); + + if (!connection) + return; + + auto statement = connection->CreateStatement( + "DELETE FROM pending_project_blocks WHERE project_id = ? AND snapshot_id = ?"); + + if (!statement) + return; + + statement->Prepare(projectId, snapshotId).Run(); +} + +std::vector +CloudProjectsDatabase::GetPendingProjectBlocks( + std::string_view projectId, std::string_view snapshotId) +{ +auto connection = GetConnection(); + + if (!connection) + return {}; + + auto statement = connection->CreateStatement( + "SELECT project_id, snapshot_id, upload_url, confirm_url, fail_url, block_id, block_sample_format, block_hash FROM pending_project_blocks WHERE project_id = ? AND snapshot_id = ?"); + + if (!statement) + return {}; + + auto result = statement->Prepare(projectId, snapshotId).Run(); + + std::vector blocks; + + for (auto row : result) + { + PendingProjectBlockData data; + + if (!row.Get(0, data.ProjectId)) + return {}; + + if (!row.Get(1, data.SnapshotId)) + return {}; + + if (!row.Get(2, data.UploadUrl)) + return {}; + + if (!row.Get(3, data.ConfirmUrl)) + return {}; + + if (!row.Get(4, data.FailUrl)) + return {}; + + if (!row.Get(5, data.BlockId)) + return {}; + + if (!row.Get(6, data.BlockSampleFormat)) + return {}; + + if (!row.Get(7, data.BlockHash)) + return {}; + + blocks.push_back(data); + } + + return blocks; +} + std::optional CloudProjectsDatabase::DoGetProjectData(sqlite::RunResult result) const { diff --git a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h index 2defdd313cdd..57d5d94ab794 100644 --- a/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h +++ b/libraries/lib-cloud-audiocom/sync/CloudProjectsDatabase.h @@ -12,6 +12,7 @@ #include #include +#include #include "sqlite/SafeConnection.h" @@ -23,20 +24,54 @@ struct DBProjectData final { std::string ProjectId; std::string SnapshotId; - int64_t SavesCount = 0; + int64_t SavesCount = 0; int64_t LastAudioPreview = 0; std::string LocalPath; int64_t LastModified = 0; - int64_t LastRead = 0; + int64_t LastRead = 0; enum SyncStatusType { - SyncStatusSynced = 0, - SyncStatusUploading = 1, + SyncStatusSynced = 0, + SyncStatusUploading = 1, SyncStatusDownloading = 2, } SyncStatus = {}; }; +struct PendingSnapshotData final +{ + std::string ProjectId; + std::string SnapshotId; + std::string ConfirmUrl; +}; + +struct PendingProjectBlobData final +{ + std::string ProjectId; + std::string SnapshotId; + + std::string UploadUrl; + std::string ConfirmUrl; + std::string FailUrl; + + std::vector BlobData; +}; + +struct PendingProjectBlockData final +{ + std::string ProjectId; + std::string SnapshotId; + + std::string UploadUrl; + std::string ConfirmUrl; + std::string FailUrl; + + int64_t BlockId {}; + int BlockSampleFormat {}; + + std::string BlockHash; +}; + class CloudProjectsDatabase final { CloudProjectsDatabase(); @@ -50,16 +85,21 @@ class CloudProjectsDatabase final sqlite::SafeConnection::Lock GetConnection(); const sqlite::SafeConnection::Lock GetConnection() const; - std::optional GetProjectData(const std::string_view& projectId) const; - std::optional GetProjectDataForPath(const std::string& projectPath) const; - bool MarkProjectAsSynced(const std::string_view& projectId, const std::string_view& snapshotId); + std::optional + GetProjectData(std::string_view projectId) const; + std::optional + GetProjectDataForPath(const std::string& projectPath) const; + bool + MarkProjectAsSynced(std::string_view projectId, std::string_view snapshotId); - void UpdateProjectBlockList(const std::string_view& projectId, const SampleBlockIDSet& blockSet); + void UpdateProjectBlockList( + std::string_view projectId, const SampleBlockIDSet& blockSet); - std::optional GetBlockHash(const std::string_view& projectId, int64_t blockId) const; + std::optional + GetBlockHash(std::string_view projectId, int64_t blockId) const; void UpdateBlockHashes( - const std::string_view& projectId, + std::string_view projectId, const std::vector>& hashes); bool UpdateProjectData(const DBProjectData& projectData); @@ -67,10 +107,33 @@ class CloudProjectsDatabase final std::string GetProjectUserSlug(std::string_view projectId); void SetProjectUserSlug(std::string_view projectId, std::string_view slug); + bool IsProjectBlockLocked(std::string_view projectId, int64_t blockId) const; + + void AddPendingSnapshot(const PendingSnapshotData& snapshotData); + void RemovePendingSnapshot( + std::string_view projectId, std::string_view snapshotId); + std::vector + GetPendingSnapshots(std::string_view projectId) const; + + void AddPendingProjectBlob(const PendingProjectBlobData& blobData); + void RemovePendingProjectBlob( + std::string_view projectId, std::string_view snapshotId); + std::optional GetPendingProjectBlob( + std::string_view projectId, std::string_view snapshotId) const; + + void AddPendingProjectBlocks( + const std::vector& blockData); + void RemovePendingProjectBlock( + std::string_view projectId, std::string_view snapshotId, int64_t blockId); + void RemovePendingProjectBlocks( + std::string_view projectId, std::string_view snapshotId); + std::vector GetPendingProjectBlocks( + std::string_view projectId, std::string_view snapshotId); + private: - std::optional DoGetProjectData(sqlite::RunResult result) const; + std::optional + DoGetProjectData(sqlite::RunResult result) const; bool OpenConnection(); std::shared_ptr mConnection; - }; } // namespace cloud::audiocom::sync diff --git a/libraries/lib-cloud-audiocom/sync/DataUploader.cpp b/libraries/lib-cloud-audiocom/sync/DataUploader.cpp index 2beeb2df4569..01469b0ae22f 100644 --- a/libraries/lib-cloud-audiocom/sync/DataUploader.cpp +++ b/libraries/lib-cloud-audiocom/sync/DataUploader.cpp @@ -181,7 +181,6 @@ struct DataUploader::Response final Request request { Target.FailUrl }; NetworkResponse = NetworkManager::GetInstance().doPost(request, nullptr, 0); - CancelContext->OnCancelled(NetworkResponse); NetworkResponse->setRequestFinishedCallback( [this](auto) @@ -198,6 +197,7 @@ struct DataUploader::Response final // Ignore other errors, server will collect garbage // and delete the file eventually }); + CancelContext->OnCancelled(NetworkResponse); } void CleanUp() diff --git a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp index 84a4164e1823..2d4d79f37a78 100644 --- a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp +++ b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.cpp @@ -32,10 +32,6 @@ #include "WaveClip.h" #include "WaveTrack.h" -#include "TransactionScope.h" - -#include "CodeConversions.h" - #include "IResponse.h" #include "NetworkManager.h" #include "Request.h" @@ -211,7 +207,9 @@ LocalProjectSnapshot::LocalProjectSnapshot( , mServiceConfig { config } , mOAuthService { oauthService } , mProjectName { std::move(name) } - , mCancellationContext { audacity::concurrency::CancellationContext::Create() } + , mCancellationContext { + audacity::concurrency::CancellationContext::Create() + } { } @@ -490,18 +488,30 @@ void LocalProjectSnapshot::OnSnapshotCreated( if (mCancelled.load(std::memory_order_acquire)) return; + StorePendingSnapshot(response, projectData); + DataUploader::Get().Upload( mCancellationContext, mServiceConfig, response.SyncState.FileUrls, projectData.ProjectSnapshot, [this](ResponseResult result) { + auto& db = CloudProjectsDatabase::Get(); + + const auto projectId = mCreateSnapshotResponse->Project.Id; + const auto snapshotId = mCreateSnapshotResponse->Snapshot.Id; + if (result.Code != ResponseResultCode::Success) { + db.RemovePendingSnapshot(projectId, snapshotId); + db.RemovePendingProjectBlob(projectId, snapshotId); + db.RemovePendingProjectBlocks(projectId, snapshotId); + DataUploadFailed(result); return; } mProjectCloudExtension.OnProjectDataUploaded(*this); + db.RemovePendingProjectBlob(projectId, snapshotId); if (mProjectBlocksLock->MissingBlocks.empty()) { @@ -517,6 +527,11 @@ void LocalProjectSnapshot::OnSnapshotCreated( const auto handledBlocks = result.UploadedBlocks + result.FailedBlocks; + if (uploadResult.Code != ResponseResultCode::ConnectionFailed) + CloudProjectsDatabase::Get().RemovePendingProjectBlock( + mCreateSnapshotResponse->Project.Id, + mCreateSnapshotResponse->Snapshot.Id, block.Id); + mProjectCloudExtension.OnBlockUploaded( *this, block.Hash, uploadResult.Code == ResponseResultCode::Success); @@ -536,6 +551,38 @@ void LocalProjectSnapshot::OnSnapshotCreated( }); } +void LocalProjectSnapshot::StorePendingSnapshot( + const CreateSnapshotResponse& response, const ProjectUploadData& projectData) +{ + CloudProjectsDatabase::Get().AddPendingSnapshot( + { response.Project.Id, response.Snapshot.Id, + mServiceConfig.GetSnapshotSyncUrl( + mProjectCloudExtension.GetCloudProjectId(), + mProjectCloudExtension.GetSnapshotId()) }); + + CloudProjectsDatabase::Get().AddPendingProjectBlob( + { response.Project.Id, response.Snapshot.Id, + response.SyncState.FileUrls.UploadUrl, + response.SyncState.FileUrls.SuccessUrl, + response.SyncState.FileUrls.FailUrl, projectData.ProjectSnapshot }); + + if (mProjectBlocksLock->MissingBlocks.empty()) + return; + + std::vector pendingBlocks; + pendingBlocks.reserve(mProjectBlocksLock->MissingBlocks.size()); + + for (const auto& block : mProjectBlocksLock->MissingBlocks) + { + pendingBlocks.push_back(PendingProjectBlockData { + response.Project.Id, response.Snapshot.Id, block.BlockUrls.UploadUrl, + block.BlockUrls.SuccessUrl, block.BlockUrls.FailUrl, block.Block.Id, + static_cast(block.Block.Format), block.Block.Hash }); + } + + CloudProjectsDatabase::Get().AddPendingProjectBlocks(pendingBlocks); +} + void LocalProjectSnapshot::MarkSnapshotSynced(int64_t blocksCount) { using namespace audacity::network_manager; @@ -557,6 +604,10 @@ void LocalProjectSnapshot::MarkSnapshotSynced(int64_t blocksCount) response->setRequestFinishedCallback( [this, response, blocksCount](auto) { + CloudProjectsDatabase::Get().RemovePendingSnapshot( + mCreateSnapshotResponse->Project.Id, + mCreateSnapshotResponse->Snapshot.Id); + if (response->getError() != NetworkError::NoError) { UploadFailed(DeduceUploadError(*response)); diff --git a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h index e710c73214be..60228f796205 100644 --- a/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h +++ b/libraries/lib-cloud-audiocom/sync/LocalProjectSnapshot.h @@ -82,6 +82,8 @@ class CLOUD_AUDIOCOM_API LocalProjectSnapshot final : void OnSnapshotCreated(const CreateSnapshotResponse& response, bool newProject); + void StorePendingSnapshot( + const CreateSnapshotResponse& response, const ProjectUploadData& data); void MarkSnapshotSynced(int64_t blocksCount); diff --git a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp index 903faa0b8efe..42d76d06c85d 100644 --- a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp +++ b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.cpp @@ -388,7 +388,18 @@ void ProjectCloudExtension::OnUpdateSaved(const ProjectSerializer& serializer) mUploadQueue.back()->Data.ProjectSnapshot = std::move(data); if (mUploadQueue.back()->Operation) + { mUploadQueue.back()->Operation->SetUploadData(mUploadQueue.back()->Data); + + auto dbData = CloudProjectsDatabase::Get().GetProjectData(mProjectId); + + if (dbData) + { + dbData->LocalPath = + audacity::ToUTF8(ProjectFileIO::Get(mProject).GetFileName()); + CloudProjectsDatabase::Get().UpdateProjectData(*dbData); + } + } else Publish({ ProjectSyncStatus::Failed }, false); } @@ -656,6 +667,16 @@ std::string ProjectCloudExtension::GetCloudProjectPage() const return GetServiceConfig().GetProjectPageUrl(userSlug, projectId); } +bool ProjectCloudExtension::IsBlockLocked(int64_t blockID) const +{ + const auto projectId = GetCloudProjectId(); + + if (projectId.empty()) + return false; + + return CloudProjectsDatabase::Get().IsProjectBlockLocked(projectId, blockID); +} + bool CloudStatusChangedMessage::IsSyncing() const noexcept { return Status == ProjectSyncStatus::Syncing; diff --git a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h index 15c68399c0b7..f79102bea5ee 100644 --- a/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h +++ b/libraries/lib-cloud-audiocom/sync/ProjectCloudExtension.h @@ -121,6 +121,8 @@ class CLOUD_AUDIOCOM_API ProjectCloudExtension final : public ClientData::Base std::string GetCloudProjectPage() const; + bool IsBlockLocked(int64_t blockID) const; + private: struct UploadQueueElement; struct CloudStatusChangedNotifier; diff --git a/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp b/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp index 2792e06180bd..c6c5d5fe4edd 100644 --- a/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp +++ b/modules/mod-cloud-audiocom/CloudProjectFileIOExtensions.cpp @@ -195,7 +195,7 @@ class IOExtension final : public ProjectFileIOExtension bool IsBlockLocked(const AudacityProject& project, int64_t blockId) const override { - return false; + return ProjectCloudExtension::Get(project).IsBlockLocked(blockId); } };