diff --git a/crates/api-common/src/lib.rs b/crates/api-common/src/lib.rs index ca6b2114..6b81cc44 100644 --- a/crates/api-common/src/lib.rs +++ b/crates/api-common/src/lib.rs @@ -342,7 +342,7 @@ pub struct CreateLibraryData { pub blocks: String, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum PublishState { Private, PendingApproval, diff --git a/crates/migrate/src/main.rs b/crates/migrate/src/main.rs index 5cb8560d..d0a99f8e 100644 --- a/crates/migrate/src/main.rs +++ b/crates/migrate/src/main.rs @@ -7,7 +7,7 @@ use std::{collections::HashMap, thread}; use crate::config::Config; use clap::Parser; -use cloud::api::{PublishState, SaveState}; +use cloud::api::SaveState; use derive_more::{Display, Error}; use futures::{future::join_all, stream::StreamExt, TryStreamExt}; use indicatif::ProgressBar; @@ -113,17 +113,7 @@ async fn download( .map(|timestamp| DateTime::from_millis(timestamp as i64)) .unwrap_or_else(DateTime::now); - let state = metadata - .public - .map(|is_public| { - if is_public { - PublishState::Public - } else { - PublishState::Private - } - }) - .unwrap_or(PublishState::Private); - + let state = metadata.state(); let project_id = cloud::api::ProjectId::new(metadata.id.to_string()); let owner = metadata.owner; let name = metadata.name; @@ -191,7 +181,7 @@ async fn upload( join_all(role_iter.map(|(_id, data)| upload_role(client, bucket, &owner, &name, data))) .await .into_iter(); - let roles: HashMap<_, _> = role_ids.zip(role_data).into_iter().collect(); + let roles: HashMap<_, _> = role_ids.zip(role_data).collect(); cloud::ProjectMetadata { id: project.id, @@ -424,15 +414,30 @@ async fn migrate_projects(config: &Config, src_db: &Database, dst_db: &Database) "owner": &metadata.owner, "name": &metadata.name, }; - let exists = dst_projects.find_one(query, None).await.unwrap().is_some(); - if !exists { + let dst_project = dst_projects.find_one(query.clone(), None).await.unwrap(); + let needs_throttle = if let Some(dst_proj) = dst_project { + // check the public state + let state = metadata.state(); + if state != dst_proj.state { + let update = doc! {"$set": {"state": state}}; + dst_projects.update_one(query, update, None).await.unwrap(); + true + } else { + false + } + } else { let project = download(&src_s3, &config.source.s3.bucket, metadata).await; let metadata = upload(&dst_s3, &config.target.s3.bucket, project).await; dst_projects.insert_one(&metadata, None).await.unwrap(); + true + }; + + progress.inc(1); + + if needs_throttle { // throttle to about 2k req/sec to avoid 503 errors from AWS thread::sleep(Duration::from_millis(10)); } - progress.inc(1); } progress.println("Project migration complete."); progress.finish(); diff --git a/crates/migrate/src/origin.rs b/crates/migrate/src/origin.rs index 0845a82f..77aed469 100644 --- a/crates/migrate/src/origin.rs +++ b/crates/migrate/src/origin.rs @@ -139,16 +139,30 @@ pub(crate) struct ProjectMetadata { pub(crate) roles: HashMap, //pub(crate) transient: Option, - //#[serde(rename = "camelCase")] + //#[serde(rename = "deleteAt")] //pub(crate) delete_at: Option, - #[serde(rename = "camelCase")] + #[serde(rename = "lastUpdateAt")] pub(crate) last_update_at: Option, - //#[serde(rename = "camelCase")] + //#[serde(rename = "lastUpdatedAt")] //pub(crate) last_updated_at: Option, - #[serde(rename = "PascalCase")] + #[serde(rename = "Public")] pub(crate) public: Option, } +impl ProjectMetadata { + pub(crate) fn state(&self) -> PublishState { + self.public + .map(|is_public| { + if is_public { + PublishState::Public + } else { + PublishState::Private + } + }) + .unwrap_or(PublishState::Private) + } +} + #[derive(Deserialize)] #[serde(rename_all = "PascalCase")] pub(crate) struct RoleMetadata { @@ -235,4 +249,19 @@ mod tests { let _group: Group = serde_json::from_str(group_str) .unwrap_or_else(|_err| panic!("Unable to parse group from {}", group_str)); } + + #[test] + fn deserialize_project_public() { + let proj_str = "{ + \"_id\": \"599aed7fc4913219dca051d2\", + \"owner\": \"brian\", + \"name\": \"test_project\", + \"collaborators\": [], + \"roles\": {}, + \"Public\": true + }"; + let project: ProjectMetadata = serde_json::from_str(proj_str).unwrap(); + + assert!(project.public.unwrap()); + } }