From 4612921db9e7e2941a04ca114c6ea56f01ce5b1b Mon Sep 17 00:00:00 2001 From: Mohsin Zaidi <2236875+smrz2001@users.noreply.github.com> Date: Tue, 25 Jun 2024 11:42:22 -0400 Subject: [PATCH] feat: expose c1 migration cmds (#186) * feat: expose c1 migration cmds * chore: build fixes * fix: use list of args for migration_cmd * fix: tests --------- Co-authored-by: Nathaniel Cook --- keramik/src/ipfs.md | 43 ++++++ operator/src/network/cas.rs | 1 + operator/src/network/ceramic.rs | 61 ++++---- operator/src/network/controller.rs | 145 +++++++++++++++++- operator/src/network/ipfs.rs | 30 +++- operator/src/network/spec.rs | 2 + runner/src/scenario/ceramic/model_instance.rs | 2 +- 7 files changed, 250 insertions(+), 34 deletions(-) diff --git a/keramik/src/ipfs.md b/keramik/src/ipfs.md index 7a3db292..f706a11b 100644 --- a/keramik/src/ipfs.md +++ b/keramik/src/ipfs.md @@ -169,3 +169,46 @@ spec: commands: - ipfs config --json Swarm.RelayClient.Enabled false ``` + +## Migration from Kubo to Ceramic One + +A Kubo blockstore can be migrated to Ceramic One by specifying the migration command in the IPFS configuration. + +Example [network config](./setup_network.md) that uses Go based IPFS (i.e. Kubo) with its defaults for Ceramic (including a default +blockstore path of `/data/ipfs`) and the Ceramic network set to `dev-unstable`. + +```yaml +apiVersion: "keramik.3box.io/v1alpha1" +kind: Network +metadata: + name: basic-network +spec: + replicas: 5 + ceramic: + - ipfs: + go: {} + networkType: dev-unstable +``` + +Example [network config](./setup_network.md) that uses Ceramic One and specifies what migration command to run before +starting up the node. + +```yaml +apiVersion: "keramik.3box.io/v1alpha1" +kind: Network +metadata: + name: basic-network +spec: + replicas: 5 + ceramic: + - ipfs: + rust: + migrationCmd: + - from-ipfs + - -i + - /data/ipfs/blocks + - -o + - /data/ipfs/ + - --network + - dev-unstable +``` diff --git a/operator/src/network/cas.rs b/operator/src/network/cas.rs index 4b9181bb..617e546b 100644 --- a/operator/src/network/cas.rs +++ b/operator/src/network/cas.rs @@ -600,6 +600,7 @@ pub fn cas_ipfs_stateful_set_spec( ..Default::default() }), spec: Some(PodSpec { + init_containers: config.ipfs.init_container(net_config).map(|c| vec![c]), containers: vec![config.ipfs.container(ipfs_info, net_config)], volumes: Some(volumes), ..Default::default() diff --git a/operator/src/network/ceramic.rs b/operator/src/network/ceramic.rs index 8f8487e8..43b9390e 100644 --- a/operator/src/network/ceramic.rs +++ b/operator/src/network/ceramic.rs @@ -664,35 +664,40 @@ pub fn stateful_set_spec(ns: &str, bundle: &CeramicBundle<'_>) -> StatefulSetSpe .ipfs .container(&bundle.info, bundle.net_config), ], - init_containers: Some(vec![Container { - command: Some(vec![ - "/bin/bash".to_owned(), - "-c".to_owned(), - "/ceramic-init/ceramic-init.sh".to_owned(), - ]), - env: Some(init_env), - image: Some(bundle.config.init_image_name.to_owned()), - image_pull_policy: Some(bundle.config.image_pull_policy.to_owned()), - name: "init-ceramic-config".to_owned(), - resources: Some(ResourceRequirements { - limits: Some(bundle.config.resource_limits.clone().into()), - requests: Some(bundle.config.resource_limits.clone().into()), - ..Default::default() - }), - volume_mounts: Some(vec![ - VolumeMount { - mount_path: "/config".to_owned(), - name: "config-volume".to_owned(), - ..Default::default() - }, - VolumeMount { - mount_path: "/ceramic-init".to_owned(), - name: "ceramic-init".to_owned(), + init_containers: Some( + vec![Container { + command: Some(vec![ + "/bin/bash".to_owned(), + "-c".to_owned(), + "/ceramic-init/ceramic-init.sh".to_owned(), + ]), + env: Some(init_env), + image: Some(bundle.config.init_image_name.to_owned()), + image_pull_policy: Some(bundle.config.image_pull_policy.to_owned()), + name: "init-ceramic-config".to_owned(), + resources: Some(ResourceRequirements { + limits: Some(bundle.config.resource_limits.clone().into()), + requests: Some(bundle.config.resource_limits.clone().into()), ..Default::default() - }, - ]), - ..Default::default() - }]), + }), + volume_mounts: Some(vec![ + VolumeMount { + mount_path: "/config".to_owned(), + name: "config-volume".to_owned(), + ..Default::default() + }, + VolumeMount { + mount_path: "/ceramic-init".to_owned(), + name: "ceramic-init".to_owned(), + ..Default::default() + }, + ]), + ..Default::default() + }] + .into_iter() + .chain(bundle.config.ipfs.init_container(bundle.net_config)) + .collect(), + ), volumes: Some(volumes), security_context: Some(PodSecurityContext { fs_group: Some(70), diff --git a/operator/src/network/controller.rs b/operator/src/network/controller.rs index 53067ed0..444fb051 100644 --- a/operator/src/network/controller.rs +++ b/operator/src/network/controller.rs @@ -861,8 +861,8 @@ async fn update_peer_status( for ceramic in ceramics { for i in 0..ceramic.info.replicas { let pod_name = ceramic.info.pod_name(i); - let pod = pods.get_status(&pod_name).await?; - if !is_pod_ready(&pod) { + let pod = pods.get_status(&pod_name).await; + if pod.map(|pod| !is_pod_ready(&pod)).unwrap_or(true) { debug!(pod_name, "peer is not ready skipping"); continue; } @@ -4700,4 +4700,145 @@ mod tests { .expect("reconciler"); timeout_after_1s(mocksrv).await; } + #[tokio::test] + #[traced_test] + async fn migration_cmd() { + // Setup network spec and status + let network = Network::test().with_spec(NetworkSpec { + ceramic: Some(vec![CeramicSpec { + ipfs: Some(IpfsSpec::Rust(RustIpfsSpec { + migration_cmd: Some( + vec![ + "from-ipfs", + "-i", + "/data/ipfs/blocks", + "-o", + "/data/ipfs/", + "--network", + "dev-unstable", + ] + .into_iter() + .map(ToOwned::to_owned) + .collect(), + ), + ..Default::default() + })), + ..Default::default() + }]), + ..Default::default() + }); + let mock_rpc_client = default_ipfs_rpc_mock(); + let mut stub = Stub::default().with_network(network.clone()); + stub.ceramics[0].stateful_set.patch(expect![[r#" + --- original + +++ modified + @@ -397,6 +397,95 @@ + "name": "ceramic-init" + } + ] + + }, + + { + + "command": [ + + "/usr/bin/ceramic-one", + + "migrations", + + "from-ipfs", + + "-i", + + "/data/ipfs/blocks", + + "-o", + + "/data/ipfs/", + + "--network", + + "dev-unstable" + + ], + + "env": [ + + { + + "name": "CERAMIC_ONE_BIND_ADDRESS", + + "value": "0.0.0.0:5001" + + }, + + { + + "name": "CERAMIC_ONE_KADEMLIA_PARALLELISM", + + "value": "1" + + }, + + { + + "name": "CERAMIC_ONE_KADEMLIA_REPLICATION", + + "value": "6" + + }, + + { + + "name": "CERAMIC_ONE_LOCAL_NETWORK_ID", + + "value": "0" + + }, + + { + + "name": "CERAMIC_ONE_METRICS_BIND_ADDRESS", + + "value": "0.0.0.0:9465" + + }, + + { + + "name": "CERAMIC_ONE_NETWORK", + + "value": "local" + + }, + + { + + "name": "CERAMIC_ONE_STORE_DIR", + + "value": "/data/ipfs" + + }, + + { + + "name": "CERAMIC_ONE_SWARM_ADDRESSES", + + "value": "/ip4/0.0.0.0/tcp/4001" + + }, + + { + + "name": "RUST_LOG", + + "value": "info,ceramic_one=debug,multipart=error" + + } + + ], + + "image": "public.ecr.aws/r5b3e0r5/3box/ceramic-one:latest", + + "imagePullPolicy": "Always", + + "name": "ipfs-migration", + + "ports": [ + + { + + "containerPort": 4001, + + "name": "swarm-tcp", + + "protocol": "TCP" + + }, + + { + + "containerPort": 5001, + + "name": "rpc", + + "protocol": "TCP" + + }, + + { + + "containerPort": 9465, + + "name": "metrics", + + "protocol": "TCP" + + } + + ], + + "resources": { + + "limits": { + + "cpu": "1", + + "ephemeral-storage": "1Gi", + + "memory": "1Gi" + + }, + + "requests": { + + "cpu": "1", + + "ephemeral-storage": "1Gi", + + "memory": "1Gi" + + } + + }, + + "volumeMounts": [ + + { + + "mountPath": "/data/ipfs", + + "name": "ipfs-data" + + } + + ] + } + ], + "securityContext": { + "#]]); + stub.cas_ipfs_stateful_set.patch(expect![[r#" + --- original + +++ modified + "#]]); + let (testctx, api_handle) = Context::test(mock_rpc_client); + let fakeserver = ApiServerVerifier::new(api_handle); + let mocksrv = stub.run(fakeserver); + reconcile(Arc::new(network), testctx) + .await + .expect("reconciler"); + timeout_after_1s(mocksrv).await; + } } diff --git a/operator/src/network/ipfs.rs b/operator/src/network/ipfs.rs index a3562a95..7c0c6228 100644 --- a/operator/src/network/ipfs.rs +++ b/operator/src/network/ipfs.rs @@ -9,6 +9,7 @@ use k8s_openapi::{ }; const IPFS_CONTAINER_NAME: &str = "ipfs"; +const IPFS_STORE_DIR: &str = "/data/ipfs"; pub const IPFS_DATA_PV_CLAIM: &str = "ipfs-data"; const IPFS_SERVICE_PORT: i32 = 5001; @@ -69,6 +70,12 @@ impl IpfsConfig { IpfsConfig::Go(config) => config.config_maps(&info), } } + pub fn init_container(&self, net_config: &NetworkConfig) -> Option { + match self { + IpfsConfig::Rust(config) => config.init_container(net_config), + _ => None, + } + } pub fn container(&self, info: impl Into, net_config: &NetworkConfig) -> Container { let info = info.into(); match self { @@ -98,6 +105,7 @@ pub struct RustIpfsConfig { storage: PersistentStorageConfig, rust_log: String, env: Option>, + migration_cmd: Option>, } impl RustIpfsConfig { @@ -129,6 +137,7 @@ impl Default for RustIpfsConfig { }, rust_log: "info,ceramic_one=debug,multipart=error".to_owned(), env: None, + migration_cmd: None, } } } @@ -149,6 +158,7 @@ impl From for RustIpfsConfig { storage: PersistentStorageConfig::from_spec(value.storage, default.storage), rust_log: value.rust_log.unwrap_or(default.rust_log), env: value.env, + migration_cmd: value.migration_cmd, } } } @@ -177,7 +187,7 @@ impl RustIpfsConfig { }, EnvVar { name: "CERAMIC_ONE_STORE_DIR".to_owned(), - value: Some("/data/ipfs".to_owned()), + value: Some(IPFS_STORE_DIR.to_owned()), ..Default::default() }, EnvVar { @@ -253,7 +263,7 @@ impl RustIpfsConfig { ..Default::default() }), volume_mounts: Some(vec![VolumeMount { - mount_path: "/data/ipfs".to_owned(), + mount_path: IPFS_STORE_DIR.to_owned(), name: IPFS_DATA_PV_CLAIM.to_owned(), ..Default::default() }]), @@ -261,6 +271,20 @@ impl RustIpfsConfig { ..Default::default() } } + + fn init_container(&self, net_config: &NetworkConfig) -> Option { + self.migration_cmd.as_ref().map(|cmd| Container { + name: "ipfs-migration".to_string(), + command: Some( + vec!["/usr/bin/ceramic-one", "migrations"] + .into_iter() + .chain(cmd.iter().map(String::as_str)) + .map(ToOwned::to_owned) + .collect(), + ), + ..self.container(net_config) + }) + } } pub struct GoIpfsConfig { @@ -364,7 +388,7 @@ ipfs config --json Swarm.ResourceMgr.MaxFileDescriptors 500000 fn container(&self, info: &IpfsInfo) -> Container { let mut volume_mounts = vec![ VolumeMount { - mount_path: "/data/ipfs".to_owned(), + mount_path: IPFS_STORE_DIR.to_owned(), name: IPFS_DATA_PV_CLAIM.to_owned(), ..Default::default() }, diff --git a/operator/src/network/spec.rs b/operator/src/network/spec.rs index 66b68852..f19a4e79 100644 --- a/operator/src/network/spec.rs +++ b/operator/src/network/spec.rs @@ -204,6 +204,8 @@ pub struct RustIpfsSpec { /// Extra env values to pass to the image. /// CAUTION: Any env vars specified in this set will override any predefined values. pub env: Option>, + /// Migration command that should run before a node comes up + pub migration_cmd: Option>, } /// Describes how the Go IPFS node for a peer should behave. diff --git a/runner/src/scenario/ceramic/model_instance.rs b/runner/src/scenario/ceramic/model_instance.rs index 292a37c0..220d4fd3 100644 --- a/runner/src/scenario/ceramic/model_instance.rs +++ b/runner/src/scenario/ceramic/model_instance.rs @@ -258,8 +258,8 @@ impl CeramicModelInstanceTestUser { config, user_info: GooseUserInfo { lead_user, - global_leader, lead_worker: is_goose_lead_worker(), + global_leader, }, small_model_id, small_model_instance_ids,