feat: add tokio-console support

Adds ability (via feature) to enable tokio-console exporting to debug
the runner or operator.

Also a few misc fixes found when writing expirements.

Cargo.toml

@@ -5,6 +5,7 @@ resolver = "2"
 [workspace.dependencies]
 anyhow = "1"
 clap = { version = "4", features = ["derive", "env"] }
+console-subscriber = "0.2"
 env_logger = "0.10.0"
 expect-patch = { path = "./expect-patch/" }
 keramik-common = { path = "./common/", default-features = false }
@@ -25,7 +26,7 @@ reqwest = { version = "0.11", features = ["json", "multipart"] }
 schemars = "0.8"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1", features = ["full", "tracing"] }
 tonic = { version = "0.8" }
 tracing = "0.1.37"
 tracing-opentelemetry = "0.18"

Makefile

@@ -1,4 +1,4 @@
-CARGO = CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUSTFLAGS="-D warnings" cargo
+CARGO = CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUSTFLAGS="--cfg tokio_unstable -D warnings" cargo
 
 .PHONY: all
 all: build check-fmt check-clippy test

common/Cargo.toml

@@ -7,15 +7,17 @@ edition = "2021"
 [features]
 default = []
 telemetry = [
-    "opentelemetry",
-    "opentelemetry-otlp",
-    "tracing-opentelemetry",
-    "tracing-subscriber",
-    "tracing",
+    "dep:opentelemetry",
+    "dep:opentelemetry-otlp",
+    "dep:tracing-opentelemetry",
+    "dep:tracing-subscriber",
+    "dep:tracing",
 ]
+tokio-console = ["telemetry", "dep:console-subscriber"]
 
 [dependencies]
 anyhow.workspace = true
+console-subscriber = { workspace = true, optional = true }
 gethostname = "0.4.2"
 opentelemetry-otlp = { workspace = true, optional = true }
 opentelemetry = { workspace = true, optional = true }

common/src/telemetry.rs

@@ -10,7 +10,7 @@ use opentelemetry::{
     },
 };
 use opentelemetry_otlp::WithExportConfig;
-use tracing_subscriber::{prelude::*, EnvFilter, Registry};
+use tracing_subscriber::{filter::LevelFilter, prelude::*, EnvFilter, Registry};
 
 /// Initialize tracing and metrics
 pub async fn init(otlp_endpoint: String) -> Result<BasicController> {
@@ -60,15 +60,32 @@ pub async fn init(otlp_endpoint: String) -> Result<BasicController> {
         // Build starts the meter and sets it as the global meter provider
         .build()?;
 
+    // Setup filters
+    // Default to INFO if no env is specified
+    let log_filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::INFO.into())
+        .from_env()?;
+    let otlp_filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::INFO.into())
+        .from_env()?;
+
     // Setup tracing layers
-    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-    let logger = tracing_subscriber::fmt::layer().with_ansi(true).compact();
-    let env_filter = EnvFilter::try_from_default_env().or_else(|_| EnvFilter::try_new("info"))?;
+    let telemetry = tracing_opentelemetry::layer()
+        .with_tracer(tracer)
+        .with_filter(otlp_filter);
+    let logger = tracing_subscriber::fmt::layer()
+        .with_ansi(true)
+        .compact()
+        .with_filter(log_filter);
+
+    let collector = Registry::default().with(telemetry).with(logger);
 
-    let collector = Registry::default()
-        .with(telemetry)
-        .with(logger)
-        .with(env_filter);
+    #[cfg(feature = "tokio-console")]
+    let collector = {
+        let console_filter = EnvFilter::builder().parse("tokio=trace,runtime=trace")?;
+        let console_layer = console_subscriber::spawn().with_filter(console_filter);
+        collector.with(console_layer)
+    };
 
     // Initialize tracing
     tracing::subscriber::set_global_default(collector)?;

k8s/operator/manifests/operator.yaml

@@ -124,7 +124,7 @@ spec:
         - name: OPERATOR_OTLP_ENDPOINT
           value: "https://otel:4317"
         - name: RUST_LOG
-          value: "info,kube=debug,keramik_operator=debug"
+          value: "info"
         #readinessProbe:
         #  httpGet:
         #    path: /health

operator/src/network/cas.rs

@@ -547,6 +547,10 @@ pub fn cas_ipfs_stateful_set_spec(
         template: PodTemplateSpec {
             metadata: Some(ObjectMeta {
                 labels: selector_labels(CAS_IPFS_APP),
+                annotations: Some(BTreeMap::from_iter(vec![(
+                    "prometheus/path".to_owned(),
+                    "/metrics".to_owned(),
+                )])),
                 ..Default::default()
             }),
             spec: Some(PodSpec {