From 69055bff6696ce6cd3ba538702dd09a9f2f9e350 Mon Sep 17 00:00:00 2001 From: Simon LUCIDO Date: Tue, 21 Nov 2023 12:03:15 +0100 Subject: [PATCH] feat(initramfs): accept other media types We used to only accept the old ManifestV1 response type. However, most images don't come with this manifest type anymore, but instead has a ManifestV2 List / OCI Spec'd manifest. This commit handles those manifests types: - OCI List - ManifestV2 List - ManifestV2 Signed-off-by: Simon LUCIDO --- initramfs/src/httpclient.rs | 20 ++- initramfs/src/main.rs | 2 +- initramfs/src/registry.rs | 285 +++++++++++++++++++++++++++++++----- 3 files changed, 264 insertions(+), 43 deletions(-) diff --git a/initramfs/src/httpclient.rs b/initramfs/src/httpclient.rs index c979384..2a0f1f2 100644 --- a/initramfs/src/httpclient.rs +++ b/initramfs/src/httpclient.rs @@ -1,14 +1,24 @@ use anyhow::{anyhow, Result}; use reqwest::{Client, Response}; -pub async fn run_get_request(url: &str, token: Option<&str>) -> Result { - let res = match token { +pub async fn run_get_request( + url: &str, + token: Option<&String>, + headers: Vec<(String, String)>, +) -> Result { + let mut res = match token { Some(token) => Client::new().get(url).bearer_auth(token), None => Client::new().get(url), + }; + + for (key, value) in headers { + res = res.header(key, value); } - .send() - .await - .map_err(|e| anyhow!(e).context(format!("Failed to run request to {}", url)))?; + + let res = res + .send() + .await + .map_err(|e| anyhow!(e).context(format!("Failed to run request to {}", url)))?; if !res.status().is_success() { return Err(anyhow!(res.text().await.unwrap_or_default()) diff --git a/initramfs/src/main.rs b/initramfs/src/main.rs index ff56603..10a6a64 100644 --- a/initramfs/src/main.rs +++ b/initramfs/src/main.rs @@ -52,7 +52,7 @@ async fn main() -> Result<(), Box> { let args = Args::parse(); debug!("Running cli with arguments : {:?}", args); - let registry = Registry::new(&args.registry_url, &args.auth_url); + let mut registry = Registry::new(&args.registry_url, &args.auth_url); info!("Downloading image {}", &args.image); let image = registry.get_image(&args.image).await?; diff --git a/initramfs/src/registry.rs b/initramfs/src/registry.rs index 81818b3..213c314 100644 --- a/initramfs/src/registry.rs +++ b/initramfs/src/registry.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use anyhow::{anyhow, Result}; -use log::{debug, info}; +use log::{debug, info, trace}; use serde::Deserialize; use crate::{ @@ -9,20 +9,94 @@ use crate::{ image::{Image, Layer, LayerMetadata}, }; -#[derive(Deserialize)] -struct TokenResponse { - token: String, +const DEFAULT_PLATFORM_OS: &str = "linux"; +const DEFAULT_PLATFORM_ARCHITECTURE: &str = "amd64"; + +#[derive(Deserialize, Debug)] +struct ManifestListV2Response { + manifests: Vec, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ManifestV2Response { + layers: Vec, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct OCIv1ListResponse { + manifests: Vec, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct LayerInfo { + digest: String, + platform: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ManifestV2ItemResponse { + digest: String, + platform: Option, +} + +#[derive(Deserialize, Debug)] +struct Platform { + architecture: String, + os: String, } #[derive(Deserialize)] -#[allow(non_snake_case)] -struct ManifestResponse { - fsLayers: Vec, +struct TokenResponse { + token: String, } pub struct Registry { url: String, auth_url: String, + token: Option, +} + +#[derive(Debug)] +pub enum ManifestType { + ManifestListV2, + ManifestV2, + OCIv1List, + OCIv1, +} + +impl TryFrom<&str> for ManifestType { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + match value { + "application/vnd.docker.distribution.manifest.list.v2+json" => { + Ok(ManifestType::ManifestListV2) + } + "application/vnd.docker.distribution.manifest.v2+json" => Ok(ManifestType::ManifestV2), + "application/vnd.oci.image.index.v1+json" => Ok(ManifestType::OCIv1List), + "application/vnd.oci.image.manifest.v1+json" => Ok(ManifestType::OCIv1List), + _ => Err(anyhow!("Unknown manifest type")), + } + } +} + +impl ToString for ManifestType { + fn to_string(&self) -> String { + match self { + ManifestType::ManifestListV2 => { + "application/vnd.docker.distribution.manifest.list.v2+json".to_string() + } + ManifestType::ManifestV2 => { + "application/vnd.docker.distribution.manifest.v2+json".to_string() + } + ManifestType::OCIv1List => "application/vnd.oci.image.index.v1+json".to_string(), + ManifestType::OCIv1 => "application/vnd.oci.image.manifest.v1+json".to_string(), + } + } } impl Registry { @@ -30,68 +104,208 @@ impl Registry { Self { url: url.to_string(), auth_url: auth_url.to_string(), + token: None, } } - async fn get_token(&self, image_name: &str) -> Result { + async fn set_token(&mut self, image_name: &str) -> Result<()> { let res = run_get_request( &format!( "{}?service=registry.docker.io&scope=repository:{}:pull", self.auth_url, image_name ), None, + Vec::new(), ) .await?; // Extract the token from the response - let token = res - .json::() - .await - .map_err(|e| anyhow!(e).context("Failed to parse response"))? - .token; + self.token = Some( + res.json::() + .await + .map_err(|e| anyhow!(e).context("Failed to parse response"))? + .token, + ); - debug!("Successfully got auth token : {:?}", token); + debug!("Successfully got auth token"); - Ok(token) + Ok(()) } async fn get_layers_metadata( &self, - token: &str, image_name: &str, image_tag: &str, ) -> Result> { + debug!( + "Url: {}", + format!("{}/{}/manifests/{}", self.url, image_name, image_tag) + ); + + let headers = vec![ + ( + "Accept".to_string(), + ManifestType::ManifestListV2.to_string(), + ), + ("Accept".to_string(), ManifestType::ManifestV2.to_string()), + ]; + let res = run_get_request( &format!("{}/{}/manifests/{}", self.url, image_name, image_tag), - Some(token), + self.token.as_ref(), + headers, ) .await?; - // Extract the information about the layers from the manifest - let layers_metadata = res - .json::() - .await - .map_err(|e| anyhow!(e).context("Failed to parse response"))? - .fsLayers; + let content_type = res + .headers() + .get("content-type") + .ok_or(anyhow!("No content type found"))? + .to_str() + .map_err(|e| anyhow!(e).context("Failed to parse content type"))?; + + let manifest_type = ManifestType::try_from(content_type).map_err(|e| { + anyhow!(e).context(format!("Failed to parse content type : {}", content_type)) + })?; + + debug!("Manifest type : {:?}", manifest_type); + + let manifest = match manifest_type { + ManifestType::ManifestV2 => res + .json::() + .await + .map_err(|e| anyhow!(e).context("Failed to parse response"))?, + ManifestType::ManifestListV2 => { + let manifest = res + .json::() + .await + .map_err(|e| anyhow!(e).context("Failed to parse response"))?; - debug!("Successfully got layers : {:?}", layers_metadata); + self.manifest_from_manifest_v2_list(manifest, image_name) + .await? + } + ManifestType::OCIv1List => { + let manifest = res + .json::() + .await + .map_err(|e| anyhow!(e).context("Failed to parse response"))?; + + self.manifest_from_oci_list(manifest, image_name).await? + } + ManifestType::OCIv1 => unimplemented!(), + }; + + let layers_metadata = manifest + .layers + .into_iter() + .map(|layer| LayerMetadata { + blob_sum: layer.digest.clone(), + }) + .collect::>(); + + debug!( + "Successfully got layers : \n\t{}", + layers_metadata + .iter() + .map(|layer| layer.blob_sum.clone()) + .collect::>() + .join("\n\t") + ); Ok(layers_metadata) } - async fn get_layer( + async fn manifest_from_manifest_v2_list( + &self, + manifest: ManifestListV2Response, + image_name: &str, + ) -> Result { + let sub_manifest = manifest + .manifests + .iter() + .find(|manifest| { + manifest + .platform + .as_ref() + .map(|platform| { + platform.os == DEFAULT_PLATFORM_OS + && platform.architecture == DEFAULT_PLATFORM_ARCHITECTURE + }) + .unwrap_or(false) + }) + .map(|manifest| manifest.digest.clone()) + .ok_or(anyhow!("No manifest found"))?; + + trace!("Sub manifest : {:?}", sub_manifest); + + self.manifest_from_oci(sub_manifest, image_name).await + } + + async fn manifest_from_oci_list( + &self, + oci_manifest: OCIv1ListResponse, + image_name: &str, + ) -> Result { + let sub_manifest = oci_manifest + .manifests + .into_iter() + .find(|manifest| { + manifest + .platform + .as_ref() + .map(|platform| { + platform.os == DEFAULT_PLATFORM_OS + && platform.architecture == DEFAULT_PLATFORM_ARCHITECTURE + }) + .unwrap_or(false) + }) + .ok_or(anyhow!("No manifest found"))?; + + trace!("Sub manifest : {:?}", sub_manifest); + + self.manifest_from_oci(sub_manifest.digest, image_name) + .await + } + + async fn manifest_from_oci( &self, - token: &str, + oci_manifest: String, image_name: &str, - layer_metadata: &LayerMetadata, - ) -> Result { + ) -> Result { + debug!("Downloading manifest for linux/amd64"); + + // Extract the information about the layers from the manifest + let headers = vec![ + ("Accept".to_string(), ManifestType::OCIv1.to_string()), + ("Accept".to_string(), ManifestType::ManifestV2.to_string()), + ]; + + let res = run_get_request( + &format!("{}/{}/manifests/{}", self.url, image_name, oci_manifest), + self.token.as_ref(), + headers, + ) + .await?; + + let res = res + .json::() + .await + .map_err(|e| anyhow!(e).context("Failed to parse response"))?; + + trace!("Manifest : {:?}", res); + + Ok(res) + } + + async fn get_layer(&self, image_name: &str, layer_metadata: &LayerMetadata) -> Result { // Make a request to the docker hub to get the layer from his blobSum let res = run_get_request( &format!( "{}/{}/blobs/{}", self.url, image_name, layer_metadata.blob_sum ), - Some(token), + self.token.as_ref(), + Vec::new(), ) .await?; @@ -112,7 +326,6 @@ impl Registry { async fn get_layers( &self, - token: &str, image_name: &str, layers_metadata: Vec, ) -> Result> { @@ -121,7 +334,7 @@ impl Registry { info!("Pulling layer {:?}", layer_metadata.blob_sum); let layer = self - .get_layer(token, image_name, layer_metadata) + .get_layer(image_name, layer_metadata) .await .map_err(|e| anyhow!(e).context("Failed to pull layer"))?; layers.push(layer); @@ -134,14 +347,12 @@ impl Registry { Ok(layers) } - pub async fn get_image(&self, image_full_name: &str) -> Result { + pub async fn get_image(&mut self, image_full_name: &str) -> Result { let mut image = Image::new(image_full_name)?; let image_name = &image.name(); - let token = self.get_token(image_name).await?; - let layers_metadata = self - .get_layers_metadata(&token, image_name, image.tag()) - .await?; - let layers = self.get_layers(&token, image_name, layers_metadata).await?; + self.set_token(image_name).await?; + let layers_metadata = self.get_layers_metadata(image_name, image.tag()).await?; + let layers = self.get_layers(image_name, layers_metadata).await?; image.layers = layers;