Skip to content

Commit

Permalink
Merge pull request #158 from kpcyrd/reqwest
Browse files Browse the repository at this point in the history
Refactor dependency on http client, replace dependency on openssl with memory-safe implementation
  • Loading branch information
kpcyrd authored Dec 11, 2024
2 parents 0eb0b86 + 74ec1f3 commit 15f1e77
Show file tree
Hide file tree
Showing 24 changed files with 324 additions and 312 deletions.
414 changes: 194 additions & 220 deletions Cargo.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
FROM rust:alpine3.18
ENV RUSTFLAGS="-C target-feature=-crt-static"
WORKDIR /usr/src/rebuilderd
RUN apk add --no-cache musl-dev openssl-dev shared-mime-info sqlite-dev xz-dev zstd-dev
RUN apk add --no-cache musl-dev shared-mime-info sqlite-dev xz-dev zstd-dev
COPY . .
RUN --mount=type=cache,target=/var/cache/buildkit \
CARGO_HOME=/var/cache/buildkit/cargo \
Expand All @@ -11,7 +11,7 @@ RUN --mount=type=cache,target=/var/cache/buildkit \
/var/cache/buildkit/target/release/rebuildctl /

FROM alpine:3.18
RUN apk add --no-cache libgcc openssl shared-mime-info sqlite-libs xz zstd-libs
RUN apk add --no-cache libgcc shared-mime-info sqlite-libs xz zstd-libs
COPY --from=0 \
/rebuilderd /rebuildctl \
/usr/local/bin/
Expand Down
2 changes: 1 addition & 1 deletion PKGBUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ pkgdesc='Independent verification system of binary packages'
url='https://github.com/kpcyrd/rebuilderd'
arch=('x86_64')
license=('GPL3')
depends=('openssl' 'shared-mime-info' 'xz' 'libzstd.so')
depends=('shared-mime-info' 'xz' 'libzstd.so')
makedepends=('cargo' 'sqlite' 'scdoc')
backup=('etc/rebuilderd.conf'
'etc/rebuilderd-sync.conf'
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ cargo run --bin rebuilderd-worker -- \

## Dependencies

Debian: pkg-config liblzma-dev libssl-dev libsqlite3-dev libzstd-dev
Debian: pkg-config liblzma-dev libsqlite3-dev libzstd-dev

# Funding

Expand Down
8 changes: 4 additions & 4 deletions common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ edition = "2021"

[dependencies]
anyhow = "1.0.58"
chrono = { version = "0.4.19", features=["serde"] }
clap = "4.5.21"
chrono = { version = "0.4.19", features = ["serde"] }
clap = { version = "4.5.21", features = ["derive"] }
colored = "2.0.0"
dirs-next = "2.0.0"
log = "0.4.17"
reqwest = { version="0.12", features=["blocking", "json"] }
serde = { version="1.0.137", features=["derive"] }
reqwest = { version = "0.12", features = ["blocking", "json", "stream", "rustls-tls-native-roots"], default-features = false }
serde = { version = "1.0.137", features = ["derive"] }
toml = "0.8"
url = "2.2.2"
86 changes: 56 additions & 30 deletions common/src/api.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
use chrono::prelude::*;
use crate::config::ConfigFile;
use crate::errors::*;
use crate::{PkgRelease, PkgArtifact, PkgGroup, Status};
use crate::auth;
use reqwest::{Client as HttpClient, RequestBuilder};
use serde::{Serialize, Deserialize};
use crate::{auth, http, PkgArtifact, PkgGroup, PkgRelease, Status};
use chrono::prelude::*;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::HashMap;
use std::env;
Expand All @@ -16,7 +14,7 @@ pub const SIGNUP_SECRET_HEADER: &str = "X-Signup-Secret";

pub struct Client {
endpoint: Url,
client: HttpClient,
client: http::Client,
is_default_endpoint: bool,
auth_cookie: Option<String>,
worker_key: Option<String>,
Expand All @@ -26,7 +24,9 @@ pub struct Client {
impl Client {
pub fn new(config: ConfigFile, endpoint: Option<String>) -> Result<Client> {
let (endpoint, auth_cookie, is_default_endpoint) = if let Some(endpoint) = endpoint {
let cookie = config.endpoints.get(&endpoint)
let cookie = config
.endpoints
.get(&endpoint)
.map(|e| e.cookie.to_string());
(endpoint, cookie, false)
} else if let Some(endpoint) = config.http.endpoint {
Expand All @@ -35,15 +35,18 @@ impl Client {
("http://127.0.0.1:8484".to_string(), None, true)
};

let mut endpoint = endpoint.parse::<Url>()
let mut endpoint = endpoint
.parse::<Url>()
.with_context(|| anyhow!("Failed to parse endpoint as url: {:?}", endpoint))?;

// If the url ends with a slash, remove it
endpoint.path_segments_mut().map_err(|_| anyhow!("Given endpoint url cannot be base"))?
endpoint
.path_segments_mut()
.map_err(|_| anyhow!("Given endpoint url cannot be base"))?
.pop_if_empty();

debug!("Setting rebuilderd endpoint to {:?}", endpoint.as_str());
let client = HttpClient::new();
let client = http::client()?;
Ok(Client {
endpoint,
client,
Expand All @@ -57,12 +60,11 @@ impl Client {
pub fn with_auth_cookie(&mut self) -> Result<&mut Self> {
if let Ok(cookie_path) = env::var("REBUILDERD_COOKIE_PATH") {
debug!("Found cookie path in environment: {:?}", cookie_path);
let auth_cookie = auth::read_cookie_from_file(cookie_path)
.context("Failed to load auth cookie")?;
let auth_cookie =
auth::read_cookie_from_file(cookie_path).context("Failed to load auth cookie")?;
Ok(self.auth_cookie(auth_cookie))
} else if self.is_default_endpoint {
let auth_cookie = auth::find_auth_cookie()
.context("Failed to load auth cookie")?;
let auth_cookie = auth::find_auth_cookie().context("Failed to load auth cookie")?;
Ok(self.auth_cookie(auth_cookie))
} else {
Ok(self)
Expand Down Expand Up @@ -94,7 +96,7 @@ impl Client {
url
}

pub fn get(&self, path: Cow<'static,str>) -> RequestBuilder {
pub fn get(&self, path: Cow<'static, str>) -> http::RequestBuilder {
let mut req = self.client.get(self.url_join(&path));
if let Some(auth_cookie) = &self.auth_cookie {
req = req.header(AUTH_COOKIE_HEADER, auth_cookie);
Expand All @@ -108,7 +110,7 @@ impl Client {
req
}

pub fn post(&self, path: Cow<'static, str>) -> RequestBuilder {
pub fn post(&self, path: Cow<'static, str>) -> http::RequestBuilder {
let mut req = self.client.post(self.url_join(&path));
if let Some(auth_cookie) = &self.auth_cookie {
req = req.header(AUTH_COOKIE_HEADER, auth_cookie);
Expand All @@ -123,7 +125,8 @@ impl Client {
}

pub async fn list_workers(&self) -> Result<Vec<Worker>> {
let workers = self.get(Cow::Borrowed("api/v0/workers"))
let workers = self
.get(Cow::Borrowed("api/v0/workers"))
.send()
.await?
.error_for_status()?
Expand All @@ -143,7 +146,8 @@ impl Client {
}

pub async fn list_pkgs(&self, list: &ListPkgs) -> Result<Vec<PkgRelease>> {
let pkgs = self.get(Cow::Borrowed("api/v0/pkgs/list"))
let pkgs = self
.get(Cow::Borrowed("api/v0/pkgs/list"))
.query(list)
.send()
.await?
Expand All @@ -160,15 +164,17 @@ impl Client {
bail!("Filter matched too many packages: {}", pkgs.len());
}

let pkg = pkgs.into_iter()
let pkg = pkgs
.into_iter()
.next()
.context("Filter didn't match any packages on this rebuilder")?;

Ok(pkg)
}

pub async fn fetch_log(&self, id: i32) -> Result<Vec<u8>> {
let log = self.get(Cow::Owned(format!("api/v0/builds/{}/log", id)))
let log = self
.get(Cow::Owned(format!("api/v0/builds/{}/log", id)))
.send()
.await?
.error_for_status()?
Expand All @@ -178,7 +184,8 @@ impl Client {
}

pub async fn fetch_diffoscope(&self, id: i32) -> Result<Vec<u8>> {
let log = self.get(Cow::Owned(format!("api/v0/builds/{}/diffoscope", id)))
let log = self
.get(Cow::Owned(format!("api/v0/builds/{}/diffoscope", id)))
.send()
.await?
.error_for_status()?
Expand All @@ -188,7 +195,8 @@ impl Client {
}

pub async fn fetch_attestation(&self, id: i32) -> Result<Vec<u8>> {
let attestation = self.get(Cow::Owned(format!("api/v0/builds/{}/attestation", id)))
let attestation = self
.get(Cow::Owned(format!("api/v0/builds/{}/attestation", id)))
.send()
.await?
.error_for_status()?
Expand All @@ -198,7 +206,8 @@ impl Client {
}

pub async fn list_queue(&self, list: &ListQueue) -> Result<QueueList> {
let pkgs = self.post(Cow::Borrowed("api/v0/queue/list"))
let pkgs = self
.post(Cow::Borrowed("api/v0/queue/list"))
.json(list)
.send()
.await?
Expand All @@ -220,7 +229,8 @@ impl Client {
}

pub async fn pop_queue(&self, query: &WorkQuery) -> Result<JobAssignment> {
let assignment = self.post(Cow::Borrowed("api/v0/queue/pop"))
let assignment = self
.post(Cow::Borrowed("api/v0/queue/pop"))
.json(query)
.send()
.await?
Expand Down Expand Up @@ -427,25 +437,41 @@ mod tests {

#[test]
fn test_endpoint_format_example_com() {
let client = Client::new(ConfigFile::default(), Some("https://example.com".into())).unwrap();
let client =
Client::new(ConfigFile::default(), Some("https://example.com".into())).unwrap();
assert_eq!(client.endpoint, "https://example.com".parse().unwrap());
}

#[test]
fn test_endpoint_format_example_com_trailing_slash() {
let client = Client::new(ConfigFile::default(), Some("https://example.com/".into())).unwrap();
let client =
Client::new(ConfigFile::default(), Some("https://example.com/".into())).unwrap();
assert_eq!(client.endpoint, "https://example.com".parse().unwrap());
}

#[test]
fn test_endpoint_format_example_com_with_path() {
let client = Client::new(ConfigFile::default(), Some("https://example.com/re/build".into())).unwrap();
assert_eq!(client.endpoint, "https://example.com/re/build".parse().unwrap());
let client = Client::new(
ConfigFile::default(),
Some("https://example.com/re/build".into()),
)
.unwrap();
assert_eq!(
client.endpoint,
"https://example.com/re/build".parse().unwrap()
);
}

#[test]
fn test_endpoint_format_example_com_with_path_trailing_slash() {
let client = Client::new(ConfigFile::default(), Some("https://example.com/re/build/".into())).unwrap();
assert_eq!(client.endpoint, "https://example.com/re/build".parse().unwrap());
let client = Client::new(
ConfigFile::default(),
Some("https://example.com/re/build/".into()),
)
.unwrap();
assert_eq!(
client.endpoint,
"https://example.com/re/build".parse().unwrap()
);
}
}
10 changes: 4 additions & 6 deletions common/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,25 +27,23 @@ pub fn load<P: AsRef<Path>>(path: Option<P>) -> Result<ConfigFile> {
}

if let Some(path) = path {
let c = load_from(path)?
.ok_or_else(|| format_err!("Failed to read config file"))?;
let c = load_from(path)?.ok_or_else(|| format_err!("Failed to read config file"))?;
config.update(c);
}

Ok(config)
}

fn config_path() -> Result<PathBuf> {
let config_dir = dirs_next::config_dir()
.ok_or_else(|| format_err!("Failed to find config dir"))?;
let config_dir =
dirs_next::config_dir().ok_or_else(|| format_err!("Failed to find config dir"))?;
Ok(config_dir.join("rebuilderd.conf"))
}

fn load_from<P: AsRef<Path>>(path: P) -> Result<Option<ConfigFile>> {
if let Ok(buf) = fs::read_to_string(path.as_ref()) {
debug!("loading config file {:?}", path.as_ref());
let config = toml::from_str(&buf)
.context("Failed to load config")?;
let config = toml::from_str(&buf).context("Failed to load config")?;
Ok(Some(config))
} else {
Ok(None)
Expand Down
4 changes: 2 additions & 2 deletions common/src/errors.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
pub use log::{trace, debug, info, warn, error};
pub use anyhow::{Error, Context, Result, anyhow, format_err, bail};
pub use anyhow::{anyhow, bail, format_err, Context, Error, Result};
pub use log::{debug, error, info, trace, warn};
10 changes: 10 additions & 0 deletions common/src/http.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
use crate::errors::*;
pub use reqwest::{Client, RequestBuilder};
use std::time::Duration;

pub fn client() -> Result<Client> {
Client::builder()
.read_timeout(Duration::from_secs(60))
.build()
.map_err(Error::from)
}
31 changes: 23 additions & 8 deletions common/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::errors::*;
use colored::*;
use chrono::NaiveDateTime;
use serde::{Serialize, Deserialize};
use colored::*;
use serde::{Deserialize, Serialize};
use std::iter::FromIterator;
use std::ops::Deref;
use std::str::FromStr;
Expand All @@ -10,6 +10,7 @@ pub mod api;
pub mod auth;
pub mod config;
pub mod errors;
pub mod http;
pub mod utils;

#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
Expand All @@ -28,7 +29,14 @@ pub struct PkgRelease {
}

impl PkgRelease {
pub fn new(name: String, version: String, distro: String, suite: String, architecture: String, artifact_url: String) -> PkgRelease {
pub fn new(
name: String,
version: String,
distro: String,
suite: String,
architecture: String,
artifact_url: String,
) -> PkgRelease {
PkgRelease {
name,
version,
Expand Down Expand Up @@ -66,7 +74,14 @@ pub struct PkgArtifact {
}

impl PkgGroup {
pub fn new(name: String, version: String, distro: String, suite: String, architecture: String, input_url: Option<String>) -> PkgGroup {
pub fn new(
name: String,
version: String,
distro: String,
suite: String,
architecture: String,
input_url: Option<String>,
) -> PkgGroup {
PkgGroup {
name,
version,
Expand Down Expand Up @@ -113,8 +128,8 @@ pub enum Status {
impl Status {
pub fn fancy(&self) -> String {
match self {
Status::Good => "GOOD ".green().to_string(),
Status::Bad => "BAD ".red().to_string(),
Status::Good => "GOOD ".green().to_string(),
Status::Bad => "BAD ".red().to_string(),
Status::Unknown => "UNKWN".yellow().to_string(),
}
}
Expand All @@ -125,8 +140,8 @@ impl Deref for Status {

fn deref(&self) -> &'static str {
match self {
Status::Good => "GOOD",
Status::Bad => "BAD",
Status::Good => "GOOD",
Status::Bad => "BAD",
Status::Unknown => "UNKWN",
}
}
Expand Down
2 changes: 1 addition & 1 deletion common/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ mod tests {

#[test]
fn test_secs_to_human_12h_10m_30s() {
let x = secs_to_human(3600*12 + 600 + 30);
let x = secs_to_human(3600 * 12 + 600 + 30);
assert_eq!(x, "12h 10m 30s");
}

Expand Down
Loading

0 comments on commit 15f1e77

Please sign in to comment.