From e2ea82a3445a2775b4e7f9a33a6c7dfde4001e06 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sat, 25 May 2024 12:05:27 +0900 Subject: [PATCH 01/16] wip: start implementing separated cert manager --- Cargo.toml | 2 +- rpxy-certs/Cargo.toml | 15 +++++++++++++++ rpxy-certs/src/lib.rs | 0 rpxy-lib/Cargo.toml | 6 ++++-- submodules/h3 | 2 +- submodules/s2n-quic-h3/Cargo.toml | 3 ++- 6 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 rpxy-certs/Cargo.toml create mode 100644 rpxy-certs/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index fce6eb22..6dde7e44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" publish = false [workspace] -members = ["rpxy-bin", "rpxy-lib"] +members = ["rpxy-bin", "rpxy-lib", "rpxy-certs"] exclude = ["submodules"] resolver = "2" diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml new file mode 100644 index 00000000..e3660b81 --- /dev/null +++ b/rpxy-certs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rpxy-certs" +description = "Cert manager library for `rpxy`" +version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +readme.workspace = true +edition.workspace = true +publish.workspace = true + +[features] + +[dependencies] diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs new file mode 100644 index 00000000..e69de29b diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index 699680a9..7baaf9c2 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -81,8 +81,10 @@ tracing = { version = "0.1.40" } # http/3 quinn = { version = "0.10.2", optional = true } -h3 = { path = "../submodules/h3/h3/", optional = true } -h3-quinn = { path = "../submodules/h3/h3-quinn/", optional = true } +# h3 = { path = "../submodules/h3/h3/", optional = true } +# h3-quinn = { path = "../submodules/h3/h3-quinn/", optional = true } +h3 = { version = "0.0.4", optional = true } +h3-quinn = { version = "0.0.5", optional = true } s2n-quic = { version = "1.37.0", default-features = false, features = [ "provider-tls-rustls", ], optional = true } diff --git a/submodules/h3 b/submodules/h3 index b44edeb6..34bf403b 160000 --- a/submodules/h3 +++ b/submodules/h3 @@ -1 +1 @@ -Subproject commit b44edeb60d234d49c45828395108f7519a048d4b +Subproject commit 34bf403bad8893adc0363e45a74782828aa93ac8 diff --git a/submodules/s2n-quic-h3/Cargo.toml b/submodules/s2n-quic-h3/Cargo.toml index d25c08a5..3c891a81 100644 --- a/submodules/s2n-quic-h3/Cargo.toml +++ b/submodules/s2n-quic-h3/Cargo.toml @@ -12,6 +12,7 @@ publish = false [dependencies] bytes = { version = "1", default-features = false } futures = { version = "0.3", default-features = false } -h3 = { path = "../h3/h3/" } +# h3 = { path = "../h3/h3/" } +h3 = { version = "0.0.4" } s2n-quic = "1.37.0" s2n-quic-core = "0.37.0" From d657f6d766e94a5cffa78747c7ee01d5c6512a33 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 27 May 2024 01:25:41 +0900 Subject: [PATCH 02/16] wip: implelented SingleServerCrypto struct --- rpxy-certs/Cargo.toml | 26 +++++++ rpxy-certs/src/certs.rs | 102 ++++++++++++++++++++++++ rpxy-certs/src/error.rs | 15 ++++ rpxy-certs/src/lib.rs | 68 ++++++++++++++++ rpxy-certs/src/service.rs | 0 rpxy-certs/src/source.rs | 160 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 371 insertions(+) create mode 100644 rpxy-certs/src/certs.rs create mode 100644 rpxy-certs/src/error.rs create mode 100644 rpxy-certs/src/service.rs create mode 100644 rpxy-certs/src/source.rs diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index e3660b81..cd5bdfb5 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -13,3 +13,29 @@ publish.workspace = true [features] [dependencies] +rustc-hash = { version = "1.1.0" } +tracing = { version = "0.1.40" } +# anyhow = "1.0.86" +derive_builder = { version = "0.20.0" } +thiserror = { version = "1.0.61" } +# hot_reload = {version = "0.1.5"} +async-trait = { version = "0.1.80" } +# tokio-rustls = { version = "0.26.0", features = ["early-data"] } +rustls = { version = "0.23.8", default-features = false, features = [ + "aws_lc_rs", +] } +rustls-pemfile = { version = "2.1.2" } +rustls-webpki = { version = "0.102.4", default-features = false, features = [ + "std", + "aws_lc_rs", +] } +x509-parser = { version = "0.16.0" } + +[dev-dependencies] +tokio = { version = "1.37.0", default-features = false, features = [ + # "net", + "rt-multi-thread", + # "time", + # "sync", + "macros", +] } diff --git a/rpxy-certs/src/certs.rs b/rpxy-certs/src/certs.rs new file mode 100644 index 00000000..7df964e2 --- /dev/null +++ b/rpxy-certs/src/certs.rs @@ -0,0 +1,102 @@ +use crate::error::*; +use rustc_hash::FxHashMap as HashMap; +use rustls::{crypto::aws_lc_rs::sign::any_supported_type, pki_types, sign::CertifiedKey}; +use std::sync::Arc; +use x509_parser::prelude::*; + +/* ------------------------------------------------ */ +/// Raw certificates in rustls format +type Certificate = rustls::pki_types::CertificateDer<'static>; +/// Raw private key in rustls format +type PrivateKey = pki_types::PrivateKeyDer<'static>; +/// Client CA trust anchors subject to the subject key identifier +type TrustAnchors = HashMap, pki_types::TrustAnchor<'static>>; + +/* ------------------------------------------------ */ +/// Raw certificates and private keys loaded from files for a single server name +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct SingleServerCrypto { + certs: Vec, + cert_keys: Arc>, + client_ca_certs: Option>, +} + +impl SingleServerCrypto { + /// Create a new instance of SingleServerCrypto + pub fn new(certs: &[Certificate], cert_keys: &Arc>, client_ca_certs: &Option>) -> Self { + Self { + certs: certs.to_owned(), + cert_keys: cert_keys.clone(), + client_ca_certs: client_ca_certs.clone(), + } + } + /// Check if mutual tls is enabled + pub fn is_mutual_tls(&self) -> bool { + self.client_ca_certs.is_some() + } + /* ------------------------------------------------ */ + /// Convert the certificates to bytes in der + pub fn certs_bytes(&self) -> Vec> { + self.certs.iter().map(|c| c.to_vec()).collect() + } + /// Convert the private keys to bytes in der + pub fn cert_keys_bytes(&self) -> Vec> { + self + .cert_keys + .iter() + .map(|k| match k { + pki_types::PrivateKeyDer::Pkcs1(pkcs1) => pkcs1.secret_pkcs1_der().to_owned(), + pki_types::PrivateKeyDer::Sec1(sec1) => sec1.secret_sec1_der().to_owned(), + pki_types::PrivateKeyDer::Pkcs8(pkcs8) => pkcs8.secret_pkcs8_der().to_owned(), + _ => unreachable!(), + }) + .collect() + } + /// Convert the client CA certificates to bytes in der + pub fn client_ca_certs_bytes(&self) -> Option>> { + self.client_ca_certs.as_ref().map(|v| v.iter().map(|c| c.to_vec()).collect()) + } + /* ------------------------------------------------ */ + /// Parse the certificates and private keys for a single server and return a rustls CertifiedKey + pub fn rustls_certified_key(&self) -> Result { + let signing_key = self + .cert_keys + .clone() + .iter() + .find_map(|k| if let Ok(sk) = any_supported_type(k) { Some(sk) } else { None }) + .ok_or_else(|| RpxyCertError::InvalidCertificateAndKey)?; + + let cert = self.certs.iter().map(|c| Certificate::from(c.to_vec())).collect::>(); + Ok(CertifiedKey::new(cert, signing_key)) + } + + /* ------------------------------------------------ */ + /// Parse the client CA certificates and return a hashmap of pairs of a subject key identifier (key) and a trust anchor (value) + pub fn rustls_trust_anchors(&self) -> Result { + //-> Result<(Vec, HashSet>), anyhow::Error> { + let Some(certs) = self.client_ca_certs.as_ref() else { + return Err(RpxyCertError::NoClientCert); + }; + let certs = certs.iter().map(|c| Certificate::from(c.to_vec())).collect::>(); + + let trust_anchors = certs + .iter() + .filter_map(|v| { + // retrieve trust anchor + let trust_anchor = webpki::anchor_from_trusted_cert(v).ok()?; + + // retrieve ca key id (subject key id) + let x509_cert = parse_x509_certificate(v).map(|v| v.1).ok()?; + let mut subject_key_ids = x509_cert.iter_extensions().filter_map(|ext| match ext.parsed_extension() { + ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), + _ => None, + }); + let skid = subject_key_ids.next()?; + + Some((skid.0.to_owned(), trust_anchor.to_owned())) + }) + .collect::>(); + + Ok(trust_anchors) + } +} diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs new file mode 100644 index 00000000..b5135a8c --- /dev/null +++ b/rpxy-certs/src/error.rs @@ -0,0 +1,15 @@ +use thiserror::Error; + +/// Describes things that can go wrong in the Rpxy certificate +#[derive(Debug, Error)] +pub enum RpxyCertError { + /// Error when reading certificates and keys + #[error("Failed to read certificates from file: {0}")] + IoError(#[from] std::io::Error), + /// Error when parsing certificates and keys to generate a rustls CertifiedKey + #[error("Unable to find a valid certificate and key")] + InvalidCertificateAndKey, + /// Error when parsing client CA certificates: No client certificate found + #[error("No client certificate found")] + NoClientCert, +} diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index e69de29b..18b7da52 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -0,0 +1,68 @@ +mod certs; +mod error; +mod service; +mod source; + +#[allow(unused_imports)] +pub(crate) mod log { + pub(crate) use tracing::{debug, error, info, warn}; +} + +pub use crate::{ + certs::SingleServerCrypto, + source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, +}; + +/* ------------------------------------------------ */ +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn read_server_crt_key_files() { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + + let crypto_elem = crypto_elem.unwrap(); + let certificed_key = crypto_elem.rustls_certified_key(); + assert!(certificed_key.is_ok()); + } + + #[tokio::test] + async fn read_server_crt_key_files_with_client_ca_crt() { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt"); + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .client_ca_cert_path(client_ca_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + + let crypto_elem = crypto_elem.unwrap(); + assert!(crypto_elem.is_mutual_tls()); + + let certificed_key = crypto_elem.rustls_certified_key(); + assert!(certificed_key.is_ok()); + + let trust_anchors = crypto_elem.rustls_trust_anchors(); + assert!(trust_anchors.is_ok()); + + let trust_anchors = trust_anchors.unwrap(); + assert_eq!(trust_anchors.len(), 1); + } +} diff --git a/rpxy-certs/src/service.rs b/rpxy-certs/src/service.rs new file mode 100644 index 00000000..e69de29b diff --git a/rpxy-certs/src/source.rs b/rpxy-certs/src/source.rs new file mode 100644 index 00000000..e4dba818 --- /dev/null +++ b/rpxy-certs/src/source.rs @@ -0,0 +1,160 @@ +use crate::{certs::SingleServerCrypto, error::*, log::*}; +use async_trait::async_trait; +use derive_builder::Builder; +use std::{ + fs::File, + io::{self, BufReader, Cursor, Read}, + path::{Path, PathBuf}, + sync::Arc, +}; + +/* ------------------------------------------------ */ +#[async_trait] +// Trait to read certs and keys anywhere from KVS, file, sqlite, etc. +pub trait CryptoSource { + type Error; + + /// read crypto materials from source + async fn read(&self) -> Result; + + /// Returns true when mutual tls is enabled + fn is_mutual_tls(&self) -> bool; +} + +/* ------------------------------------------------ */ +#[derive(Builder, Debug, Clone)] +/// Crypto-related file reader implementing `CryptoSource`` trait +pub struct CryptoFileSource { + #[builder(setter(custom))] + /// Always exist + pub tls_cert_path: PathBuf, + + #[builder(setter(custom))] + /// Always exist + pub tls_cert_key_path: PathBuf, + + #[builder(setter(custom), default)] + /// This may not exist + pub client_ca_cert_path: Option, +} + +impl CryptoFileSourceBuilder { + pub fn tls_cert_path>(&mut self, v: T) -> &mut Self { + self.tls_cert_path = Some(v.as_ref().to_path_buf()); + self + } + pub fn tls_cert_key_path>(&mut self, v: T) -> &mut Self { + self.tls_cert_key_path = Some(v.as_ref().to_path_buf()); + self + } + pub fn client_ca_cert_path>(&mut self, v: Option) -> &mut Self { + self.client_ca_cert_path = Some(v.map(|p| p.as_ref().to_path_buf())); + self + } +} + +/* ------------------------------------------------ */ +#[async_trait] +impl CryptoSource for CryptoFileSource { + type Error = RpxyCertError; + /// read crypto materials from source + async fn read(&self) -> Result { + read_certs_and_keys( + &self.tls_cert_path, + &self.tls_cert_key_path, + self.client_ca_cert_path.as_ref(), + ) + } + /// Returns true when mutual tls is enabled + fn is_mutual_tls(&self) -> bool { + self.client_ca_cert_path.is_some() + } +} + +/* ------------------------------------------------ */ +/// Read certificates and private keys from file +fn read_certs_and_keys( + cert_path: &PathBuf, + cert_key_path: &PathBuf, + client_ca_cert_path: Option<&PathBuf>, +) -> Result { + debug!("Read TLS server certificates and private key"); + + // certificates + let raw_certs = { + let mut reader = BufReader::new(File::open(cert_path).map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the certificates [{}]: {e}", cert_path.display()), + ) + })?); + rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the certificates"))? + }; + + // private keys + let raw_cert_keys = { + let encoded_keys = { + let mut encoded_keys = vec![]; + File::open(cert_key_path) + .map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the certificate keys [{}]: {e}", cert_key_path.display()), + ) + })? + .read_to_end(&mut encoded_keys)?; + encoded_keys + }; + let mut reader = Cursor::new(encoded_keys); + let pkcs8_keys = rustls_pemfile::pkcs8_private_keys(&mut reader) + .map(|v| v.map(rustls::pki_types::PrivateKeyDer::Pkcs8)) + .collect::, _>>() + .map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Unable to parse the certificates private keys (PKCS8)", + ) + })?; + reader.set_position(0); + let mut rsa_keys = rustls_pemfile::rsa_private_keys(&mut reader) + .map(|v| v.map(rustls::pki_types::PrivateKeyDer::Pkcs1)) + .collect::, _>>()?; + let mut keys = pkcs8_keys; + keys.append(&mut rsa_keys); + if keys.is_empty() { + return Err(RpxyCertError::IoError(io::Error::new( + io::ErrorKind::InvalidInput, + "No private keys found - Make sure that they are in PKCS#8/PEM format", + ))); + } + keys + }; + + // client ca certificates + let client_ca_certs = if let Some(path) = client_ca_cert_path { + debug!("Read CA certificates for client authentication"); + // Reads client certificate and returns client + let certs = { + let mut reader = BufReader::new(File::open(path).map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the client certificates [{}]: {e}", path.display()), + ) + })?); + rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the client certificates"))? + }; + Some(certs) + } else { + None + }; + + Ok(SingleServerCrypto::new( + &raw_certs, + &Arc::new(raw_cert_keys), + &client_ca_certs, + )) +} From 377096c14e5dde27d311b1a133eac064ddcd7e93 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 27 May 2024 11:55:22 +0900 Subject: [PATCH 03/16] wip: started to implement reloader service --- rpxy-certs/src/certs.rs | 62 +++++++++++++++++++++++++++++++++++++-- rpxy-certs/src/lib.rs | 59 ++----------------------------------- rpxy-certs/src/service.rs | 15 ++++++++++ rpxy-certs/src/source.rs | 10 +++---- 4 files changed, 82 insertions(+), 64 deletions(-) diff --git a/rpxy-certs/src/certs.rs b/rpxy-certs/src/certs.rs index 7df964e2..92578b39 100644 --- a/rpxy-certs/src/certs.rs +++ b/rpxy-certs/src/certs.rs @@ -9,19 +9,21 @@ use x509_parser::prelude::*; type Certificate = rustls::pki_types::CertificateDer<'static>; /// Raw private key in rustls format type PrivateKey = pki_types::PrivateKeyDer<'static>; +/// Subject Key ID in bytes +type SubjectKeyIdentifier = Vec; /// Client CA trust anchors subject to the subject key identifier -type TrustAnchors = HashMap, pki_types::TrustAnchor<'static>>; +type TrustAnchors = HashMap>; /* ------------------------------------------------ */ /// Raw certificates and private keys loaded from files for a single server name #[derive(Debug, PartialEq, Eq, Clone)] -pub struct SingleServerCrypto { +pub struct SingleServerCertsKeys { certs: Vec, cert_keys: Arc>, client_ca_certs: Option>, } -impl SingleServerCrypto { +impl SingleServerCertsKeys { /// Create a new instance of SingleServerCrypto pub fn new(certs: &[Certificate], cert_keys: &Arc>, client_ca_certs: &Option>) -> Self { Self { @@ -100,3 +102,57 @@ impl SingleServerCrypto { Ok(trust_anchors) } } + +/* ------------------------------------------------ */ +#[cfg(test)] +mod tests { + use super::super::*; + + #[tokio::test] + async fn read_server_crt_key_files() { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + + let crypto_elem = crypto_elem.unwrap(); + let certificed_key = crypto_elem.rustls_certified_key(); + assert!(certificed_key.is_ok()); + } + + #[tokio::test] + async fn read_server_crt_key_files_with_client_ca_crt() { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt"); + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .client_ca_cert_path(client_ca_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + + let crypto_elem = crypto_elem.unwrap(); + assert!(crypto_elem.is_mutual_tls()); + + let certificed_key = crypto_elem.rustls_certified_key(); + assert!(certificed_key.is_ok()); + + let trust_anchors = crypto_elem.rustls_trust_anchors(); + assert!(trust_anchors.is_ok()); + + let trust_anchors = trust_anchors.unwrap(); + assert_eq!(trust_anchors.len(), 1); + } +} diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index 18b7da52..75b107c3 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -4,65 +4,12 @@ mod service; mod source; #[allow(unused_imports)] -pub(crate) mod log { +mod log { pub(crate) use tracing::{debug, error, info, warn}; } pub use crate::{ - certs::SingleServerCrypto, + certs::SingleServerCertsKeys, + service::{ServerCrypto, ServerNameBytes, ServerNameCryptoMap}, source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, }; - -/* ------------------------------------------------ */ -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn read_server_crt_key_files() { - let tls_cert_path = "../example-certs/server.crt"; - let tls_cert_key_path = "../example-certs/server.key"; - let crypto_file_source = CryptoFileSourceBuilder::default() - .tls_cert_key_path(tls_cert_key_path) - .tls_cert_path(tls_cert_path) - .build(); - assert!(crypto_file_source.is_ok()); - - let crypto_file_source = crypto_file_source.unwrap(); - let crypto_elem = crypto_file_source.read().await; - assert!(crypto_elem.is_ok()); - - let crypto_elem = crypto_elem.unwrap(); - let certificed_key = crypto_elem.rustls_certified_key(); - assert!(certificed_key.is_ok()); - } - - #[tokio::test] - async fn read_server_crt_key_files_with_client_ca_crt() { - let tls_cert_path = "../example-certs/server.crt"; - let tls_cert_key_path = "../example-certs/server.key"; - let client_ca_cert_path = Some("../example-certs/client.ca.crt"); - let crypto_file_source = CryptoFileSourceBuilder::default() - .tls_cert_key_path(tls_cert_key_path) - .tls_cert_path(tls_cert_path) - .client_ca_cert_path(client_ca_cert_path) - .build(); - assert!(crypto_file_source.is_ok()); - - let crypto_file_source = crypto_file_source.unwrap(); - let crypto_elem = crypto_file_source.read().await; - assert!(crypto_elem.is_ok()); - - let crypto_elem = crypto_elem.unwrap(); - assert!(crypto_elem.is_mutual_tls()); - - let certificed_key = crypto_elem.rustls_certified_key(); - assert!(certificed_key.is_ok()); - - let trust_anchors = crypto_elem.rustls_trust_anchors(); - assert!(trust_anchors.is_ok()); - - let trust_anchors = trust_anchors.unwrap(); - assert_eq!(trust_anchors.len(), 1); - } -} diff --git a/rpxy-certs/src/service.rs b/rpxy-certs/src/service.rs index e69de29b..31f24a1f 100644 --- a/rpxy-certs/src/service.rs +++ b/rpxy-certs/src/service.rs @@ -0,0 +1,15 @@ +use rustc_hash::FxHashMap as HashMap; +use rustls::ServerConfig; +use std::sync::Arc; + +/// ServerName in bytes type +pub type ServerNameBytes = Vec; +/// ServerName (SNI) to ServerConfig map type +pub type ServerNameCryptoMap = HashMap>; +/// ServerName (SNI) to ServerConfig map +pub struct ServerCrypto { + // For Quic/HTTP3, only servers with no client authentication + pub inner_global_no_client_auth: Arc, + // // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers + pub inner_local_map: Arc, +} diff --git a/rpxy-certs/src/source.rs b/rpxy-certs/src/source.rs index e4dba818..4eae1048 100644 --- a/rpxy-certs/src/source.rs +++ b/rpxy-certs/src/source.rs @@ -1,4 +1,4 @@ -use crate::{certs::SingleServerCrypto, error::*, log::*}; +use crate::{certs::SingleServerCertsKeys, error::*, log::*}; use async_trait::async_trait; use derive_builder::Builder; use std::{ @@ -15,7 +15,7 @@ pub trait CryptoSource { type Error; /// read crypto materials from source - async fn read(&self) -> Result; + async fn read(&self) -> Result; /// Returns true when mutual tls is enabled fn is_mutual_tls(&self) -> bool; @@ -58,7 +58,7 @@ impl CryptoFileSourceBuilder { impl CryptoSource for CryptoFileSource { type Error = RpxyCertError; /// read crypto materials from source - async fn read(&self) -> Result { + async fn read(&self) -> Result { read_certs_and_keys( &self.tls_cert_path, &self.tls_cert_key_path, @@ -77,7 +77,7 @@ fn read_certs_and_keys( cert_path: &PathBuf, cert_key_path: &PathBuf, client_ca_cert_path: Option<&PathBuf>, -) -> Result { +) -> Result { debug!("Read TLS server certificates and private key"); // certificates @@ -152,7 +152,7 @@ fn read_certs_and_keys( None }; - Ok(SingleServerCrypto::new( + Ok(SingleServerCertsKeys::new( &raw_certs, &Arc::new(raw_cert_keys), &client_ca_certs, From e18fafe4e63f0d1d2267e3acb6ef5fc11e6c99da Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 03:19:59 +0900 Subject: [PATCH 04/16] wip: implemented reloader service with trait object for future support of acme --- rpxy-certs/Cargo.toml | 2 +- .../src/{source.rs => crypto_source.rs} | 0 rpxy-certs/src/lib.rs | 9 +- rpxy-certs/src/reloader_service.rs | 83 +++++++++++++++++++ .../src/{service.rs => server_crypto.rs} | 12 ++- 5 files changed, 100 insertions(+), 6 deletions(-) rename rpxy-certs/src/{source.rs => crypto_source.rs} (100%) create mode 100644 rpxy-certs/src/reloader_service.rs rename rpxy-certs/src/{service.rs => server_crypto.rs} (52%) diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index cd5bdfb5..26febe49 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -18,7 +18,7 @@ tracing = { version = "0.1.40" } # anyhow = "1.0.86" derive_builder = { version = "0.20.0" } thiserror = { version = "1.0.61" } -# hot_reload = {version = "0.1.5"} +hot_reload = { version = "0.1.5" } async-trait = { version = "0.1.80" } # tokio-rustls = { version = "0.26.0", features = ["early-data"] } rustls = { version = "0.23.8", default-features = false, features = [ diff --git a/rpxy-certs/src/source.rs b/rpxy-certs/src/crypto_source.rs similarity index 100% rename from rpxy-certs/src/source.rs rename to rpxy-certs/src/crypto_source.rs diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index 75b107c3..ebb47891 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -1,7 +1,8 @@ mod certs; +mod crypto_source; mod error; -mod service; -mod source; +mod reloader_service; +mod server_crypto; #[allow(unused_imports)] mod log { @@ -10,6 +11,6 @@ mod log { pub use crate::{ certs::SingleServerCertsKeys, - service::{ServerCrypto, ServerNameBytes, ServerNameCryptoMap}, - source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, + crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, + server_crypto::{ServerCrypto, ServerNameBytes, ServerNameCryptoMap}, }; diff --git a/rpxy-certs/src/reloader_service.rs b/rpxy-certs/src/reloader_service.rs new file mode 100644 index 00000000..06f5ad69 --- /dev/null +++ b/rpxy-certs/src/reloader_service.rs @@ -0,0 +1,83 @@ +use crate::{ + crypto_source::CryptoSource, + error::*, + log::*, + server_crypto::{ServerCryptoBase, ServerNameBytes}, +}; +use async_trait::async_trait; +use hot_reload::{Reload, ReloaderError}; +use rustc_hash::FxHashMap as HashMap; +use std::sync::Arc; + +/* ------------------------------------------------ */ +/// Boxed CryptoSource trait object with Send and Sync +/// TODO: support for not only `CryptoFileSource` but also other type of sources +type DynCryptoSource = dyn CryptoSource + Send + Sync + 'static; + +/// Reloader service for certificates and keys for TLS +pub struct CryptoReloader { + inner: HashMap>>, +} + +impl Extend<(ServerNameBytes, T)> for CryptoReloader +where + T: CryptoSource + Send + Sync + 'static, +{ + fn extend>(&mut self, iter: I) { + let iter = iter + .into_iter() + .map(|(k, v)| (k, Arc::new(Box::new(v) as Box))); + self.inner.extend(iter); + } +} + +#[async_trait] +impl Reload for CryptoReloader { + type Source = HashMap>>; + + async fn new(source: &Self::Source) -> Result> { + let mut inner = HashMap::default(); + inner.extend(source.clone()); + Ok(Self { inner }) + } + + async fn reload(&self) -> Result, ReloaderError> { + let mut server_crypto_base = ServerCryptoBase::default(); + + for (server_name_bytes, crypto_source) in self.inner.iter() { + let certs_keys = crypto_source.read().await.map_err(|e| { + error!("Failed to reload cert, key or ca cert: {e}"); + ReloaderError::::Reload("Failed to reload cert, key or ca cert") + })?; + server_crypto_base.inner.insert(server_name_bytes.clone(), certs_keys); + } + + Ok(Some(server_crypto_base)) + } +} +/* ------------------------------------------------ */ + +#[cfg(test)] +mod tests { + use super::*; + use crate::crypto_source::CryptoFileSourceBuilder; + + #[tokio::test] + async fn test_crypto_reloader() { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt"); + + let mut crypto_reloader = CryptoReloader::new(&HashMap::default()).await.unwrap(); + let crypto_source = CryptoFileSourceBuilder::default() + .tls_cert_path(tls_cert_path) + .tls_cert_key_path(tls_cert_key_path) + .client_ca_cert_path(client_ca_cert_path) + .build() + .unwrap(); + crypto_reloader.extend(vec![(b"localhost".to_vec(), crypto_source)]); + + let server_crypto_base = crypto_reloader.reload().await.unwrap().unwrap(); + assert_eq!(server_crypto_base.inner.len(), 1); + } +} diff --git a/rpxy-certs/src/service.rs b/rpxy-certs/src/server_crypto.rs similarity index 52% rename from rpxy-certs/src/service.rs rename to rpxy-certs/src/server_crypto.rs index 31f24a1f..44943972 100644 --- a/rpxy-certs/src/service.rs +++ b/rpxy-certs/src/server_crypto.rs @@ -1,8 +1,10 @@ +use crate::SingleServerCertsKeys; use rustc_hash::FxHashMap as HashMap; use rustls::ServerConfig; use std::sync::Arc; -/// ServerName in bytes type +/* ------------------------------------------------ */ +/// ServerName in bytes type (TODO: this may be changed to define `common` layer defining types of names. or should be independent?) pub type ServerNameBytes = Vec; /// ServerName (SNI) to ServerConfig map type pub type ServerNameCryptoMap = HashMap>; @@ -13,3 +15,11 @@ pub struct ServerCrypto { // // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers pub inner_local_map: Arc, } + +/* ------------------------------------------------ */ +/// Reloader target for the certificate reloader service +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct ServerCryptoBase { + /// Map of server name to certs and keys + pub(super) inner: HashMap, +} From b168d1cdc5f4d2194c3b3c84f50875e9f6b36c00 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 03:36:05 +0900 Subject: [PATCH 05/16] wip builder stub --- rpxy-certs/src/error.rs | 3 +++ rpxy-certs/src/lib.rs | 30 +++++++++++++++++++++++++++++- rpxy-certs/src/reloader_service.rs | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs index b5135a8c..34bd2276 100644 --- a/rpxy-certs/src/error.rs +++ b/rpxy-certs/src/error.rs @@ -12,4 +12,7 @@ pub enum RpxyCertError { /// Error when parsing client CA certificates: No client certificate found #[error("No client certificate found")] NoClientCert, + /// Error for hot reload certificate reloader + #[error("Certificate reload error: {0}")] + CertificateReloadError(#[from] hot_reload::ReloaderError), } diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index ebb47891..2c616a22 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -8,9 +8,37 @@ mod server_crypto; mod log { pub(crate) use tracing::{debug, error, info, warn}; } - +/* ------------------------------------------------ */ pub use crate::{ certs::SingleServerCertsKeys, crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, server_crypto::{ServerCrypto, ServerNameBytes, ServerNameCryptoMap}, }; + +use crate::{error::*, reloader_service::CryptoReloader, server_crypto::ServerCryptoBase}; +use hot_reload::{ReloaderReceiver, ReloaderService}; + +/* ------------------------------------------------ */ +/// Constants TODO: define from outside +const CERTS_WATCH_DELAY_SECS: u32 = 60; +const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true; + +/* ------------------------------------------------ */ +/// Result type inner of certificate reloader service +type ReloaderServiceResultInner = ( + ReloaderService, + ReloaderReceiver, +); +/// Build certificate reloader service +pub async fn build_cert_reloader() -> Result +// where +// T: CryptoSource + Clone + Send + Sync + 'static, +{ + // TODO: fix later + let source = rustc_hash::FxHashMap::default(); + + let (cert_reloader_service, cert_reloader_rx) = + ReloaderService::::new(&source, CERTS_WATCH_DELAY_SECS, !LOAD_CERTS_ONLY_WHEN_UPDATED) + .await?; + Ok((cert_reloader_service, cert_reloader_rx)) +} diff --git a/rpxy-certs/src/reloader_service.rs b/rpxy-certs/src/reloader_service.rs index 06f5ad69..8427f3a2 100644 --- a/rpxy-certs/src/reloader_service.rs +++ b/rpxy-certs/src/reloader_service.rs @@ -14,6 +14,7 @@ use std::sync::Arc; /// TODO: support for not only `CryptoFileSource` but also other type of sources type DynCryptoSource = dyn CryptoSource + Send + Sync + 'static; +#[derive(Clone)] /// Reloader service for certificates and keys for TLS pub struct CryptoReloader { inner: HashMap>>, From 2935cbd59b952e5131262f61f7d9f27482cbf99e Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 13:41:55 +0900 Subject: [PATCH 06/16] wip: implemented try_into from the disseminated cert data to rustls server config --- rpxy-certs/Cargo.toml | 4 +- rpxy-certs/src/certs.rs | 5 +- rpxy-certs/src/crypto_source.rs | 2 +- rpxy-certs/src/error.rs | 3 + rpxy-certs/src/lib.rs | 8 +- rpxy-certs/src/server_crypto.rs | 134 ++++++++++++++++++++++++++++++-- 6 files changed, 140 insertions(+), 16 deletions(-) diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index 26febe49..732901d9 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -11,16 +11,16 @@ edition.workspace = true publish.workspace = true [features] +default = ["http3"] +http3 = [] [dependencies] rustc-hash = { version = "1.1.0" } tracing = { version = "0.1.40" } -# anyhow = "1.0.86" derive_builder = { version = "0.20.0" } thiserror = { version = "1.0.61" } hot_reload = { version = "0.1.5" } async-trait = { version = "0.1.80" } -# tokio-rustls = { version = "0.26.0", features = ["early-data"] } rustls = { version = "0.23.8", default-features = false, features = [ "aws_lc_rs", ] } diff --git a/rpxy-certs/src/certs.rs b/rpxy-certs/src/certs.rs index 92578b39..8c8a2fb3 100644 --- a/rpxy-certs/src/certs.rs +++ b/rpxy-certs/src/certs.rs @@ -74,8 +74,7 @@ impl SingleServerCertsKeys { /* ------------------------------------------------ */ /// Parse the client CA certificates and return a hashmap of pairs of a subject key identifier (key) and a trust anchor (value) - pub fn rustls_trust_anchors(&self) -> Result { - //-> Result<(Vec, HashSet>), anyhow::Error> { + pub fn rustls_client_certs_trust_anchors(&self) -> Result { let Some(certs) = self.client_ca_certs.as_ref() else { return Err(RpxyCertError::NoClientCert); }; @@ -149,7 +148,7 @@ mod tests { let certificed_key = crypto_elem.rustls_certified_key(); assert!(certificed_key.is_ok()); - let trust_anchors = crypto_elem.rustls_trust_anchors(); + let trust_anchors = crypto_elem.rustls_client_certs_trust_anchors(); assert!(trust_anchors.is_ok()); let trust_anchors = trust_anchors.unwrap(); diff --git a/rpxy-certs/src/crypto_source.rs b/rpxy-certs/src/crypto_source.rs index 4eae1048..c204fbb7 100644 --- a/rpxy-certs/src/crypto_source.rs +++ b/rpxy-certs/src/crypto_source.rs @@ -23,7 +23,7 @@ pub trait CryptoSource { /* ------------------------------------------------ */ #[derive(Builder, Debug, Clone)] -/// Crypto-related file reader implementing `CryptoSource`` trait +/// Crypto-related file reader implementing `CryptoSource` trait pub struct CryptoFileSource { #[builder(setter(custom))] /// Always exist diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs index 34bd2276..26e16b75 100644 --- a/rpxy-certs/src/error.rs +++ b/rpxy-certs/src/error.rs @@ -15,4 +15,7 @@ pub enum RpxyCertError { /// Error for hot reload certificate reloader #[error("Certificate reload error: {0}")] CertificateReloadError(#[from] hot_reload::ReloaderError), + /// Error when converting server name bytes to string + #[error("Failed to convert server name bytes to string: {0}")] + ServerNameBytesToString(#[from] std::string::FromUtf8Error), } diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index 2c616a22..b8abda8d 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -9,15 +9,15 @@ mod log { pub(crate) use tracing::{debug, error, info, warn}; } /* ------------------------------------------------ */ +use crate::{error::*, reloader_service::CryptoReloader}; +use hot_reload::{ReloaderReceiver, ReloaderService}; + pub use crate::{ certs::SingleServerCertsKeys, crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, - server_crypto::{ServerCrypto, ServerNameBytes, ServerNameCryptoMap}, + server_crypto::{ServerCrypto, ServerCryptoBase}, }; -use crate::{error::*, reloader_service::CryptoReloader, server_crypto::ServerCryptoBase}; -use hot_reload::{ReloaderReceiver, ReloaderService}; - /* ------------------------------------------------ */ /// Constants TODO: define from outside const CERTS_WATCH_DELAY_SECS: u32 = 60; diff --git a/rpxy-certs/src/server_crypto.rs b/rpxy-certs/src/server_crypto.rs index 44943972..53fbcc3f 100644 --- a/rpxy-certs/src/server_crypto.rs +++ b/rpxy-certs/src/server_crypto.rs @@ -1,19 +1,29 @@ -use crate::SingleServerCertsKeys; +use crate::{certs::SingleServerCertsKeys, error::*, log::*}; use rustc_hash::FxHashMap as HashMap; -use rustls::ServerConfig; +use rustls::{ + server::{ResolvesServerCertUsingSni, WebPkiClientVerifier}, + RootCertStore, ServerConfig, +}; use std::sync::Arc; /* ------------------------------------------------ */ /// ServerName in bytes type (TODO: this may be changed to define `common` layer defining types of names. or should be independent?) pub type ServerNameBytes = Vec; +/// Convert ServerName in bytes to string +fn server_name_bytes_to_string(server_name_bytes: &ServerNameBytes) -> Result { + let server_name = String::from_utf8(server_name_bytes.to_ascii_lowercase())?; + Ok(server_name) +} + /// ServerName (SNI) to ServerConfig map type pub type ServerNameCryptoMap = HashMap>; + /// ServerName (SNI) to ServerConfig map pub struct ServerCrypto { - // For Quic/HTTP3, only servers with no client authentication - pub inner_global_no_client_auth: Arc, - // // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers - pub inner_local_map: Arc, + // For Quic/HTTP3, only servers with no client authentication, aggregated server config + pub aggregated_config_no_client_auth: Arc, + // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers + pub individual_config_map: Arc, } /* ------------------------------------------------ */ @@ -23,3 +33,115 @@ pub struct ServerCryptoBase { /// Map of server name to certs and keys pub(super) inner: HashMap, } + +impl TryInto> for &ServerCryptoBase { + type Error = RpxyCertError; + + fn try_into(self) -> Result, Self::Error> { + let aggregated = self.build_aggrated_server_crypto()?; + let individual = self.build_individual_server_crypto_map()?; + + Ok(Arc::new(ServerCrypto { + aggregated_config_no_client_auth: Arc::new(aggregated), + individual_config_map: Arc::new(individual), + })) + } +} + +impl ServerCryptoBase { + /// Build individual server crypto inner object + fn build_individual_server_crypto_map(&self) -> Result { + let mut server_crypto_map: ServerNameCryptoMap = HashMap::default(); + + for (server_name_bytes, certs_keys) in self.inner.iter() { + let server_name = server_name_bytes_to_string(server_name_bytes)?; + + // Parse server certificates and private keys + let Ok(certified_key) = certs_keys.rustls_certified_key() else { + warn!("Failed to add certificate for {server_name}"); + continue; + }; + + let mut resolver_local = ResolvesServerCertUsingSni::new(); + if let Err(e) = resolver_local.add(&server_name, certified_key) { + error!("{server_name}: Failed to read some certificates and keys {e}"); + }; + + // With no client authentication case + if !certs_keys.is_mutual_tls() { + let mut server_crypto_local = ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver_local)); + #[cfg(feature = "http3")] + { + server_crypto_local.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]; + } + #[cfg(not(feature = "http3"))] + { + server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + } + server_crypto_map.insert(server_name_bytes.clone(), Arc::new(server_crypto_local)); + continue; + } + + // With client authentication case, enable only http2 and http1.1 + let mut client_ca_roots_local = RootCertStore::empty(); + let Ok(trust_anchors) = certs_keys.rustls_client_certs_trust_anchors() else { + warn!("Failed to add client CA certificate for {server_name}"); + continue; + }; + let trust_anchors_without_skid = trust_anchors.values().map(|ta| ta.to_owned()); + client_ca_roots_local.extend(trust_anchors_without_skid); + + let Ok(client_cert_verifier) = WebPkiClientVerifier::builder(Arc::new(client_ca_roots_local)).build() else { + warn!("Failed to build client CA certificate verifier for {server_name}"); + continue; + }; + let mut server_crypto_local = ServerConfig::builder() + .with_client_cert_verifier(client_cert_verifier) + .with_cert_resolver(Arc::new(resolver_local)); + server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + server_crypto_map.insert(server_name_bytes.clone(), Arc::new(server_crypto_local)); + } + + Ok(server_crypto_map) + } + + /* ------------------------------------------------ */ + /// Build aggregated server crypto inner object for no client auth server especially for http3 + fn build_aggrated_server_crypto(&self) -> Result { + let mut resolver_global = ResolvesServerCertUsingSni::new(); + + for (server_name_bytes, certs_keys) in self.inner.iter() { + let server_name = server_name_bytes_to_string(server_name_bytes)?; + + // Parse server certificates and private keys + let Ok(certified_key) = certs_keys.rustls_certified_key() else { + warn!("Failed to add certificate for {server_name}"); + continue; + }; + // Add server certificates and private keys to resolver only if client CA certs are not present + if !certs_keys.is_mutual_tls() { + // aggregated server config for no client auth server for http3 + if let Err(e) = resolver_global.add(&server_name, certified_key) { + error!("{server_name}: Failed to read some certificates and keys {e}"); + }; + } + } + + let mut server_crypto_global = ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver_global)); + + #[cfg(feature = "http3")] + { + server_crypto_global.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]; + } + #[cfg(not(feature = "http3"))] + { + server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + } + + Ok(server_crypto_global) + } +} From e5bfc2c4eac74e0ed857bc37cca51c1e7aa6d602 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 14:41:19 +0900 Subject: [PATCH 07/16] wip: refactor --- rpxy-certs/src/lib.rs | 45 ++++++++++++++++++++---------- rpxy-certs/src/reloader_service.rs | 2 +- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index b8abda8d..106dbdd4 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -6,12 +6,18 @@ mod server_crypto; #[allow(unused_imports)] mod log { - pub(crate) use tracing::{debug, error, info, warn}; + pub(super) use tracing::{debug, error, info, warn}; } -/* ------------------------------------------------ */ -use crate::{error::*, reloader_service::CryptoReloader}; + +use crate::{ + error::*, + reloader_service::{CryptoReloader, DynCryptoSource}, +}; use hot_reload::{ReloaderReceiver, ReloaderService}; +use rustc_hash::FxHashMap as HashMap; +use std::sync::Arc; +/* ------------------------------------------------ */ pub use crate::{ certs::SingleServerCertsKeys, crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, @@ -19,26 +25,37 @@ pub use crate::{ }; /* ------------------------------------------------ */ -/// Constants TODO: define from outside -const CERTS_WATCH_DELAY_SECS: u32 = 60; +// Constants +/// Default delay in seconds to watch certificates +const DEFAULT_CERTS_WATCH_DELAY_SECS: u32 = 60; +/// Load certificates only when updated const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true; -/* ------------------------------------------------ */ /// Result type inner of certificate reloader service type ReloaderServiceResultInner = ( ReloaderService, ReloaderReceiver, ); -/// Build certificate reloader service -pub async fn build_cert_reloader() -> Result -// where -// T: CryptoSource + Clone + Send + Sync + 'static, +/// Build certificate reloader service, which accepts a map of server names to `CryptoSource` instances +pub async fn build_cert_reloader( + crypto_source_map: &HashMap, + certs_watch_period: Option, +) -> Result +where + T: CryptoSource + Send + Sync + Clone + 'static, { - // TODO: fix later - let source = rustc_hash::FxHashMap::default(); + let source = crypto_source_map + .iter() + .map(|(k, v)| { + let server_name_bytes = k.as_bytes().to_vec().to_ascii_lowercase(); + let dyn_crypto_source = Arc::new(Box::new(v.clone()) as Box); + (server_name_bytes, dyn_crypto_source) + }) + .collect::>(); + + let certs_watch_period = certs_watch_period.unwrap_or(DEFAULT_CERTS_WATCH_DELAY_SECS); let (cert_reloader_service, cert_reloader_rx) = - ReloaderService::::new(&source, CERTS_WATCH_DELAY_SECS, !LOAD_CERTS_ONLY_WHEN_UPDATED) - .await?; + ReloaderService::::new(&source, certs_watch_period, !LOAD_CERTS_ONLY_WHEN_UPDATED).await?; Ok((cert_reloader_service, cert_reloader_rx)) } diff --git a/rpxy-certs/src/reloader_service.rs b/rpxy-certs/src/reloader_service.rs index 8427f3a2..c3d1fcd7 100644 --- a/rpxy-certs/src/reloader_service.rs +++ b/rpxy-certs/src/reloader_service.rs @@ -12,7 +12,7 @@ use std::sync::Arc; /* ------------------------------------------------ */ /// Boxed CryptoSource trait object with Send and Sync /// TODO: support for not only `CryptoFileSource` but also other type of sources -type DynCryptoSource = dyn CryptoSource + Send + Sync + 'static; +pub(super) type DynCryptoSource = dyn CryptoSource + Send + Sync + 'static; #[derive(Clone)] /// Reloader service for certificates and keys for TLS From e25c6fa81fa1ac796e192e25d54300cbf16110c5 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 17:52:00 +0900 Subject: [PATCH 08/16] wip: integrate certmanager to rpxy-bin along with existing old rustls --- rpxy-bin/Cargo.toml | 8 ++- rpxy-bin/src/config/mod.rs | 2 +- rpxy-bin/src/config/parse.rs | 43 ++++++++++++---- rpxy-bin/src/main.rs | 90 +++++++++++++++++++-------------- rpxy-certs/Cargo.toml | 4 +- rpxy-certs/src/error.rs | 6 +++ rpxy-certs/src/lib.rs | 11 ++-- rpxy-certs/src/server_crypto.rs | 67 ++++++++++++++++++++++-- 8 files changed, 170 insertions(+), 61 deletions(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 42427415..d6dd8429 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -26,6 +26,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [ "sticky-cookie", ] } +mimalloc = { version = "*", default-features = false } anyhow = "1.0.86" rustc-hash = "1.1.0" serde = { version = "1.0.202", default-features = false, features = ["derive"] } @@ -39,7 +40,7 @@ tokio = { version = "1.37.0", default-features = false, features = [ ] } async-trait = "0.1.80" rustls-pemfile = "1.0.4" -mimalloc = { version = "*", default-features = false } + # config clap = { version = "4.5.4", features = ["std", "cargo", "wrap_help"] } @@ -50,5 +51,10 @@ hot_reload = "0.1.5" tracing = { version = "0.1.40" } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +################################ +# cert management +rpxy-certs = { path = "../rpxy-certs/", default-features = false, features = [ + "http3", +] } [dev-dependencies] diff --git a/rpxy-bin/src/config/mod.rs b/rpxy-bin/src/config/mod.rs index 09ec2b98..adc4ff28 100644 --- a/rpxy-bin/src/config/mod.rs +++ b/rpxy-bin/src/config/mod.rs @@ -4,6 +4,6 @@ mod toml; pub use { self::toml::ConfigToml, - parse::{build_settings, parse_opts}, + parse::{build_cert_manager, build_settings, parse_opts}, service::ConfigTomlReloader, }; diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 15ff2407..049f5ee8 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -4,7 +4,10 @@ use crate::{ error::{anyhow, ensure}, }; use clap::{Arg, ArgAction}; +use hot_reload::{ReloaderReceiver, ReloaderService}; +use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase}; use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; +use rustc_hash::FxHashMap as HashMap; /// Parsed options pub struct Opts { @@ -37,20 +40,13 @@ pub fn parse_opts() -> Result { let config_file_path = matches.get_one::("config_file").unwrap().to_owned(); let watch = matches.get_one::("watch").unwrap().to_owned(); - Ok(Opts { - config_file_path, - watch, - }) + Ok(Opts { config_file_path, watch }) } -pub fn build_settings( - config: &ConfigToml, -) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { - /////////////////////////////////// +pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { // build proxy config let proxy_config: ProxyConfig = config.try_into()?; - /////////////////////////////////// // backend_apps let apps = config.apps.clone().ok_or(anyhow!("Missing application spec"))?; @@ -95,3 +91,32 @@ pub fn build_settings( Ok((proxy_config, app_config_list)) } + +/* ----------------------- */ +/// Build cert map +pub async fn build_cert_manager( + config: &ConfigToml, +) -> Result< + ( + ReloaderService, + ReloaderReceiver, + ), + anyhow::Error, +> { + let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?; + let mut crypto_source_map = HashMap::default(); + for app in apps.0.values() { + if let Some(tls) = app.tls.as_ref() { + ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); + let server_name = app.server_name.as_ref().ok_or(anyhow!("No server name"))?; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_path(tls.tls_cert_path.as_ref().unwrap()) + .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap()) + .client_ca_cert_path(tls.client_ca_cert_path.as_deref()) + .build()?; + crypto_source_map.insert(server_name.to_owned(), crypto_file_source); + } + } + let res = build_cert_reloader(&crypto_source_map, None).await?; + Ok(res) +} diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 9aeb9713..114e6db1 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -8,8 +8,9 @@ mod error; mod log; use crate::{ - config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader}, + config::{build_cert_manager, build_settings, parse_opts, ConfigToml, ConfigTomlReloader}, constants::CONFIG_WATCH_DELAY_SECS, + error::*, log::*, }; use hot_reload::{ReloaderReceiver, ReloaderService}; @@ -36,13 +37,10 @@ fn main() { std::process::exit(1); } } else { - let (config_service, config_rx) = ReloaderService::::new( - &parsed_opts.config_file_path, - CONFIG_WATCH_DELAY_SECS, - false, - ) - .await - .unwrap(); + let (config_service, config_rx) = + ReloaderService::::new(&parsed_opts.config_file_path, CONFIG_WATCH_DELAY_SECS, false) + .await + .unwrap(); tokio::select! { Err(e) = config_service.start() => { @@ -53,6 +51,9 @@ fn main() { error!("rpxy service existed: {e}"); std::process::exit(1); } + else => { + std::process::exit(0); + } } } }); @@ -63,23 +64,22 @@ async fn rpxy_service_without_watcher( runtime_handle: tokio::runtime::Handle, ) -> Result<(), anyhow::Error> { info!("Start rpxy service"); - let config_toml = match ConfigToml::new(config_file_path) { - Ok(v) => v, - Err(e) => { - error!("Invalid toml file: {e}"); - std::process::exit(1); + let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?; + let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; + let (cert_service, cert_rx) = build_cert_manager(&config_toml) + .await + .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; + + tokio::select! { + rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => { + error!("rpxy entrypoint exited"); + rpxy_res.map_err(|e| anyhow!(e)) } - }; - let (proxy_conf, app_conf) = match build_settings(&config_toml) { - Ok(v) => v, - Err(e) => { - error!("Invalid configuration: {e}"); - return Err(anyhow::anyhow!(e)); + cert_res = cert_service.start() => { + error!("cert reloader service exited"); + cert_res.map_err(|e| anyhow!(e)) } - }; - entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) - .await - .map_err(|e| anyhow::anyhow!(e)) + } } async fn rpxy_service_with_watcher( @@ -89,14 +89,15 @@ async fn rpxy_service_with_watcher( info!("Start rpxy service with dynamic config reloader"); // Initial loading config_rx.changed().await?; - let config_toml = config_rx.borrow().clone().unwrap(); - let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) { - Ok(v) => v, - Err(e) => { - error!("Invalid configuration: {e}"); - return Err(anyhow::anyhow!(e)); - } - }; + let config_toml = config_rx + .borrow() + .clone() + .ok_or(anyhow!("Something wrong in config reloader receiver"))?; + let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; + + let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml) + .await + .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; // Notifier for proxy service termination let term_notify = std::sync::Arc::new(tokio::sync::Notify::new()); @@ -104,16 +105,15 @@ async fn rpxy_service_with_watcher( // Continuous monitoring loop { tokio::select! { - _ = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => { + rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => { error!("rpxy entrypoint exited"); - break; + return rpxy_res.map_err(|e| anyhow!(e)); } _ = config_rx.changed() => { - if config_rx.borrow().is_none() { + let Some(config_toml) = config_rx.borrow().clone() else { error!("Something wrong in config reloader receiver"); - break; - } - let config_toml = config_rx.borrow().clone().unwrap(); + return Err(anyhow!("Something wrong in config reloader receiver")); + }; match build_settings(&config_toml) { Ok((p, a)) => { (proxy_conf, app_conf) = (p, a) @@ -123,13 +123,27 @@ async fn rpxy_service_with_watcher( continue; } }; + match build_cert_manager(&config_toml).await { + Ok((c, r)) => { + (cert_service, cert_rx) = (c, r) + }, + Err(e) => { + error!("Invalid cert configuration. Configuration does not updated: {e}"); + continue; + } + }; + info!("Configuration updated. Terminate all spawned proxy services and force to re-bind TCP/UDP sockets"); term_notify.notify_waiters(); // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } + cert_res = cert_service.start() => { + error!("cert reloader service exited"); + return cert_res.map_err(|e| anyhow!(e)); + } else => break } } - Err(anyhow::anyhow!("rpxy or continuous monitoring service exited")) + Ok(()) } diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index 732901d9..2205ebba 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -22,6 +22,7 @@ thiserror = { version = "1.0.61" } hot_reload = { version = "0.1.5" } async-trait = { version = "0.1.80" } rustls = { version = "0.23.8", default-features = false, features = [ + "std", "aws_lc_rs", ] } rustls-pemfile = { version = "2.1.2" } @@ -33,9 +34,6 @@ x509-parser = { version = "0.16.0" } [dev-dependencies] tokio = { version = "1.37.0", default-features = false, features = [ - # "net", "rt-multi-thread", - # "time", - # "sync", "macros", ] } diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs index 26e16b75..7b0ebe8b 100644 --- a/rpxy-certs/src/error.rs +++ b/rpxy-certs/src/error.rs @@ -18,4 +18,10 @@ pub enum RpxyCertError { /// Error when converting server name bytes to string #[error("Failed to convert server name bytes to string: {0}")] ServerNameBytesToString(#[from] std::string::FromUtf8Error), + /// Rustls error + #[error("Rustls error: {0}")] + RustlsError(#[from] rustls::Error), + /// Rustls CryptoProvider error + #[error("Rustls No default CryptoProvider error")] + NoDefaultCryptoProvider, } diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index 106dbdd4..6a262f04 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -9,18 +9,17 @@ mod log { pub(super) use tracing::{debug, error, info, warn}; } -use crate::{ - error::*, - reloader_service::{CryptoReloader, DynCryptoSource}, -}; +use crate::{error::*, log::*, reloader_service::DynCryptoSource}; use hot_reload::{ReloaderReceiver, ReloaderService}; use rustc_hash::FxHashMap as HashMap; +use rustls::crypto::{aws_lc_rs, CryptoProvider}; use std::sync::Arc; /* ------------------------------------------------ */ pub use crate::{ certs::SingleServerCertsKeys, crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, + reloader_service::CryptoReloader, server_crypto::{ServerCrypto, ServerCryptoBase}, }; @@ -44,6 +43,10 @@ pub async fn build_cert_reloader( where T: CryptoSource + Send + Sync + Clone + 'static, { + info!("Building certificate reloader service"); + // Install aws_lc_rs as default crypto provider for rustls + let _ = CryptoProvider::install_default(aws_lc_rs::default_provider()); + let source = crypto_source_map .iter() .map(|(k, v)| { diff --git a/rpxy-certs/src/server_crypto.rs b/rpxy-certs/src/server_crypto.rs index 53fbcc3f..72932d61 100644 --- a/rpxy-certs/src/server_crypto.rs +++ b/rpxy-certs/src/server_crypto.rs @@ -1,6 +1,7 @@ use crate::{certs::SingleServerCertsKeys, error::*, log::*}; use rustc_hash::FxHashMap as HashMap; use rustls::{ + crypto::CryptoProvider, server::{ResolvesServerCertUsingSni, WebPkiClientVerifier}, RootCertStore, ServerConfig, }; @@ -40,7 +41,6 @@ impl TryInto> for &ServerCryptoBase { fn try_into(self) -> Result, Self::Error> { let aggregated = self.build_aggrated_server_crypto()?; let individual = self.build_individual_server_crypto_map()?; - Ok(Arc::new(ServerCrypto { aggregated_config_no_client_auth: Arc::new(aggregated), individual_config_map: Arc::new(individual), @@ -53,6 +53,9 @@ impl ServerCryptoBase { fn build_individual_server_crypto_map(&self) -> Result { let mut server_crypto_map: ServerNameCryptoMap = HashMap::default(); + // AWS LC provider by default + let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?; + for (server_name_bytes, certs_keys) in self.inner.iter() { let server_name = server_name_bytes_to_string(server_name_bytes)?; @@ -69,9 +72,11 @@ impl ServerCryptoBase { // With no client authentication case if !certs_keys.is_mutual_tls() { - let mut server_crypto_local = ServerConfig::builder() + let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_no_client_auth() .with_cert_resolver(Arc::new(resolver_local)); + #[cfg(feature = "http3")] { server_crypto_local.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]; @@ -93,11 +98,14 @@ impl ServerCryptoBase { let trust_anchors_without_skid = trust_anchors.values().map(|ta| ta.to_owned()); client_ca_roots_local.extend(trust_anchors_without_skid); - let Ok(client_cert_verifier) = WebPkiClientVerifier::builder(Arc::new(client_ca_roots_local)).build() else { + let Ok(client_cert_verifier) = + WebPkiClientVerifier::builder_with_provider(Arc::new(client_ca_roots_local), provider.clone()).build() + else { warn!("Failed to build client CA certificate verifier for {server_name}"); continue; }; - let mut server_crypto_local = ServerConfig::builder() + let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_client_cert_verifier(client_cert_verifier) .with_cert_resolver(Arc::new(resolver_local)); server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; @@ -112,6 +120,9 @@ impl ServerCryptoBase { fn build_aggrated_server_crypto(&self) -> Result { let mut resolver_global = ResolvesServerCertUsingSni::new(); + // AWS LC provider by default + let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?; + for (server_name_bytes, certs_keys) in self.inner.iter() { let server_name = server_name_bytes_to_string(server_name_bytes)?; @@ -129,7 +140,8 @@ impl ServerCryptoBase { } } - let mut server_crypto_global = ServerConfig::builder() + let mut server_crypto_global = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_no_client_auth() .with_cert_resolver(Arc::new(resolver_global)); @@ -145,3 +157,48 @@ impl ServerCryptoBase { Ok(server_crypto_global) } } + +/* ------------------------------------------------ */ +#[cfg(test)] +mod tests { + use super::*; + use crate::{CryptoFileSourceBuilder, CryptoSource}; + use std::convert::TryInto; + + async fn read_file_source() -> SingleServerCertsKeys { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt"); + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .client_ca_cert_path(client_ca_cert_path) + .build(); + crypto_file_source.unwrap().read().await.unwrap() + } + + #[tokio::test] + async fn test_server_crypto_base_try_into() { + let mut server_crypto_base = ServerCryptoBase::default(); + + let single_certs_keys = read_file_source().await; + server_crypto_base.inner.insert(b"localhost".to_vec(), single_certs_keys); + let server_crypto: Arc = (&server_crypto_base).try_into().unwrap(); + assert_eq!(server_crypto.individual_config_map.len(), 1); + + #[cfg(feature = "http3")] + { + assert_eq!( + server_crypto.aggregated_config_no_client_auth.alpn_protocols, + vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + #[cfg(not(feature = "http3"))] + { + assert_eq!( + server_crypto.aggregated_config_no_client_auth.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + } +} From 7632b1fdebbcf743e709b67a1ce666dd9b96473a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 17:56:40 +0900 Subject: [PATCH 09/16] fix test --- .gitmodules | 3 --- rpxy-certs/src/server_crypto.rs | 2 ++ submodules/h3 | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 160000 submodules/h3 diff --git a/.gitmodules b/.gitmodules index 47ebad0b..0d6a4041 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "submodules/h3"] - path = submodules/h3 - url = git@github.com:junkurihara/h3.git [submodule "submodules/rusty-http-cache-semantics"] path = submodules/rusty-http-cache-semantics url = git@github.com:junkurihara/rusty-http-cache-semantics.git diff --git a/rpxy-certs/src/server_crypto.rs b/rpxy-certs/src/server_crypto.rs index 72932d61..c6061d2e 100644 --- a/rpxy-certs/src/server_crypto.rs +++ b/rpxy-certs/src/server_crypto.rs @@ -179,6 +179,8 @@ mod tests { #[tokio::test] async fn test_server_crypto_base_try_into() { + let _ = CryptoProvider::install_default(rustls::crypto::aws_lc_rs::default_provider()); + let mut server_crypto_base = ServerCryptoBase::default(); let single_certs_keys = read_file_source().await; diff --git a/submodules/h3 b/submodules/h3 deleted file mode 160000 index 34bf403b..00000000 --- a/submodules/h3 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 34bf403bad8893adc0363e45a74782828aa93ac8 From 2f9f0a1122b93fa6b51dd98b831cfce5e2024602 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 20:03:46 +0900 Subject: [PATCH 10/16] wip: started to integrate rpxy-certs to rpxy-lib --- Cargo.toml | 2 +- rpxy-bin/src/cert_file_reader.rs | 10 +++--- rpxy-bin/src/config/parse.rs | 9 +++-- rpxy-bin/src/main.rs | 59 +++++++++++++++++++++----------- rpxy-lib/Cargo.toml | 4 ++- rpxy-lib/src/globals.rs | 2 ++ rpxy-lib/src/lib.rs | 7 +++- rpxy-lib/src/proxy/proxy_main.rs | 20 +++++++++++ 8 files changed, 81 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6dde7e44..f51c4db3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.7.2" +version = "0.8.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" diff --git a/rpxy-bin/src/cert_file_reader.rs b/rpxy-bin/src/cert_file_reader.rs index ee9a591c..30c67947 100644 --- a/rpxy-bin/src/cert_file_reader.rs +++ b/rpxy-bin/src/cert_file_reader.rs @@ -69,12 +69,10 @@ fn read_certs_and_keys( let certs: Vec<_> = { let certs_path_str = cert_path.display().to_string(); - let mut reader = BufReader::new(File::open(cert_path).map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the certificates [{certs_path_str}]: {e}"), - ) - })?); + let mut reader = BufReader::new( + File::open(cert_path) + .map_err(|e| io::Error::new(e.kind(), format!("Unable to load the certificates [{certs_path_str}]: {e}")))?, + ); rustls_pemfile::certs(&mut reader) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the certificates"))? } diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 049f5ee8..535954fd 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -97,13 +97,16 @@ pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, pub async fn build_cert_manager( config: &ConfigToml, ) -> Result< - ( + Option<( ReloaderService, ReloaderReceiver, - ), + )>, anyhow::Error, > { let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?; + if config.listen_port_tls.is_none() { + return Ok(None); + } let mut crypto_source_map = HashMap::default(); for app in apps.0.values() { if let Some(tls) = app.tls.as_ref() { @@ -118,5 +121,5 @@ pub async fn build_cert_manager( } } let res = build_cert_reloader(&crypto_source_map, None).await?; - Ok(res) + Ok(Some(res)) } diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 114e6db1..3c0a63f1 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -66,20 +66,14 @@ async fn rpxy_service_without_watcher( info!("Start rpxy service"); let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?; let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; - let (cert_service, cert_rx) = build_cert_manager(&config_toml) + + let cert_service_and_rx = build_cert_manager(&config_toml) .await .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; - tokio::select! { - rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => { - error!("rpxy entrypoint exited"); - rpxy_res.map_err(|e| anyhow!(e)) - } - cert_res = cert_service.start() => { - error!("cert reloader service exited"); - cert_res.map_err(|e| anyhow!(e)) - } - } + rpxy_entrypoint(&proxy_conf, &app_conf, cert_service_and_rx.as_ref(), &runtime_handle, None) + .await + .map_err(|e| anyhow!(e)) } async fn rpxy_service_with_watcher( @@ -95,7 +89,7 @@ async fn rpxy_service_with_watcher( .ok_or(anyhow!("Something wrong in config reloader receiver"))?; let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; - let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml) + let mut cert_service_and_rx = build_cert_manager(&config_toml) .await .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; @@ -105,8 +99,8 @@ async fn rpxy_service_with_watcher( // Continuous monitoring loop { tokio::select! { - rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => { - error!("rpxy entrypoint exited"); + rpxy_res = rpxy_entrypoint(&proxy_conf, &app_conf, cert_service_and_rx.as_ref(), &runtime_handle, Some(term_notify.clone())) => { + error!("rpxy entrypoint or cert service exited"); return rpxy_res.map_err(|e| anyhow!(e)); } _ = config_rx.changed() => { @@ -124,8 +118,8 @@ async fn rpxy_service_with_watcher( } }; match build_cert_manager(&config_toml).await { - Ok((c, r)) => { - (cert_service, cert_rx) = (c, r) + Ok(c) => { + cert_service_and_rx = c; }, Err(e) => { error!("Invalid cert configuration. Configuration does not updated: {e}"); @@ -137,13 +131,38 @@ async fn rpxy_service_with_watcher( term_notify.notify_waiters(); // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } - cert_res = cert_service.start() => { - error!("cert reloader service exited"); - return cert_res.map_err(|e| anyhow!(e)); - } else => break } } Ok(()) } + +/// Wrapper of entry point for rpxy service with certificate management service +async fn rpxy_entrypoint( + proxy_config: &rpxy_lib::ProxyConfig, + app_config_list: &rpxy_lib::AppConfigList, + cert_service_and_rx: Option<&( + ReloaderService, + ReloaderReceiver, + )>, // TODO: + runtime_handle: &tokio::runtime::Handle, + term_notify: Option>, +) -> Result<(), anyhow::Error> { + if let Some((cert_service, cert_rx)) = cert_service_and_rx { + tokio::select! { + rpxy_res = entrypoint(proxy_config, app_config_list, Some(cert_rx), runtime_handle, term_notify) => { + error!("rpxy entrypoint exited"); + rpxy_res.map_err(|e| anyhow!(e)) + } + cert_res = cert_service.start() => { + error!("cert reloader service exited"); + cert_res.map_err(|e| anyhow!(e)) + } + } + } else { + entrypoint(proxy_config, app_config_list, None, runtime_handle, term_notify) + .await + .map_err(|e| anyhow!(e)) + } +} diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index 7baaf9c2..fd5df4da 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -14,13 +14,14 @@ publish.workspace = true [features] default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] -http3-quinn = ["socket2", "quinn", "h3", "h3-quinn"] +http3-quinn = ["socket2", "quinn", "h3", "h3-quinn", "rpxy-certs/http3"] http3-s2n = [ "h3", "s2n-quic", "s2n-quic-core", "s2n-quic-rustls", "s2n-quic-h3", + "rpxy-certs/http3", ] cache = ["http-cache-semantics", "lru", "sha2", "base64"] sticky-cookie = ["base64", "sha2", "chrono"] @@ -70,6 +71,7 @@ hyper-rustls = { version = "0.27.1", default-features = false, features = [ ], optional = true } # tls and cert management for server +rpxy-certs = { path = "../rpxy-certs/", default-features = false } hot_reload = "0.1.5" rustls = { version = "0.21.12", default-features = false } tokio-rustls = { version = "0.24.1", features = ["early-data"] } diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index e4bff9e1..1c272dc2 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -19,6 +19,8 @@ pub struct Globals { pub term_notify: Option>, /// Shared context - Certificate reloader service receiver pub cert_reloader_rx: Option>, + /// Shared context - Certificate reloader service receiver // TODO: newer one + pub cert_reloader_rx_new: Option>, } /// Configuration parameters for proxy transport and request handlers diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 115b78a4..4d728a4e 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -10,14 +10,17 @@ mod log; mod message_handler; mod name_exp; mod proxy; - +/* ------------------------------------------------ */ use crate::{ crypto::build_cert_reloader, error::*, forwarder::Forwarder, globals::Globals, log::*, message_handler::HttpMessageHandlerBuilder, proxy::Proxy, }; use futures::future::select_all; +use hot_reload::ReloaderReceiver; +use rpxy_certs::ServerCryptoBase; use std::sync::Arc; +/* ------------------------------------------------ */ pub use crate::{ crypto::{CertsAndKeys, CryptoSource}, globals::{AppConfig, AppConfigList, ProxyConfig, ReverseProxyConfig, TlsConfig, UpstreamUri}, @@ -31,6 +34,7 @@ pub mod reexports { pub async fn entrypoint( proxy_config: &ProxyConfig, app_config_list: &AppConfigList, + cert_rx: Option<&ReloaderReceiver>, // TODO: runtime_handle: &tokio::runtime::Handle, term_notify: Option>, ) -> RpxyResult<()> @@ -94,6 +98,7 @@ where runtime_handle: runtime_handle.clone(), term_notify: term_notify.clone(), cert_reloader_rx: cert_reloader_rx.clone(), + cert_reloader_rx_new: cert_rx.cloned(), // TODO: newer one }); // 4. build message handler containing Arc-ed http_client and backends, and make it contained in Arc as well diff --git a/rpxy-lib/src/proxy/proxy_main.rs b/rpxy-lib/src/proxy/proxy_main.rs index 67eeb30b..9d3baa88 100644 --- a/rpxy-lib/src/proxy/proxy_main.rs +++ b/rpxy-lib/src/proxy/proxy_main.rs @@ -164,6 +164,10 @@ where let Some(mut server_crypto_rx) = self.globals.cert_reloader_rx.clone() else { return Err(RpxyError::NoCertificateReloader); }; + // TODO: newer one + let Some(mut server_crypto_rx_new) = self.globals.cert_reloader_rx_new.clone() else { + return Err(RpxyError::NoCertificateReloader); + }; let tcp_socket = bind_tcp_socket(&self.listening_on)?; let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?; info!("Start TCP proxy serving with HTTPS request for configured host names"); @@ -237,6 +241,22 @@ where }; server_crypto_map = Some(server_crypto.inner_local_map.clone()); } + // TODO: newer one + _ = server_crypto_rx_new.changed().fuse() => { + if server_crypto_rx_new.borrow().is_none() { + error!("Reloader is broken"); + break; + } + let cert_keys_map = server_crypto_rx_new.borrow().clone().unwrap(); + // let Some(server_crypto) = cert_keys_map.try_into().ok() else { + // break; + // }; + // let Some(server_crypto): Option> = (&cert_keys_map).try_into().ok() else { + // error!("Failed to update server crypto"); + // break; + // }; + // server_crypto_map = Some(server_crypto.inner_local_map.clone()); + } } } Ok(()) From 0c6f3edf1865e79535b02dc68856506db8f98d79 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 20:49:11 +0900 Subject: [PATCH 11/16] wip: support rustls-0.23 for http1.1 and 1.2 --- rpxy-bin/Cargo.toml | 5 +- rpxy-bin/src/cert_file_reader.rs | 183 ------------------ rpxy-bin/src/config/parse.rs | 9 +- rpxy-bin/src/config/toml.rs | 13 +- rpxy-bin/src/main.rs | 3 +- rpxy-lib/Cargo.toml | 7 +- rpxy-lib/src/backend/backend_main.rs | 59 ++---- rpxy-lib/src/backend/upstream.rs | 8 +- rpxy-lib/src/constants.rs | 11 -- rpxy-lib/src/error.rs | 4 +- rpxy-lib/src/globals.rs | 31 +-- rpxy-lib/src/lib.rs | 63 ++---- rpxy-lib/src/message_handler/handler_main.rs | 18 +- .../handler_manipulate_messages.rs | 6 +- rpxy-lib/src/proxy/mod.rs | 6 + rpxy-lib/src/proxy/proxy_main.rs | 47 ++--- 16 files changed, 80 insertions(+), 393 deletions(-) delete mode 100644 rpxy-bin/src/cert_file_reader.rs diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index d6dd8429..a8c89e65 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -13,7 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["http3-quinn", "cache", "rustls-backend"] +# default = ["http3-quinn", "cache", "rustls-backend"] +default = ["cache", "rustls-backend"] http3-quinn = ["rpxy-lib/http3-quinn"] http3-s2n = ["rpxy-lib/http3-s2n"] native-tls-backend = ["rpxy-lib/native-tls-backend"] @@ -30,7 +31,6 @@ mimalloc = { version = "*", default-features = false } anyhow = "1.0.86" rustc-hash = "1.1.0" serde = { version = "1.0.202", default-features = false, features = ["derive"] } -derive_builder = "0.20.0" tokio = { version = "1.37.0", default-features = false, features = [ "net", "rt-multi-thread", @@ -39,7 +39,6 @@ tokio = { version = "1.37.0", default-features = false, features = [ "macros", ] } async-trait = "0.1.80" -rustls-pemfile = "1.0.4" # config diff --git a/rpxy-bin/src/cert_file_reader.rs b/rpxy-bin/src/cert_file_reader.rs deleted file mode 100644 index 30c67947..00000000 --- a/rpxy-bin/src/cert_file_reader.rs +++ /dev/null @@ -1,183 +0,0 @@ -use crate::log::*; -use async_trait::async_trait; -use derive_builder::Builder; -use rpxy_lib::{ - reexports::{Certificate, PrivateKey}, - CertsAndKeys, CryptoSource, -}; -use std::{ - fs::File, - io::{self, BufReader, Cursor, Read}, - path::{Path, PathBuf}, -}; - -#[derive(Builder, Debug, Clone)] -/// Crypto-related file reader implementing certs::CryptoRead trait -pub struct CryptoFileSource { - #[builder(setter(custom))] - /// Always exist - pub tls_cert_path: PathBuf, - - #[builder(setter(custom))] - /// Always exist - pub tls_cert_key_path: PathBuf, - - #[builder(setter(custom), default)] - /// This may not exist - pub client_ca_cert_path: Option, -} - -impl CryptoFileSourceBuilder { - pub fn tls_cert_path>(&mut self, v: T) -> &mut Self { - self.tls_cert_path = Some(v.as_ref().to_path_buf()); - self - } - pub fn tls_cert_key_path>(&mut self, v: T) -> &mut Self { - self.tls_cert_key_path = Some(v.as_ref().to_path_buf()); - self - } - pub fn client_ca_cert_path>(&mut self, v: Option) -> &mut Self { - self.client_ca_cert_path = Some(v.map(|p| p.as_ref().to_path_buf())); - self - } -} - -#[async_trait] -impl CryptoSource for CryptoFileSource { - type Error = io::Error; - /// read crypto materials from source - async fn read(&self) -> Result { - read_certs_and_keys( - &self.tls_cert_path, - &self.tls_cert_key_path, - self.client_ca_cert_path.as_ref(), - ) - } - /// Returns true when mutual tls is enabled - fn is_mutual_tls(&self) -> bool { - self.client_ca_cert_path.is_some() - } -} - -/// Read certificates and private keys from file -fn read_certs_and_keys( - cert_path: &PathBuf, - cert_key_path: &PathBuf, - client_ca_cert_path: Option<&PathBuf>, -) -> Result { - debug!("Read TLS server certificates and private key"); - - let certs: Vec<_> = { - let certs_path_str = cert_path.display().to_string(); - let mut reader = BufReader::new( - File::open(cert_path) - .map_err(|e| io::Error::new(e.kind(), format!("Unable to load the certificates [{certs_path_str}]: {e}")))?, - ); - rustls_pemfile::certs(&mut reader) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the certificates"))? - } - .drain(..) - .map(Certificate) - .collect(); - - let cert_keys: Vec<_> = { - let cert_key_path_str = cert_key_path.display().to_string(); - let encoded_keys = { - let mut encoded_keys = vec![]; - File::open(cert_key_path) - .map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the certificate keys [{cert_key_path_str}]: {e}"), - ) - })? - .read_to_end(&mut encoded_keys)?; - encoded_keys - }; - let mut reader = Cursor::new(encoded_keys); - let pkcs8_keys = rustls_pemfile::pkcs8_private_keys(&mut reader).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Unable to parse the certificates private keys (PKCS8)", - ) - })?; - reader.set_position(0); - let mut rsa_keys = rustls_pemfile::rsa_private_keys(&mut reader)?; - let mut keys = pkcs8_keys; - keys.append(&mut rsa_keys); - if keys.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "No private keys found - Make sure that they are in PKCS#8/PEM format", - )); - } - keys.drain(..).map(PrivateKey).collect() - }; - - let client_ca_certs = if let Some(path) = client_ca_cert_path { - debug!("Read CA certificates for client authentication"); - // Reads client certificate and returns client - let certs: Vec<_> = { - let certs_path_str = path.display().to_string(); - let mut reader = BufReader::new(File::open(path).map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the client certificates [{certs_path_str}]: {e}"), - ) - })?); - rustls_pemfile::certs(&mut reader) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the client certificates"))? - } - .drain(..) - .map(Certificate) - .collect(); - Some(certs) - } else { - None - }; - - Ok(CertsAndKeys { - certs, - cert_keys, - client_ca_certs, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - #[tokio::test] - async fn read_server_crt_key_files() { - let tls_cert_path = "../example-certs/server.crt"; - let tls_cert_key_path = "../example-certs/server.key"; - let crypto_file_source = CryptoFileSourceBuilder::default() - .tls_cert_key_path(tls_cert_key_path) - .tls_cert_path(tls_cert_path) - .build(); - assert!(crypto_file_source.is_ok()); - - let crypto_file_source = crypto_file_source.unwrap(); - let crypto_elem = crypto_file_source.read().await; - assert!(crypto_elem.is_ok()); - } - - #[tokio::test] - async fn read_server_crt_key_files_with_client_ca_crt() { - let tls_cert_path = "../example-certs/server.crt"; - let tls_cert_key_path = "../example-certs/server.key"; - let client_ca_cert_path = Some("../example-certs/client.ca.crt"); - let crypto_file_source = CryptoFileSourceBuilder::default() - .tls_cert_key_path(tls_cert_key_path) - .tls_cert_path(tls_cert_path) - .client_ca_cert_path(client_ca_cert_path) - .build(); - assert!(crypto_file_source.is_ok()); - - let crypto_file_source = crypto_file_source.unwrap(); - let crypto_elem = crypto_file_source.read().await; - assert!(crypto_elem.is_ok()); - - let crypto_elem = crypto_elem.unwrap(); - assert!(crypto_elem.client_ca_certs.is_some()); - } -} diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 535954fd..95c87d20 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -1,8 +1,5 @@ use super::toml::ConfigToml; -use crate::{ - cert_file_reader::CryptoFileSource, - error::{anyhow, ensure}, -}; +use crate::error::{anyhow, ensure}; use clap::{Arg, ArgAction}; use hot_reload::{ReloaderReceiver, ReloaderService}; use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase}; @@ -43,7 +40,7 @@ pub fn parse_opts() -> Result { Ok(Opts { config_file_path, watch }) } -pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { +pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { // build proxy config let proxy_config: ProxyConfig = config.try_into()?; @@ -74,7 +71,7 @@ pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, } // build applications - let mut app_config_list_inner = Vec::>::new(); + let mut app_config_list_inner = Vec::::new(); // let mut backends = Backends::new(); for (app_name, app) in apps.0.iter() { diff --git a/rpxy-bin/src/config/toml.rs b/rpxy-bin/src/config/toml.rs index 7cd3653c..9b6502b3 100644 --- a/rpxy-bin/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -1,5 +1,4 @@ use crate::{ - cert_file_reader::{CryptoFileSource, CryptoFileSourceBuilder}, constants::*, error::{anyhow, ensure}, }; @@ -214,7 +213,7 @@ impl ConfigToml { } impl Application { - pub fn build_app_config(&self, app_name: &str) -> std::result::Result, anyhow::Error> { + pub fn build_app_config(&self, app_name: &str) -> std::result::Result { let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; // reverse proxy settings @@ -224,11 +223,6 @@ impl Application { let tls_config = if self.tls.is_some() { let tls = self.tls.as_ref().unwrap(); ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); - let inner = CryptoFileSourceBuilder::default() - .tls_cert_path(tls.tls_cert_path.as_ref().unwrap()) - .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap()) - .client_ca_cert_path(tls.client_ca_cert_path.as_deref()) - .build()?; let https_redirection = if tls.https_redirection.is_none() { true // Default true @@ -236,10 +230,7 @@ impl Application { tls.https_redirection.unwrap() }; - Some(TlsConfig { - inner, - https_redirection, - }) + Some(TlsConfig { https_redirection }) } else { None }; diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 3c0a63f1..d3988e31 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -1,7 +1,6 @@ #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; -mod cert_file_reader; mod config; mod constants; mod error; @@ -141,7 +140,7 @@ async fn rpxy_service_with_watcher( /// Wrapper of entry point for rpxy service with certificate management service async fn rpxy_entrypoint( proxy_config: &rpxy_lib::ProxyConfig, - app_config_list: &rpxy_lib::AppConfigList, + app_config_list: &rpxy_lib::AppConfigList, cert_service_and_rx: Option<&( ReloaderService, ReloaderReceiver, diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index fd5df4da..c597d65a 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -13,7 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] +default = ["sticky-cookie", "cache", "rustls-backend"] +# default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] http3-quinn = ["socket2", "quinn", "h3", "h3-quinn", "rpxy-certs/http3"] http3-s2n = [ "h3", @@ -73,8 +74,8 @@ hyper-rustls = { version = "0.27.1", default-features = false, features = [ # tls and cert management for server rpxy-certs = { path = "../rpxy-certs/", default-features = false } hot_reload = "0.1.5" -rustls = { version = "0.21.12", default-features = false } -tokio-rustls = { version = "0.24.1", features = ["early-data"] } +rustls = { version = "0.23.8", default-features = false } +tokio-rustls = { version = "0.26.0", features = ["early-data"] } webpki = "0.22.4" x509-parser = "0.16.0" diff --git a/rpxy-lib/src/backend/backend_main.rs b/rpxy-lib/src/backend/backend_main.rs index d9fa649f..663120e6 100644 --- a/rpxy-lib/src/backend/backend_main.rs +++ b/rpxy-lib/src/backend/backend_main.rs @@ -1,5 +1,4 @@ use crate::{ - crypto::CryptoSource, error::*, log::*, name_exp::{ByteName, ServerName}, @@ -13,10 +12,7 @@ use super::upstream::PathManager; /// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings. #[derive(Builder)] -pub struct BackendApp -where - T: CryptoSource, -{ +pub struct BackendApp { #[builder(setter(into))] /// backend application name, e.g., app1 pub app_name: String, @@ -28,50 +24,27 @@ where /// tls settings: https redirection with 30x #[builder(default)] pub https_redirection: Option, - /// TLS settings: source meta for server cert, key, client ca cert - #[builder(default)] - pub crypto_source: Option, } -impl<'a, T> BackendAppBuilder -where - T: CryptoSource, -{ +impl<'a> BackendAppBuilder { pub fn server_name(&mut self, server_name: impl Into>) -> &mut Self { self.server_name = Some(server_name.to_server_name()); self } } +#[derive(Default)] /// HashMap and some meta information for multiple Backend structs. -pub struct BackendAppManager -where - T: CryptoSource, -{ +pub struct BackendAppManager { /// HashMap of Backend structs, key is server name - pub apps: HashMap>, + pub apps: HashMap, /// for plaintext http pub default_server_name: Option, } -impl Default for BackendAppManager -where - T: CryptoSource, -{ - fn default() -> Self { - Self { - apps: HashMap::>::default(), - default_server_name: None, - } - } -} - -impl TryFrom<&AppConfig> for BackendApp -where - T: CryptoSource + Clone, -{ +impl TryFrom<&AppConfig> for BackendApp { type Error = RpxyError; - fn try_from(app_config: &AppConfig) -> Result { + fn try_from(app_config: &AppConfig) -> Result { let mut backend_builder = BackendAppBuilder::default(); let path_manager = PathManager::try_from(app_config)?; backend_builder @@ -83,28 +56,20 @@ where backend_builder.build()? } else { let tls = app_config.tls.as_ref().unwrap(); - backend_builder - .https_redirection(Some(tls.https_redirection)) - .crypto_source(Some(tls.inner.clone())) - .build()? + backend_builder.https_redirection(Some(tls.https_redirection)).build()? }; Ok(backend) } } -impl TryFrom<&AppConfigList> for BackendAppManager -where - T: CryptoSource + Clone, -{ +impl TryFrom<&AppConfigList> for BackendAppManager { type Error = RpxyError; - fn try_from(config_list: &AppConfigList) -> Result { + fn try_from(config_list: &AppConfigList) -> Result { let mut manager = Self::default(); for app_config in config_list.inner.iter() { - let backend: BackendApp = BackendApp::try_from(app_config)?; - manager - .apps - .insert(app_config.server_name.clone().to_server_name(), backend); + let backend: BackendApp = BackendApp::try_from(app_config)?; + manager.apps.insert(app_config.server_name.clone().to_server_name(), backend); info!( "Registering application {} ({})", diff --git a/rpxy-lib/src/backend/upstream.rs b/rpxy-lib/src/backend/upstream.rs index 702be290..7d9d569f 100644 --- a/rpxy-lib/src/backend/upstream.rs +++ b/rpxy-lib/src/backend/upstream.rs @@ -6,7 +6,6 @@ use super::load_balance::{ // use super::{BytesName, LbContext, PathNameBytesExp, UpstreamOption}; use super::upstream_opts::UpstreamOption; use crate::{ - crypto::CryptoSource, error::RpxyError, globals::{AppConfig, UpstreamUri}, log::*, @@ -28,12 +27,9 @@ pub struct PathManager { inner: HashMap, } -impl TryFrom<&AppConfig> for PathManager -where - T: CryptoSource, -{ +impl TryFrom<&AppConfig> for PathManager { type Error = RpxyError; - fn try_from(app_config: &AppConfig) -> Result { + fn try_from(app_config: &AppConfig) -> Result { let mut inner: HashMap = HashMap::default(); app_config.reverse_proxy.iter().for_each(|rpc| { diff --git a/rpxy-lib/src/constants.rs b/rpxy-lib/src/constants.rs index acc93810..9e2bc6a7 100644 --- a/rpxy-lib/src/constants.rs +++ b/rpxy-lib/src/constants.rs @@ -1,21 +1,10 @@ pub const RESPONSE_HEADER_SERVER: &str = "rpxy"; -// pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; -// pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; pub const TCP_LISTEN_BACKLOG: u32 = 1024; -// pub const HTTP_LISTEN_PORT: u16 = 8080; -// pub const HTTPS_LISTEN_PORT: u16 = 8443; pub const PROXY_IDLE_TIMEOUT_SEC: u64 = 20; pub const UPSTREAM_IDLE_TIMEOUT_SEC: u64 = 20; pub const TLS_HANDSHAKE_TIMEOUT_SEC: u64 = 15; // default as with firefox browser pub const MAX_CLIENTS: usize = 512; pub const MAX_CONCURRENT_STREAMS: u32 = 64; -pub const CERTS_WATCH_DELAY_SECS: u32 = 60; -pub const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true; - -// #[cfg(feature = "http3")] -// pub const H3_RESPONSE_BUF_SIZE: usize = 65_536; // 64KB -// #[cfg(feature = "http3")] -// pub const H3_REQUEST_BUF_SIZE: usize = 65_536; // 64KB // handled by quinn #[allow(non_snake_case)] #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] diff --git a/rpxy-lib/src/error.rs b/rpxy-lib/src/error.rs index 3b1afc98..85c05f44 100644 --- a/rpxy-lib/src/error.rs +++ b/rpxy-lib/src/error.rs @@ -59,8 +59,8 @@ pub enum RpxyError { // certificate reloader errors #[error("No certificate reloader when building a proxy for TLS")] NoCertificateReloader, - #[error("Certificate reload error: {0}")] - CertificateReloadError(#[from] hot_reload::ReloaderError), + // #[error("Certificate reload error: {0}")] + // CertificateReloadError(#[from] hot_reload::ReloaderError), // backend errors #[error("Invalid reverse proxy setting")] diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index 1c272dc2..fec0a53d 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -1,9 +1,6 @@ -use crate::{ - constants::*, - count::RequestCount, - crypto::{CryptoSource, ServerCryptoBase}, -}; +use crate::{constants::*, count::RequestCount}; use hot_reload::ReloaderReceiver; +use rpxy_certs::ServerCryptoBase; use std::{net::SocketAddr, sync::Arc, time::Duration}; /// Global object containing proxy configurations and shared object like counters. @@ -17,10 +14,8 @@ pub struct Globals { pub runtime_handle: tokio::runtime::Handle, /// Shared context - Notify object to stop async tasks pub term_notify: Option>, - /// Shared context - Certificate reloader service receiver - pub cert_reloader_rx: Option>, /// Shared context - Certificate reloader service receiver // TODO: newer one - pub cert_reloader_rx_new: Option>, + pub cert_reloader_rx: Option>, } /// Configuration parameters for proxy transport and request handlers @@ -129,24 +124,18 @@ impl Default for ProxyConfig { /// Configuration parameters for backend applications #[derive(PartialEq, Eq, Clone)] -pub struct AppConfigList -where - T: CryptoSource, -{ - pub inner: Vec>, +pub struct AppConfigList { + pub inner: Vec, pub default_app: Option, } /// Configuration parameters for single backend application #[derive(PartialEq, Eq, Clone)] -pub struct AppConfig -where - T: CryptoSource, -{ +pub struct AppConfig { pub app_name: String, pub server_name: String, pub reverse_proxy: Vec, - pub tls: Option>, + pub tls: Option, } /// Configuration parameters for single reverse proxy corresponding to the path @@ -167,10 +156,6 @@ pub struct UpstreamUri { /// Configuration parameters on TLS for a single backend application #[derive(PartialEq, Eq, Clone)] -pub struct TlsConfig -where - T: CryptoSource, -{ - pub inner: T, +pub struct TlsConfig { pub https_redirection: bool, } diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 4d728a4e..ccb647e9 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -1,7 +1,6 @@ mod backend; mod constants; mod count; -mod crypto; mod error; mod forwarder; mod globals; @@ -12,8 +11,13 @@ mod name_exp; mod proxy; /* ------------------------------------------------ */ use crate::{ - crypto::build_cert_reloader, error::*, forwarder::Forwarder, globals::Globals, log::*, - message_handler::HttpMessageHandlerBuilder, proxy::Proxy, + // crypto::build_cert_reloader, + error::*, + forwarder::Forwarder, + globals::Globals, + log::*, + message_handler::HttpMessageHandlerBuilder, + proxy::Proxy, }; use futures::future::select_all; use hot_reload::ReloaderReceiver; @@ -21,26 +25,19 @@ use rpxy_certs::ServerCryptoBase; use std::sync::Arc; /* ------------------------------------------------ */ -pub use crate::{ - crypto::{CertsAndKeys, CryptoSource}, - globals::{AppConfig, AppConfigList, ProxyConfig, ReverseProxyConfig, TlsConfig, UpstreamUri}, -}; +pub use crate::globals::{AppConfig, AppConfigList, ProxyConfig, ReverseProxyConfig, TlsConfig, UpstreamUri}; pub mod reexports { pub use hyper::Uri; - pub use rustls::{Certificate, PrivateKey}; } /// Entrypoint that creates and spawns tasks of reverse proxy services -pub async fn entrypoint( +pub async fn entrypoint( proxy_config: &ProxyConfig, - app_config_list: &AppConfigList, + app_config_list: &AppConfigList, cert_rx: Option<&ReloaderReceiver>, // TODO: runtime_handle: &tokio::runtime::Handle, term_notify: Option>, -) -> RpxyResult<()> -where - T: CryptoSource + Clone + Send + Sync + 'static, -{ +) -> RpxyResult<()> { #[cfg(all(feature = "http3-quinn", feature = "http3-s2n"))] warn!("Both \"http3-quinn\" and \"http3-s2n\" features are enabled. \"http3-quinn\" will be used"); @@ -82,26 +79,16 @@ where // 1. build backends, and make it contained in Arc let app_manager = Arc::new(backend::BackendAppManager::try_from(app_config_list)?); - // 2. build crypto reloader service - let (cert_reloader_service, cert_reloader_rx) = match proxy_config.https_port { - Some(_) => { - let (s, r) = build_cert_reloader(&app_manager).await?; - (Some(s), Some(r)) - } - None => (None, None), - }; - - // 3. build global shared context + // 2. build global shared context let globals = Arc::new(Globals { proxy_config: proxy_config.clone(), request_count: Default::default(), runtime_handle: runtime_handle.clone(), term_notify: term_notify.clone(), - cert_reloader_rx: cert_reloader_rx.clone(), - cert_reloader_rx_new: cert_rx.cloned(), // TODO: newer one + cert_reloader_rx: cert_rx.cloned(), }); - // 4. build message handler containing Arc-ed http_client and backends, and make it contained in Arc as well + // 3. build message handler containing Arc-ed http_client and backends, and make it contained in Arc as well let forwarder = Arc::new(Forwarder::try_new(&globals).await?); let message_handler = Arc::new( HttpMessageHandlerBuilder::default() @@ -111,7 +98,7 @@ where .build()?, ); - // 5. spawn each proxy for a given socket with copied Arc-ed message_handler. + // 4. spawn each proxy for a given socket with copied Arc-ed message_handler. // build hyper connection builder shared with proxy instances let connection_builder = proxy::connection_builder(&globals); @@ -132,23 +119,9 @@ where globals.runtime_handle.spawn(async move { proxy.start().await }) }); - // wait for all future - match cert_reloader_service { - Some(cert_service) => { - tokio::select! { - _ = cert_service.start() => { - error!("Certificate reloader service got down"); - } - _ = select_all(futures_iter) => { - error!("Some proxy services are down"); - } - } - } - None => { - if let (Ok(Err(e)), _, _) = select_all(futures_iter).await { - error!("Some proxy services are down: {}", e); - } - } + if let (Ok(Err(e)), _, _) = select_all(futures_iter).await { + error!("Some proxy services are down: {}", e); + return Err(e); } Ok(()) diff --git a/rpxy-lib/src/message_handler/handler_main.rs b/rpxy-lib/src/message_handler/handler_main.rs index c46ac85d..9ce63f8f 100644 --- a/rpxy-lib/src/message_handler/handler_main.rs +++ b/rpxy-lib/src/message_handler/handler_main.rs @@ -7,7 +7,6 @@ use super::{ }; use crate::{ backend::{BackendAppManager, LoadBalanceContext}, - crypto::CryptoSource, error::*, forwarder::{ForwardRequest, Forwarder}, globals::Globals, @@ -34,20 +33,18 @@ pub(super) struct HandlerContext { #[derive(Clone, Builder)] /// HTTP message handler for requests from clients and responses from backend applications, /// responsible to manipulate and forward messages to upstream backends and downstream clients. -pub struct HttpMessageHandler +pub struct HttpMessageHandler where C: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone, { forwarder: Arc>, pub(super) globals: Arc, - app_manager: Arc>, + app_manager: Arc, } -impl HttpMessageHandler +impl HttpMessageHandler where C: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone, { /// Handle incoming request message from a client. /// Responsible to passthrough responses from backend applications or generate synthetic error responses. @@ -64,14 +61,7 @@ where log_data.client_addr(&client_addr); let http_result = self - .handle_request_inner( - &mut log_data, - req, - client_addr, - listen_addr, - tls_enabled, - tls_server_name, - ) + .handle_request_inner(&mut log_data, req, client_addr, listen_addr, tls_enabled, tls_server_name) .await; // passthrough or synthetic response diff --git a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs index fe55fe5f..2d77d2c2 100644 --- a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs +++ b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs @@ -3,17 +3,15 @@ use crate::{ backend::{BackendApp, UpstreamCandidates}, constants::RESPONSE_HEADER_SERVER, log::*, - CryptoSource, }; use anyhow::{anyhow, ensure, Result}; use http::{header, HeaderValue, Request, Response, Uri}; use hyper_util::client::legacy::connect::Connect; use std::net::SocketAddr; -impl HttpMessageHandler +impl HttpMessageHandler where C: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone, { //////////////////////////////////////////////////// // Functions to generate messages @@ -21,7 +19,7 @@ where #[allow(unused_variables)] /// Manipulate a response message sent from a backend application to forward downstream to a client. - pub(super) fn generate_response_forwarded(&self, response: &mut Response, backend_app: &BackendApp) -> Result<()> { + pub(super) fn generate_response_forwarded(&self, response: &mut Response, backend_app: &BackendApp) -> Result<()> { let headers = response.headers_mut(); remove_connection_header(headers); remove_hop_header(headers); diff --git a/rpxy-lib/src/proxy/mod.rs b/rpxy-lib/src/proxy/mod.rs index 2cc9b755..59bfc3ff 100644 --- a/rpxy-lib/src/proxy/mod.rs +++ b/rpxy-lib/src/proxy/mod.rs @@ -11,10 +11,16 @@ mod proxy_quic_s2n; use crate::{ globals::Globals, hyper_ext::rt::{LocalExecutor, TokioTimer}, + name_exp::ServerName, }; use hyper_util::server::{self, conn::auto::Builder as ConnectionBuilder}; +use rustc_hash::FxHashMap as HashMap; +use rustls::ServerConfig; use std::sync::Arc; +/// SNI to ServerConfig map type +pub type SniServerCryptoMap = HashMap>; + pub(crate) use proxy_main::Proxy; /// build connection builder shared with proxy instances diff --git a/rpxy-lib/src/proxy/proxy_main.rs b/rpxy-lib/src/proxy/proxy_main.rs index 9d3baa88..21b2c6bd 100644 --- a/rpxy-lib/src/proxy/proxy_main.rs +++ b/rpxy-lib/src/proxy/proxy_main.rs @@ -1,7 +1,6 @@ use super::socket::bind_tcp_socket; use crate::{ constants::TLS_HANDSHAKE_TIMEOUT_SEC, - crypto::{CryptoSource, ServerCrypto, SniServerCryptoMap}, error::*, globals::Globals, hyper_ext::{ @@ -20,14 +19,15 @@ use hyper::{ service::service_fn, }; use hyper_util::{client::legacy::connect::Connect, rt::TokioIo, server::conn::auto::Builder as ConnectionBuilder}; +use rpxy_certs::ServerCrypto; use std::{net::SocketAddr, sync::Arc, time::Duration}; use tokio::time::timeout; /// Wrapper function to handle request for HTTP/1.1 and HTTP/2 /// HTTP/3 is handled in proxy_h3.rs which directly calls the message handler -async fn serve_request( +async fn serve_request( req: Request, - handler: Arc>, + handler: Arc>, client_addr: SocketAddr, listen_addr: SocketAddr, tls_enabled: bool, @@ -35,7 +35,6 @@ async fn serve_request( ) -> RpxyResult> where T: Send + Sync + Connect + Clone, - U: CryptoSource + Clone, { handler .handle_request( @@ -50,10 +49,9 @@ where #[derive(Clone)] /// Proxy main object responsible to serve requests received from clients at the given socket address. -pub(crate) struct Proxy +pub(crate) struct Proxy where T: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone + Sync + Send + 'static, { /// global context shared among async tasks pub globals: Arc, @@ -64,13 +62,12 @@ where /// hyper connection builder serving http request pub connection_builder: Arc>, /// message handler serving incoming http request - pub message_handler: Arc>, + pub message_handler: Arc>, } -impl Proxy +impl Proxy where T: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone + Sync + Send + 'static, { /// Serves requests from clients fn serve_connection(&self, stream: I, peer_addr: SocketAddr, tls_server_name: Option) @@ -164,15 +161,11 @@ where let Some(mut server_crypto_rx) = self.globals.cert_reloader_rx.clone() else { return Err(RpxyError::NoCertificateReloader); }; - // TODO: newer one - let Some(mut server_crypto_rx_new) = self.globals.cert_reloader_rx_new.clone() else { - return Err(RpxyError::NoCertificateReloader); - }; let tcp_socket = bind_tcp_socket(&self.listening_on)?; let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?; info!("Start TCP proxy serving with HTTPS request for configured host names"); - let mut server_crypto_map: Option> = None; + let mut server_crypto_map: Option> = None; loop { select! { tcp_cnx = tcp_listener.accept().fuse() => { @@ -234,28 +227,16 @@ where error!("Reloader is broken"); break; } - let cert_keys_map = server_crypto_rx.borrow().clone().unwrap(); - let Some(server_crypto): Option> = (&cert_keys_map).try_into().ok() else { + let server_crypto_base = server_crypto_rx.borrow().clone().unwrap(); + let Some(server_config): Option> = (&server_crypto_base).try_into().ok() else { error!("Failed to update server crypto"); break; }; - server_crypto_map = Some(server_crypto.inner_local_map.clone()); - } - // TODO: newer one - _ = server_crypto_rx_new.changed().fuse() => { - if server_crypto_rx_new.borrow().is_none() { - error!("Reloader is broken"); - break; - } - let cert_keys_map = server_crypto_rx_new.borrow().clone().unwrap(); - // let Some(server_crypto) = cert_keys_map.try_into().ok() else { - // break; - // }; - // let Some(server_crypto): Option> = (&cert_keys_map).try_into().ok() else { - // error!("Failed to update server crypto"); - // break; - // }; - // server_crypto_map = Some(server_crypto.inner_local_map.clone()); + let map = server_config.individual_config_map.clone().iter().map(|(k,v)| { + let server_name = ServerName::from(k.as_slice()); + (server_name, v.clone()) + }).collect::>(); + server_crypto_map = Some(Arc::new(map)); } } } From 234abae5dda98a3bf0f7be562c42674f3ac6274d Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 21:39:38 +0900 Subject: [PATCH 12/16] wip: support rustls-0.23 for http3-quinn --- rpxy-bin/Cargo.toml | 4 +- rpxy-bin/src/config/parse.rs | 1 - rpxy-bin/src/config/toml.rs | 5 +- rpxy-lib/Cargo.toml | 10 +-- rpxy-lib/src/backend/backend_main.rs | 8 ++- rpxy-lib/src/error.rs | 12 ++-- rpxy-lib/src/globals.rs | 1 + .../handler_manipulate_messages.rs | 10 +-- rpxy-lib/src/proxy/proxy_h3.rs | 4 +- rpxy-lib/src/proxy/proxy_quic_quinn.rs | 61 +++++++++---------- 10 files changed, 63 insertions(+), 53 deletions(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index a8c89e65..395f33cb 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -# default = ["http3-quinn", "cache", "rustls-backend"] -default = ["cache", "rustls-backend"] +default = ["http3-quinn", "cache", "rustls-backend"] +# default = ["cache", "rustls-backend"] http3-quinn = ["rpxy-lib/http3-quinn"] http3-s2n = ["rpxy-lib/http3-s2n"] native-tls-backend = ["rpxy-lib/native-tls-backend"] diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 95c87d20..f45ca17c 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -73,7 +73,6 @@ pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, // build applications let mut app_config_list_inner = Vec::::new(); - // let mut backends = Backends::new(); for (app_name, app) in apps.0.iter() { let _server_name_string = app.server_name.as_ref().ok_or(anyhow!("No server name"))?; let registered_app_name = app_name.to_ascii_lowercase(); diff --git a/rpxy-bin/src/config/toml.rs b/rpxy-bin/src/config/toml.rs index 9b6502b3..957296cf 100644 --- a/rpxy-bin/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -230,7 +230,10 @@ impl Application { tls.https_redirection.unwrap() }; - Some(TlsConfig { https_redirection }) + Some(TlsConfig { + mutual_tls: tls.client_ca_cert_path.is_some(), + https_redirection, + }) } else { None }; diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index c597d65a..eab620b0 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["sticky-cookie", "cache", "rustls-backend"] -# default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] +# default = ["sticky-cookie", "cache", "rustls-backend"] +default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] http3-quinn = ["socket2", "quinn", "h3", "h3-quinn", "rpxy-certs/http3"] http3-s2n = [ "h3", @@ -83,11 +83,11 @@ x509-parser = "0.16.0" tracing = { version = "0.1.40" } # http/3 -quinn = { version = "0.10.2", optional = true } +quinn = { version = "0.11.1", optional = true } # h3 = { path = "../submodules/h3/h3/", optional = true } # h3-quinn = { path = "../submodules/h3/h3-quinn/", optional = true } -h3 = { version = "0.0.4", optional = true } -h3-quinn = { version = "0.0.5", optional = true } +h3 = { version = "0.0.5", optional = true } +h3-quinn = { version = "0.0.6", optional = true } s2n-quic = { version = "1.37.0", default-features = false, features = [ "provider-tls-rustls", ], optional = true } diff --git a/rpxy-lib/src/backend/backend_main.rs b/rpxy-lib/src/backend/backend_main.rs index 663120e6..0282504f 100644 --- a/rpxy-lib/src/backend/backend_main.rs +++ b/rpxy-lib/src/backend/backend_main.rs @@ -24,6 +24,9 @@ pub struct BackendApp { /// tls settings: https redirection with 30x #[builder(default)] pub https_redirection: Option, + /// tls settings: mutual TLS is enabled + #[builder(default)] + pub mutual_tls: Option, } impl<'a> BackendAppBuilder { pub fn server_name(&mut self, server_name: impl Into>) -> &mut Self { @@ -56,7 +59,10 @@ impl TryFrom<&AppConfig> for BackendApp { backend_builder.build()? } else { let tls = app_config.tls.as_ref().unwrap(); - backend_builder.https_redirection(Some(tls.https_redirection)).build()? + backend_builder + .https_redirection(Some(tls.https_redirection)) + .mutual_tls(Some(tls.mutual_tls)) + .build()? }; Ok(backend) } diff --git a/rpxy-lib/src/error.rs b/rpxy-lib/src/error.rs index 85c05f44..a775b179 100644 --- a/rpxy-lib/src/error.rs +++ b/rpxy-lib/src/error.rs @@ -16,10 +16,14 @@ pub enum RpxyError { NoServerNameInClientHello, #[error("No TLS serving app: {0}")] NoTlsServingApp(String), - #[error("Failed to update server crypto: {0}")] - FailedToUpdateServerCrypto(String), - #[error("No server crypto: {0}")] - NoServerCrypto(String), + #[error("No default crypto provider")] + NoDefaultCryptoProvider, + #[error("Failed to build server config: {0}")] + FailedToBuildServerConfig(String), + // #[error("Failed to update server crypto: {0}")] + // FailedToUpdateServerCrypto(String), + // #[error("No server crypto: {0}")] + // NoServerCrypto(String), // hyper errors #[error("hyper body manipulation error: {0}")] diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index fec0a53d..3582cdb8 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -157,5 +157,6 @@ pub struct UpstreamUri { /// Configuration parameters on TLS for a single backend application #[derive(PartialEq, Eq, Clone)] pub struct TlsConfig { + pub mutual_tls: bool, pub https_redirection: bool, } diff --git a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs index 2d77d2c2..0ee4dab3 100644 --- a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs +++ b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs @@ -29,15 +29,15 @@ where { // Manipulate ALT_SVC allowing h3 in response message only when mutual TLS is not enabled // TODO: This is a workaround for avoiding a client authentication in HTTP/3 - if self.globals.proxy_config.http3 && backend_app.crypto_source.as_ref().is_some_and(|v| !v.is_mutual_tls()) { + if self.globals.proxy_config.http3 + && backend_app.https_redirection.is_some() + && backend_app.mutual_tls.as_ref().is_some_and(|v| !v) + { if let Some(port) = self.globals.proxy_config.https_port { add_header_entry_overwrite_if_exist( headers, header::ALT_SVC.as_str(), - format!( - "h3=\":{}\"; ma={}, h3-29=\":{}\"; ma={}", - port, self.globals.proxy_config.h3_alt_svc_max_age, port, self.globals.proxy_config.h3_alt_svc_max_age - ), + format!("h3=\":{}\"; ma={}", port, self.globals.proxy_config.h3_alt_svc_max_age), )?; } } else { diff --git a/rpxy-lib/src/proxy/proxy_h3.rs b/rpxy-lib/src/proxy/proxy_h3.rs index 1e0f24f0..998b6ee8 100644 --- a/rpxy-lib/src/proxy/proxy_h3.rs +++ b/rpxy-lib/src/proxy/proxy_h3.rs @@ -1,6 +1,5 @@ use super::proxy_main::Proxy; use crate::{ - crypto::CryptoSource, error::*, hyper_ext::body::{IncomingLike, RequestBody}, log::*, @@ -17,10 +16,9 @@ use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestSt #[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream}; -impl Proxy +impl Proxy where T: Connect + Clone + Sync + Send + 'static, - U: CryptoSource + Clone + Sync + Send + 'static, { pub(super) async fn h3_serve_connection( &self, diff --git a/rpxy-lib/src/proxy/proxy_quic_quinn.rs b/rpxy-lib/src/proxy/proxy_quic_quinn.rs index 9c4bf4e7..c316ed9b 100644 --- a/rpxy-lib/src/proxy/proxy_quic_quinn.rs +++ b/rpxy-lib/src/proxy/proxy_quic_quinn.rs @@ -1,20 +1,17 @@ -use super::proxy_main::Proxy; -use super::socket::bind_udp_socket; -use crate::{ - crypto::{CryptoSource, ServerCrypto}, - error::*, - log::*, - name_exp::ByteName, -}; +use super::{proxy_main::Proxy, socket::bind_udp_socket}; +use crate::{error::*, log::*, name_exp::ByteName}; use hyper_util::client::legacy::connect::Connect; -use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig}; +use quinn::{ + crypto::rustls::{HandshakeData, QuicServerConfig}, + Endpoint, TransportConfig, +}; +use rpxy_certs::ServerCrypto; use rustls::ServerConfig; use std::sync::Arc; -impl Proxy +impl Proxy where T: Send + Sync + Connect + Clone + 'static, - U: CryptoSource + Clone + Sync + Send + 'static, { pub(super) async fn h3_listener_service(&self) -> RpxyResult<()> { let Some(mut server_crypto_rx) = self.globals.cert_reloader_rx.clone() else { @@ -22,13 +19,14 @@ where }; info!("Start UDP proxy serving with HTTP/3 request for configured host names [quinn]"); // first set as null config server - let rustls_server_config = ServerConfig::builder() - .with_safe_default_cipher_suites() - .with_safe_default_kx_groups() + // AWS LC provider by default + let provider = rustls::crypto::CryptoProvider::get_default().ok_or(RpxyError::NoDefaultCryptoProvider)?; + let rustls_server_config = ServerConfig::builder_with_provider(provider.clone()) .with_protocol_versions(&[&rustls::version::TLS13]) - .map_err(|e| RpxyError::QuinnInvalidTlsProtocolVersion(e.to_string()))? + .map_err(|e| RpxyError::FailedToBuildServerConfig(format!("TLS 1.3 server config failed: {e}")))? .with_no_client_auth() .with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new())); + let quinn_server_config_crypto = QuicServerConfig::try_from(Arc::new(rustls_server_config)).unwrap(); let mut transport_config_quic = TransportConfig::default(); transport_config_quic @@ -42,20 +40,15 @@ where .map(|v| quinn::IdleTimeout::try_from(v).unwrap()), ); - let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config)); + let mut server_config_h3 = quinn::ServerConfig::with_crypto(Arc::new(quinn_server_config_crypto)); server_config_h3.transport = Arc::new(transport_config_quic); - server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections); + server_config_h3.max_incoming(self.globals.proxy_config.h3_max_concurrent_connections as usize); // To reuse address let udp_socket = bind_udp_socket(&self.listening_on)?; - let runtime = quinn::default_runtime() - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?; - let endpoint = Endpoint::new( - quinn::EndpointConfig::default(), - Some(server_config_h3), - udp_socket, - runtime, - )?; + let runtime = + quinn::default_runtime().ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?; + let endpoint = Endpoint::new(quinn::EndpointConfig::default(), Some(server_config_h3), udp_socket, runtime)?; let mut server_crypto: Option> = None; loop { @@ -64,8 +57,10 @@ where if server_crypto.is_none() || new_conn.is_none() { continue; } - let mut conn: quinn::Connecting = new_conn.unwrap(); - let Ok(hsd) = conn.handshake_data().await else { + let Ok(mut incoming) = new_conn.unwrap().accept() else { + continue + }; + let Ok(hsd) = incoming.handshake_data().await else { continue }; @@ -84,8 +79,8 @@ where // TODO: 通常のTLSと同じenumか何かにまとめたい let self_clone = self.clone(); self.globals.runtime_handle.spawn(async move { - let client_addr = conn.remote_address(); - let quic_connection = match conn.await { + let client_addr = incoming.remote_address(); + let quic_connection = match incoming.await { Ok(new_conn) => { info!("New connection established"); h3_quinn::Connection::new(new_conn) @@ -114,8 +109,12 @@ where error!("Failed to update server crypto for h3"); break; }; - endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone()))); - + let rustls_server_config = inner.aggregated_config_no_client_auth.clone(); + let Ok(quinn_server_config_crypto) = QuicServerConfig::try_from(rustls_server_config) else { + error!("Failed to update server crypto for h3"); + break; + }; + endpoint.set_server_config(Some(quinn::ServerConfig::with_crypto(Arc::new(quinn_server_config_crypto)))); } else => break } From 53055ab0684bf5e4f5b893ef40d38b865e40cfea Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 22:35:42 +0900 Subject: [PATCH 13/16] feat: support rustls-0.23, quinn-0.11, and s2n-quic-0.38(unreleased) --- .gitmodules | 3 + CHANGELOG.md | 5 +- rpxy-bin/Cargo.toml | 4 +- rpxy-lib/Cargo.toml | 26 +- rpxy-lib/src/crypto/certs.rs | 91 ----- rpxy-lib/src/crypto/mod.rs | 36 -- rpxy-lib/src/crypto/service.rs | 251 ------------ rpxy-lib/src/error.rs | 12 +- rpxy-lib/src/proxy/proxy_quic_s2n.rs | 41 +- submodules/s2n-quic | 1 + submodules/s2n-quic-h3/Cargo.toml | 18 - submodules/s2n-quic-h3/README.md | 10 - submodules/s2n-quic-h3/src/lib.rs | 7 - submodules/s2n-quic-h3/src/s2n_quic.rs | 506 ------------------------- 14 files changed, 51 insertions(+), 960 deletions(-) delete mode 100644 rpxy-lib/src/crypto/certs.rs delete mode 100644 rpxy-lib/src/crypto/mod.rs delete mode 100644 rpxy-lib/src/crypto/service.rs create mode 160000 submodules/s2n-quic delete mode 100644 submodules/s2n-quic-h3/Cargo.toml delete mode 100644 submodules/s2n-quic-h3/README.md delete mode 100644 submodules/s2n-quic-h3/src/lib.rs delete mode 100644 submodules/s2n-quic-h3/src/s2n_quic.rs diff --git a/.gitmodules b/.gitmodules index 0d6a4041..c07680bf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/rusty-http-cache-semantics"] path = submodules/rusty-http-cache-semantics url = git@github.com:junkurihara/rusty-http-cache-semantics.git +[submodule "submodules/s2n-quic"] + path = submodules/s2n-quic + url = git@github.com:junkurihara/s2n-quic.git diff --git a/CHANGELOG.md b/CHANGELOG.md index c17283bd..c439fe9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ ## 0.8.0 (Unreleased) -## 0.7.1 -- 0.7.3 +- Breaking: Support for `rustls`-0.23.x for http/1.1, 2 and 3. No configuration update is needed at this point. +- Breaking: Along with `rustls`, the cert manager was split from `rpxy-lib` and moved to a new inner crate `rpxy-cert`. This change is to make the cert manager reusable for other projects and to support not only static file based certificates but also other types, e.g., dynamic fetching and management via ACME, in the future. + +## 0.7.1 -- 0.7.2 - deps and patches diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 395f33cb..d0fdbf2b 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["http3-quinn", "cache", "rustls-backend"] -# default = ["cache", "rustls-backend"] +# default = ["http3-quinn", "cache", "rustls-backend"] +default = ["http3-s2n", "cache", "rustls-backend"] http3-quinn = ["rpxy-lib/http3-quinn"] http3-s2n = ["rpxy-lib/http3-s2n"] native-tls-backend = ["rpxy-lib/native-tls-backend"] diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index eab620b0..318b4713 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -# default = ["sticky-cookie", "cache", "rustls-backend"] -default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] +default = ["http3-s2n", "sticky-cookie", "cache", "rustls-backend"] +# default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] http3-quinn = ["socket2", "quinn", "h3", "h3-quinn", "rpxy-certs/http3"] http3-s2n = [ "h3", @@ -76,24 +76,28 @@ rpxy-certs = { path = "../rpxy-certs/", default-features = false } hot_reload = "0.1.5" rustls = { version = "0.23.8", default-features = false } tokio-rustls = { version = "0.26.0", features = ["early-data"] } -webpki = "0.22.4" -x509-parser = "0.16.0" # logging tracing = { version = "0.1.40" } # http/3 quinn = { version = "0.11.1", optional = true } -# h3 = { path = "../submodules/h3/h3/", optional = true } -# h3-quinn = { path = "../submodules/h3/h3-quinn/", optional = true } h3 = { version = "0.0.5", optional = true } h3-quinn = { version = "0.0.6", optional = true } -s2n-quic = { version = "1.37.0", default-features = false, features = [ +### TODO: workaround for s2n-quic, waiting for release of s2n-quic-0.38.0 +s2n-quic = { path = "../submodules/s2n-quic/quic/s2n-quic", optional = true, default-features = false, features = [ "provider-tls-rustls", -], optional = true } -s2n-quic-core = { version = "0.37.0", default-features = false, optional = true } -s2n-quic-h3 = { path = "../submodules/s2n-quic-h3/", optional = true } -s2n-quic-rustls = { version = "0.37.0", optional = true } +] } +s2n-quic-core = { path = "../submodules/s2n-quic/quic/s2n-quic-core", optional = true, default-features = false } +s2n-quic-rustls = { path = "../submodules/s2n-quic/quic/s2n-quic-rustls", optional = true } +s2n-quic-h3 = { path = "../submodules/s2n-quic/quic/s2n-quic-h3", optional = true } +# s2n-quic = { version = "1.37.0", default-features = false, features = [ +# "provider-tls-rustls", +# ], optional = true } +# s2n-quic-core = { version = "0.37.0", default-features = false, optional = true } +# s2n-quic-h3 = { path = "../submodules/s2n-quic-h3/", optional = true } +# s2n-quic-rustls = { version = "0.37.0", optional = true } +########## # for UDP socket wit SO_REUSEADDR when h3 with quinn socket2 = { version = "0.5.7", features = ["all"], optional = true } diff --git a/rpxy-lib/src/crypto/certs.rs b/rpxy-lib/src/crypto/certs.rs deleted file mode 100644 index c9cfafd5..00000000 --- a/rpxy-lib/src/crypto/certs.rs +++ /dev/null @@ -1,91 +0,0 @@ -use async_trait::async_trait; -use rustc_hash::FxHashSet as HashSet; -use rustls::{ - sign::{any_supported_type, CertifiedKey}, - Certificate, OwnedTrustAnchor, PrivateKey, -}; -use std::io; -use x509_parser::prelude::*; - -#[async_trait] -// Trait to read certs and keys anywhere from KVS, file, sqlite, etc. -pub trait CryptoSource { - type Error; - - /// read crypto materials from source - async fn read(&self) -> Result; - - /// Returns true when mutual tls is enabled - fn is_mutual_tls(&self) -> bool; -} - -/// Certificates and private keys in rustls loaded from files -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct CertsAndKeys { - pub certs: Vec, - pub cert_keys: Vec, - pub client_ca_certs: Option>, -} - -impl CertsAndKeys { - pub fn parse_server_certs_and_keys(&self) -> Result { - // for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { - let signing_key = self - .cert_keys - .iter() - .find_map(|k| { - if let Ok(sk) = any_supported_type(k) { - Some(sk) - } else { - None - } - }) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Unable to find a valid certificate and key", - ) - })?; - Ok(CertifiedKey::new(self.certs.clone(), signing_key)) - } - - pub fn parse_client_ca_certs(&self) -> Result<(Vec, HashSet>), anyhow::Error> { - let certs = self.client_ca_certs.as_ref().ok_or(anyhow::anyhow!("No client cert"))?; - - let owned_trust_anchors: Vec<_> = certs - .iter() - .map(|v| { - // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( - trust_anchor.subject, - trust_anchor.spki, - trust_anchor.name_constraints, - ) - }) - .collect(); - - // TODO: SKID is not used currently - let subject_key_identifiers: HashSet<_> = certs - .iter() - .filter_map(|v| { - // retrieve ca key id (subject key id) - let cert = parse_x509_certificate(&v.0).unwrap().1; - let subject_key_ids = cert - .iter_extensions() - .filter_map(|ext| match ext.parsed_extension() { - ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), - _ => None, - }) - .collect::>(); - if !subject_key_ids.is_empty() { - Some(subject_key_ids[0].0.to_owned()) - } else { - None - } - }) - .collect(); - - Ok((owned_trust_anchors, subject_key_identifiers)) - } -} diff --git a/rpxy-lib/src/crypto/mod.rs b/rpxy-lib/src/crypto/mod.rs deleted file mode 100644 index 7b8935c9..00000000 --- a/rpxy-lib/src/crypto/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -mod certs; -mod service; - -use crate::{ - backend::BackendAppManager, - constants::{CERTS_WATCH_DELAY_SECS, LOAD_CERTS_ONLY_WHEN_UPDATED}, - error::RpxyResult, -}; -use hot_reload::{ReloaderReceiver, ReloaderService}; -use service::CryptoReloader; -use std::sync::Arc; - -pub use certs::{CertsAndKeys, CryptoSource}; -pub use service::{ServerCrypto, ServerCryptoBase, SniServerCryptoMap}; - -/// Result type inner of certificate reloader service -type ReloaderServiceResultInner = ( - ReloaderService, ServerCryptoBase>, - ReloaderReceiver, -); -/// Build certificate reloader service -pub(crate) async fn build_cert_reloader( - app_manager: &Arc>, -) -> RpxyResult> -where - T: CryptoSource + Clone + Send + Sync + 'static, -{ - let (cert_reloader_service, cert_reloader_rx) = ReloaderService::< - service::CryptoReloader, - service::ServerCryptoBase, - >::new( - app_manager, CERTS_WATCH_DELAY_SECS, !LOAD_CERTS_ONLY_WHEN_UPDATED - ) - .await?; - Ok((cert_reloader_service, cert_reloader_rx)) -} diff --git a/rpxy-lib/src/crypto/service.rs b/rpxy-lib/src/crypto/service.rs deleted file mode 100644 index cd198cb7..00000000 --- a/rpxy-lib/src/crypto/service.rs +++ /dev/null @@ -1,251 +0,0 @@ -use super::certs::{CertsAndKeys, CryptoSource}; -use crate::{backend::BackendAppManager, log::*, name_exp::ServerName}; -use async_trait::async_trait; -use hot_reload::*; -use rustc_hash::FxHashMap as HashMap; -use rustls::{server::ResolvesServerCertUsingSni, sign::CertifiedKey, RootCertStore, ServerConfig}; -use std::sync::Arc; - -#[derive(Clone)] -/// Reloader service for certificates and keys for TLS -pub struct CryptoReloader -where - T: CryptoSource, -{ - inner: Arc>, -} - -/// SNI to ServerConfig map type -pub type SniServerCryptoMap = HashMap>; -/// SNI to ServerConfig map -pub struct ServerCrypto { - // For Quic/HTTP3, only servers with no client authentication - #[cfg(feature = "http3-quinn")] - pub inner_global_no_client_auth: Arc, - #[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] - pub inner_global_no_client_auth: s2n_quic_rustls::Server, - // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers - pub inner_local_map: Arc, -} - -/// Reloader target for the certificate reloader service -#[derive(Debug, PartialEq, Eq, Clone, Default)] -pub struct ServerCryptoBase { - inner: HashMap, -} - -#[async_trait] -impl Reload for CryptoReloader -where - T: CryptoSource + Sync + Send, -{ - type Source = Arc>; - async fn new(source: &Self::Source) -> Result> { - Ok(Self { inner: source.clone() }) - } - - async fn reload(&self) -> Result, ReloaderError> { - let mut certs_and_keys_map = ServerCryptoBase::default(); - - for (server_name_bytes_exp, backend) in self.inner.apps.iter() { - if let Some(crypto_source) = &backend.crypto_source { - let certs_and_keys = crypto_source - .read() - .await - .map_err(|_e| ReloaderError::::Reload("Failed to reload cert, key or ca cert"))?; - certs_and_keys_map - .inner - .insert(server_name_bytes_exp.to_owned(), certs_and_keys); - } - } - - Ok(Some(certs_and_keys_map)) - } -} - -impl TryInto> for &ServerCryptoBase { - type Error = anyhow::Error; - - fn try_into(self) -> Result, Self::Error> { - #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] - let server_crypto_global = self.build_server_crypto_global()?; - let server_crypto_local_map: SniServerCryptoMap = self.build_server_crypto_local_map()?; - - Ok(Arc::new(ServerCrypto { - #[cfg(feature = "http3-quinn")] - inner_global_no_client_auth: Arc::new(server_crypto_global), - #[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] - inner_global_no_client_auth: server_crypto_global, - inner_local_map: Arc::new(server_crypto_local_map), - })) - } -} - -impl ServerCryptoBase { - fn build_server_crypto_local_map(&self) -> Result> { - let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default(); - - for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { - let server_name: String = server_name_bytes_exp.try_into()?; - - // Parse server certificates and private keys - let Ok(certified_key): Result = certs_and_keys.parse_server_certs_and_keys() else { - warn!("Failed to add certificate for {}", server_name); - continue; - }; - - let mut resolver_local = ResolvesServerCertUsingSni::new(); - let mut client_ca_roots_local = RootCertStore::empty(); - - // add server certificate and key - if let Err(e) = resolver_local.add(server_name.as_str(), certified_key.to_owned()) { - error!("{}: Failed to read some certificates and keys {}", server_name.as_str(), e) - } - - // add client certificate if specified - if certs_and_keys.client_ca_certs.is_some() { - // add client certificate if specified - match certs_and_keys.parse_client_ca_certs() { - Ok((owned_trust_anchors, _subject_key_ids)) => { - client_ca_roots_local.add_trust_anchors(owned_trust_anchors.into_iter()); - } - Err(e) => { - warn!("Failed to add client CA certificate for {}: {}", server_name.as_str(), e); - } - } - } - - let mut server_config_local = if client_ca_roots_local.is_empty() { - // with no client auth, enable http1.1 -- 3 - #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))] - { - ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_local)) - } - #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] - { - let mut sc = ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_local)); - sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later? - sc - } - } else { - // with client auth, enable only http1.1 and 2 - // let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots); - let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local); - ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(client_certs_verifier)) - .with_cert_resolver(Arc::new(resolver_local)) - }; - server_config_local.alpn_protocols.push(b"h2".to_vec()); - server_config_local.alpn_protocols.push(b"http/1.1".to_vec()); - - server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local)); - } - Ok(server_crypto_local_map) - } - - #[cfg(feature = "http3-quinn")] - fn build_server_crypto_global(&self) -> Result> { - let mut resolver_global = ResolvesServerCertUsingSni::new(); - - for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { - let server_name: String = server_name_bytes_exp.try_into()?; - - // Parse server certificates and private keys - let Ok(certified_key): Result = certs_and_keys.parse_server_certs_and_keys() else { - warn!("Failed to add certificate for {}", server_name); - continue; - }; - - if certs_and_keys.client_ca_certs.is_none() { - // aggregated server config for no client auth server for http3 - if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { - error!("{}: Failed to read some certificates and keys {}", server_name.as_str(), e) - } - } - } - - ////////////// - let mut server_crypto_global = ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_global)); - - ////////////////////////////// - - server_crypto_global.alpn_protocols = vec![ - b"h3".to_vec(), - b"hq-29".to_vec(), // TODO: remove later? - b"h2".to_vec(), - b"http/1.1".to_vec(), - ]; - Ok(server_crypto_global) - } - - #[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] - fn build_server_crypto_global(&self) -> Result> { - let mut resolver_global = s2n_quic_rustls::rustls::server::ResolvesServerCertUsingSni::new(); - - for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { - let server_name: String = server_name_bytes_exp.try_into()?; - - // Parse server certificates and private keys - let Ok(certified_key) = parse_server_certs_and_keys_s2n(certs_and_keys) else { - warn!("Failed to add certificate for {}", server_name); - continue; - }; - - if certs_and_keys.client_ca_certs.is_none() { - // aggregated server config for no client auth server for http3 - if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { - error!("{}: Failed to read some certificates and keys {}", server_name.as_str(), e) - } - } - } - let alpn = [ - b"h3".to_vec(), - b"hq-29".to_vec(), // TODO: remove later? - b"h2".to_vec(), - b"http/1.1".to_vec(), - ]; - let server_crypto_global = s2n_quic::provider::tls::rustls::Server::builder() - .with_cert_resolver(Arc::new(resolver_global)) - .map_err(|e| anyhow::anyhow!(e))? - .with_application_protocols(alpn.iter()) - .map_err(|e| anyhow::anyhow!(e))? - .build() - .map_err(|e| anyhow::anyhow!(e))?; - Ok(server_crypto_global) - } -} - -#[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] -/// This is workaround for the version difference between rustls and s2n-quic-rustls -fn parse_server_certs_and_keys_s2n( - certs_and_keys: &CertsAndKeys, -) -> Result { - let signing_key = certs_and_keys - .cert_keys - .iter() - .find_map(|k| { - let s2n_private_key = s2n_quic_rustls::PrivateKey(k.0.clone()); - if let Ok(sk) = s2n_quic_rustls::rustls::sign::any_supported_type(&s2n_private_key) { - Some(sk) - } else { - None - } - }) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "Unable to find a valid certificate and key"))?; - let certs: Vec<_> = certs_and_keys - .certs - .iter() - .map(|c| s2n_quic_rustls::rustls::Certificate(c.0.clone())) - .collect(); - Ok(s2n_quic_rustls::rustls::sign::CertifiedKey::new(certs, signing_key)) -} diff --git a/rpxy-lib/src/error.rs b/rpxy-lib/src/error.rs index a775b179..0b7741f6 100644 --- a/rpxy-lib/src/error.rs +++ b/rpxy-lib/src/error.rs @@ -20,10 +20,10 @@ pub enum RpxyError { NoDefaultCryptoProvider, #[error("Failed to build server config: {0}")] FailedToBuildServerConfig(String), - // #[error("Failed to update server crypto: {0}")] - // FailedToUpdateServerCrypto(String), - // #[error("No server crypto: {0}")] - // NoServerCrypto(String), + #[error("Failed to update server crypto: {0}")] + FailedToUpdateServerCrypto(String), + #[error("No server crypto: {0}")] + NoServerCrypto(String), // hyper errors #[error("hyper body manipulation error: {0}")] @@ -63,8 +63,8 @@ pub enum RpxyError { // certificate reloader errors #[error("No certificate reloader when building a proxy for TLS")] NoCertificateReloader, - // #[error("Certificate reload error: {0}")] - // CertificateReloadError(#[from] hot_reload::ReloaderError), + #[error("Certificate reload error: {0}")] + CertificateReloadError(#[from] hot_reload::ReloaderError), // backend errors #[error("Invalid reverse proxy setting")] diff --git a/rpxy-lib/src/proxy/proxy_quic_s2n.rs b/rpxy-lib/src/proxy/proxy_quic_s2n.rs index 13a88027..0fd1c7c6 100644 --- a/rpxy-lib/src/proxy/proxy_quic_s2n.rs +++ b/rpxy-lib/src/proxy/proxy_quic_s2n.rs @@ -1,21 +1,15 @@ use super::proxy_main::Proxy; -use crate::{ - crypto::CryptoSource, - crypto::{ServerCrypto, ServerCryptoBase}, - error::*, - log::*, - name_exp::ByteName, -}; +use crate::{error::*, log::*, name_exp::ByteName}; use anyhow::anyhow; use hot_reload::ReloaderReceiver; use hyper_util::client::legacy::connect::Connect; +use rpxy_certs::{ServerCrypto, ServerCryptoBase}; use s2n_quic::provider; use std::sync::Arc; -impl Proxy +impl Proxy where T: Connect + Clone + Sync + Send + 'static, - U: CryptoSource + Clone + Sync + Send + 'static, { /// Start UDP proxy serving with HTTP/3 request for configured host names pub(super) async fn h3_listener_service(&self) -> RpxyResult<()> { @@ -25,7 +19,7 @@ where info!("Start UDP proxy serving with HTTP/3 request for configured host names [s2n-quic]"); // initially wait for receipt - let mut server_crypto: Option> = { + let mut server_crypto: Option = { let _ = server_crypto_rx.changed().await; let sc = self.receive_server_crypto(server_crypto_rx.clone())?; Some(sc) @@ -57,16 +51,24 @@ where } /// Receive server crypto from reloader - fn receive_server_crypto( - &self, - server_crypto_rx: ReloaderReceiver, - ) -> RpxyResult> { + fn receive_server_crypto(&self, server_crypto_rx: ReloaderReceiver) -> RpxyResult { let cert_keys_map = server_crypto_rx.borrow().clone().ok_or_else(|| { error!("Reloader is broken"); RpxyError::CertificateReloadError(anyhow!("Reloader is broken").into()) })?; - let server_crypto: Option> = (&cert_keys_map).try_into().ok(); + let server_crypto: Option = (&cert_keys_map).try_into().ok().and_then(|v: Arc| { + let rustls_server_config = v.aggregated_config_no_client_auth.clone(); + let resolver = rustls_server_config.cert_resolver.clone(); + let alpn = rustls_server_config.alpn_protocols.clone(); + #[allow(deprecated)] + let tls = provider::tls::rustls::server::Builder::default() + .with_cert_resolver(resolver) + .and_then(|t| t.with_application_protocols(alpn.iter())) + .and_then(|t| t.build()) + .ok(); + tls + }); server_crypto.ok_or_else(|| { error!("Failed to update server crypto for h3 [s2n-quic]"); RpxyError::FailedToUpdateServerCrypto("Failed to update server crypto for h3 [s2n-quic]".to_string()) @@ -74,7 +76,7 @@ where } /// Event loop for UDP proxy serving with HTTP/3 request for configured host names - async fn h3_listener_service_inner(&self, server_crypto: &Option>) -> RpxyResult<()> { + async fn h3_listener_service_inner(&self, server_crypto: &Option) -> RpxyResult<()> { // setup UDP socket let io = provider::io::tokio::Builder::default() .with_receive_address(self.listening_on)? @@ -97,14 +99,11 @@ where // setup tls let Some(server_crypto) = server_crypto else { warn!("No server crypto is given [s2n-quic]"); - return Err(RpxyError::NoServerCrypto( - "No server crypto is given [s2n-quic]".to_string(), - )); + return Err(RpxyError::NoServerCrypto("No server crypto is given [s2n-quic]".to_string())); }; - let tls = server_crypto.inner_global_no_client_auth.clone(); let mut server = s2n_quic::Server::builder() - .with_tls(tls)? + .with_tls(server_crypto.to_owned())? .with_io(io)? .with_limits(limits)? .start()?; diff --git a/submodules/s2n-quic b/submodules/s2n-quic new file mode 160000 index 00000000..d03cc470 --- /dev/null +++ b/submodules/s2n-quic @@ -0,0 +1 @@ +Subproject commit d03cc470fa9812d06d204e312e4ada00079e96df diff --git a/submodules/s2n-quic-h3/Cargo.toml b/submodules/s2n-quic-h3/Cargo.toml deleted file mode 100644 index 3c891a81..00000000 --- a/submodules/s2n-quic-h3/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "s2n-quic-h3" -# this in an unpublished internal crate so the version should not be changed -version = "0.1.0" -authors = ["AWS s2n"] -edition = "2021" -rust-version = "1.63" -license = "Apache-2.0" -# this contains an http3 implementation for testing purposes and should not be published -publish = false - -[dependencies] -bytes = { version = "1", default-features = false } -futures = { version = "0.3", default-features = false } -# h3 = { path = "../h3/h3/" } -h3 = { version = "0.0.4" } -s2n-quic = "1.37.0" -s2n-quic-core = "0.37.0" diff --git a/submodules/s2n-quic-h3/README.md b/submodules/s2n-quic-h3/README.md deleted file mode 100644 index aed94754..00000000 --- a/submodules/s2n-quic-h3/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# s2n-quic-h3 - -This is an internal crate used by [s2n-quic](https://github.com/aws/s2n-quic) written as a proof of concept for implementing HTTP3 on top of s2n-quic. The API is not currently stable and should not be used directly. - -## License - -This project is licensed under the [Apache-2.0 License][license-url]. - -[license-badge]: https://img.shields.io/badge/license-apache-blue.svg -[license-url]: https://aws.amazon.com/apache-2-0/ diff --git a/submodules/s2n-quic-h3/src/lib.rs b/submodules/s2n-quic-h3/src/lib.rs deleted file mode 100644 index c85f197f..00000000 --- a/submodules/s2n-quic-h3/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -mod s2n_quic; - -pub use self::s2n_quic::*; -pub use h3; diff --git a/submodules/s2n-quic-h3/src/s2n_quic.rs b/submodules/s2n-quic-h3/src/s2n_quic.rs deleted file mode 100644 index dffa19b2..00000000 --- a/submodules/s2n-quic-h3/src/s2n_quic.rs +++ /dev/null @@ -1,506 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -use bytes::{Buf, Bytes}; -use futures::ready; -use h3::quic::{self, Error, StreamId, WriteBuf}; -use s2n_quic::stream::{BidirectionalStream, ReceiveStream}; -use s2n_quic_core::varint::VarInt; -use std::{ - convert::TryInto, - fmt::{self, Display}, - sync::Arc, - task::{self, Poll}, -}; - -pub struct Connection { - conn: s2n_quic::connection::Handle, - bidi_acceptor: s2n_quic::connection::BidirectionalStreamAcceptor, - recv_acceptor: s2n_quic::connection::ReceiveStreamAcceptor, -} - -impl Connection { - pub fn new(new_conn: s2n_quic::Connection) -> Self { - let (handle, acceptor) = new_conn.split(); - let (bidi, recv) = acceptor.split(); - - Self { - conn: handle, - bidi_acceptor: bidi, - recv_acceptor: recv, - } - } -} - -#[derive(Debug)] -pub struct ConnectionError(s2n_quic::connection::Error); - -impl std::error::Error for ConnectionError {} - -impl fmt::Display for ConnectionError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl Error for ConnectionError { - fn is_timeout(&self) -> bool { - matches!(self.0, s2n_quic::connection::Error::IdleTimerExpired { .. }) - } - - fn err_code(&self) -> Option { - match self.0 { - s2n_quic::connection::Error::Application { error, .. } => Some(error.into()), - _ => None, - } - } -} - -impl From for ConnectionError { - fn from(e: s2n_quic::connection::Error) -> Self { - Self(e) - } -} - -impl quic::Connection for Connection -where - B: Buf, -{ - type BidiStream = BidiStream; - type SendStream = SendStream; - type RecvStream = RecvStream; - type OpenStreams = OpenStreams; - type Error = ConnectionError; - - fn poll_accept_recv( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll, Self::Error>> { - let recv = match ready!(self.recv_acceptor.poll_accept_receive_stream(cx))? { - Some(x) => x, - None => return Poll::Ready(Ok(None)), - }; - Poll::Ready(Ok(Some(Self::RecvStream::new(recv)))) - } - - fn poll_accept_bidi( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll, Self::Error>> { - let (recv, send) = match ready!(self.bidi_acceptor.poll_accept_bidirectional_stream(cx))? { - Some(x) => x.split(), - None => return Poll::Ready(Ok(None)), - }; - Poll::Ready(Ok(Some(Self::BidiStream { - send: Self::SendStream::new(send), - recv: Self::RecvStream::new(recv), - }))) - } - - fn poll_open_bidi( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll> { - let stream = ready!(self.conn.poll_open_bidirectional_stream(cx))?; - Ok(stream.into()).into() - } - - fn poll_open_send( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll> { - let stream = ready!(self.conn.poll_open_send_stream(cx))?; - Ok(stream.into()).into() - } - - fn opener(&self) -> Self::OpenStreams { - OpenStreams { - conn: self.conn.clone(), - } - } - - fn close(&mut self, code: h3::error::Code, _reason: &[u8]) { - self.conn.close( - code.value() - .try_into() - .expect("s2n-quic supports error codes up to 2^62-1"), - ); - } -} - -pub struct OpenStreams { - conn: s2n_quic::connection::Handle, -} - -impl quic::OpenStreams for OpenStreams -where - B: Buf, -{ - type BidiStream = BidiStream; - type SendStream = SendStream; - type RecvStream = RecvStream; - type Error = ConnectionError; - - fn poll_open_bidi( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll> { - let stream = ready!(self.conn.poll_open_bidirectional_stream(cx))?; - Ok(stream.into()).into() - } - - fn poll_open_send( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll> { - let stream = ready!(self.conn.poll_open_send_stream(cx))?; - Ok(stream.into()).into() - } - - fn close(&mut self, code: h3::error::Code, _reason: &[u8]) { - self.conn.close( - code.value() - .try_into() - .unwrap_or_else(|_| VarInt::MAX.into()), - ); - } -} - -impl Clone for OpenStreams { - fn clone(&self) -> Self { - Self { - conn: self.conn.clone(), - } - } -} - -pub struct BidiStream -where - B: Buf, -{ - send: SendStream, - recv: RecvStream, -} - -impl quic::BidiStream for BidiStream -where - B: Buf, -{ - type SendStream = SendStream; - type RecvStream = RecvStream; - - fn split(self) -> (Self::SendStream, Self::RecvStream) { - (self.send, self.recv) - } -} - -impl quic::RecvStream for BidiStream -where - B: Buf, -{ - type Buf = Bytes; - type Error = ReadError; - - fn poll_data( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll, Self::Error>> { - self.recv.poll_data(cx) - } - - fn stop_sending(&mut self, error_code: u64) { - self.recv.stop_sending(error_code) - } - - fn recv_id(&self) -> StreamId { - self.recv.stream.id().try_into().expect("invalid stream id") - } -} - -impl quic::SendStream for BidiStream -where - B: Buf, -{ - type Error = SendStreamError; - - fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { - self.send.poll_ready(cx) - } - - fn poll_finish(&mut self, cx: &mut task::Context<'_>) -> Poll> { - self.send.poll_finish(cx) - } - - fn reset(&mut self, reset_code: u64) { - self.send.reset(reset_code) - } - - fn send_data>>(&mut self, data: D) -> Result<(), Self::Error> { - self.send.send_data(data) - } - - fn send_id(&self) -> StreamId { - self.send.stream.id().try_into().expect("invalid stream id") - } -} - -impl From for BidiStream -where - B: Buf, -{ - fn from(bidi: BidirectionalStream) -> Self { - let (recv, send) = bidi.split(); - BidiStream { - send: send.into(), - recv: recv.into(), - } - } -} - -pub struct RecvStream { - stream: s2n_quic::stream::ReceiveStream, -} - -impl RecvStream { - fn new(stream: s2n_quic::stream::ReceiveStream) -> Self { - Self { stream } - } -} - -impl quic::RecvStream for RecvStream { - type Buf = Bytes; - type Error = ReadError; - - fn poll_data( - &mut self, - cx: &mut task::Context<'_>, - ) -> Poll, Self::Error>> { - let buf = ready!(self.stream.poll_receive(cx))?; - Ok(buf).into() - } - - fn stop_sending(&mut self, error_code: u64) { - let _ = self.stream.stop_sending( - s2n_quic::application::Error::new(error_code) - .expect("s2n-quic supports error codes up to 2^62-1"), - ); - } - - fn recv_id(&self) -> StreamId { - self.stream.id().try_into().expect("invalid stream id") - } -} - -impl From for RecvStream { - fn from(recv: ReceiveStream) -> Self { - RecvStream::new(recv) - } -} - -#[derive(Debug)] -pub struct ReadError(s2n_quic::stream::Error); - -impl std::error::Error for ReadError {} - -impl fmt::Display for ReadError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl From for Arc { - fn from(e: ReadError) -> Self { - Arc::new(e) - } -} - -impl From for ReadError { - fn from(e: s2n_quic::stream::Error) -> Self { - Self(e) - } -} - -impl Error for ReadError { - fn is_timeout(&self) -> bool { - matches!( - self.0, - s2n_quic::stream::Error::ConnectionError { - error: s2n_quic::connection::Error::IdleTimerExpired { .. }, - .. - } - ) - } - - fn err_code(&self) -> Option { - match self.0 { - s2n_quic::stream::Error::ConnectionError { - error: s2n_quic::connection::Error::Application { error, .. }, - .. - } => Some(error.into()), - s2n_quic::stream::Error::StreamReset { error, .. } => Some(error.into()), - _ => None, - } - } -} - -pub struct SendStream { - stream: s2n_quic::stream::SendStream, - chunk: Option, - buf: Option>, // TODO: Replace with buf: PhantomData - // after https://github.com/hyperium/h3/issues/78 is resolved -} - -impl SendStream -where - B: Buf, -{ - fn new(stream: s2n_quic::stream::SendStream) -> SendStream { - Self { - stream, - chunk: None, - buf: Default::default(), - } - } -} - -impl quic::SendStream for SendStream -where - B: Buf, -{ - type Error = SendStreamError; - - fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { - loop { - // try to flush the current chunk if we have one - if let Some(chunk) = self.chunk.as_mut() { - ready!(self.stream.poll_send(chunk, cx))?; - - // s2n-quic will take the whole chunk on send, even if it exceeds the limits - debug_assert!(chunk.is_empty()); - self.chunk = None; - } - - // try to take the next chunk from the WriteBuf - if let Some(ref mut data) = self.buf { - let len = data.chunk().len(); - - // if the write buf is empty, then clear it and break - if len == 0 { - self.buf = None; - break; - } - - // copy the first chunk from WriteBuf and prepare it to flush - let chunk = data.copy_to_bytes(len); - self.chunk = Some(chunk); - - // loop back around to flush the chunk - continue; - } - - // if we didn't have either a chunk or WriteBuf, then we're ready - break; - } - - Poll::Ready(Ok(())) - - // TODO: Replace with following after https://github.com/hyperium/h3/issues/78 is resolved - // self.available_bytes = ready!(self.stream.poll_send_ready(cx))?; - // Poll::Ready(Ok(())) - } - - fn send_data>>(&mut self, data: D) -> Result<(), Self::Error> { - if self.buf.is_some() { - return Err(Self::Error::NotReady); - } - self.buf = Some(data.into()); - Ok(()) - - // TODO: Replace with following after https://github.com/hyperium/h3/issues/78 is resolved - // let mut data = data.into(); - // while self.available_bytes > 0 && data.has_remaining() { - // let len = data.chunk().len(); - // let chunk = data.copy_to_bytes(len); - // self.stream.send_data(chunk)?; - // self.available_bytes = self.available_bytes.saturating_sub(len); - // } - // Ok(()) - } - - fn poll_finish(&mut self, cx: &mut task::Context<'_>) -> Poll> { - // ensure all chunks are flushed to the QUIC stream before finishing - ready!(self.poll_ready(cx))?; - self.stream.finish()?; - Ok(()).into() - } - - fn reset(&mut self, reset_code: u64) { - let _ = self - .stream - .reset(reset_code.try_into().unwrap_or_else(|_| VarInt::MAX.into())); - } - - fn send_id(&self) -> StreamId { - self.stream.id().try_into().expect("invalid stream id") - } -} - -impl From for SendStream -where - B: Buf, -{ - fn from(send: s2n_quic::stream::SendStream) -> Self { - SendStream::new(send) - } -} - -#[derive(Debug)] -pub enum SendStreamError { - Write(s2n_quic::stream::Error), - NotReady, -} - -impl std::error::Error for SendStreamError {} - -impl Display for SendStreamError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } -} - -impl From for SendStreamError { - fn from(e: s2n_quic::stream::Error) -> Self { - Self::Write(e) - } -} - -impl Error for SendStreamError { - fn is_timeout(&self) -> bool { - matches!( - self, - Self::Write(s2n_quic::stream::Error::ConnectionError { - error: s2n_quic::connection::Error::IdleTimerExpired { .. }, - .. - }) - ) - } - - fn err_code(&self) -> Option { - match self { - Self::Write(s2n_quic::stream::Error::StreamReset { error, .. }) => { - Some((*error).into()) - } - Self::Write(s2n_quic::stream::Error::ConnectionError { - error: s2n_quic::connection::Error::Application { error, .. }, - .. - }) => Some((*error).into()), - _ => None, - } - } -} - -impl From for Arc { - fn from(e: SendStreamError) -> Self { - Arc::new(e) - } -} From 079ebdea4057bbbe01dd029993d4ad40c0c38f5f Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 22:49:32 +0900 Subject: [PATCH 14/16] chore: deps and docs --- CHANGELOG.md | 9 ++++++++- rpxy-bin/Cargo.toml | 6 +++--- rpxy-lib/Cargo.toml | 8 ++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c439fe9e..6f10b48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,17 @@ ## 0.8.0 (Unreleased) +### Important Changes + - Breaking: Support for `rustls`-0.23.x for http/1.1, 2 and 3. No configuration update is needed at this point. - Breaking: Along with `rustls`, the cert manager was split from `rpxy-lib` and moved to a new inner crate `rpxy-cert`. This change is to make the cert manager reusable for other projects and to support not only static file based certificates but also other types, e.g., dynamic fetching and management via ACME, in the future. -## 0.7.1 -- 0.7.2 +### Improvement + +- Refactor: lots of minor improvements + + +## 0.7.1 - deps and patches diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index d0fdbf2b..6cbbb488 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -# default = ["http3-quinn", "cache", "rustls-backend"] -default = ["http3-s2n", "cache", "rustls-backend"] +default = ["http3-quinn", "cache", "rustls-backend"] +# default = ["http3-s2n", "cache", "rustls-backend"] http3-quinn = ["rpxy-lib/http3-quinn"] http3-s2n = ["rpxy-lib/http3-s2n"] native-tls-backend = ["rpxy-lib/native-tls-backend"] @@ -30,7 +30,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [ mimalloc = { version = "*", default-features = false } anyhow = "1.0.86" rustc-hash = "1.1.0" -serde = { version = "1.0.202", default-features = false, features = ["derive"] } +serde = { version = "1.0.203", default-features = false, features = ["derive"] } tokio = { version = "1.37.0", default-features = false, features = [ "net", "rt-multi-thread", diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index 318b4713..a746052d 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -13,8 +13,8 @@ publish.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["http3-s2n", "sticky-cookie", "cache", "rustls-backend"] -# default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] +# default = ["http3-s2n", "sticky-cookie", "cache", "rustls-backend"] +default = ["http3-quinn", "sticky-cookie", "cache", "rustls-backend"] http3-quinn = ["socket2", "quinn", "h3", "h3-quinn", "rpxy-certs/http3"] http3-s2n = [ "h3", @@ -55,7 +55,7 @@ thiserror = "1.0.61" http = "1.1.0" http-body-util = "0.1.1" hyper = { version = "1.3.1", default-features = false } -hyper-util = { version = "0.1.3", features = ["full"] } +hyper-util = { version = "0.1.5", features = ["full"] } futures-util = { version = "0.3.30", default-features = false } futures-channel = { version = "0.3.30", default-features = false } @@ -64,7 +64,7 @@ hyper-tls = { version = "0.6.0", features = [ "alpn", "vendored", ], optional = true } -hyper-rustls = { version = "0.27.1", default-features = false, features = [ +hyper-rustls = { version = "0.27.2", default-features = false, features = [ "ring", "native-tokio", "http1", From c8e6a1b65a1f73e585d77f26091b3f92d89b7081 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 29 May 2024 09:50:04 +0900 Subject: [PATCH 15/16] chore: submodule --- submodules/s2n-quic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/s2n-quic b/submodules/s2n-quic index d03cc470..a3e8d34d 160000 --- a/submodules/s2n-quic +++ b/submodules/s2n-quic @@ -1 +1 @@ -Subproject commit d03cc470fa9812d06d204e312e4ada00079e96df +Subproject commit a3e8d34d74aa653acb53eff53781796db2fd6e39 From 84223827f2797670de1ef3c0c7dea6732e1f6c47 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 2 Jun 2024 02:51:57 +0900 Subject: [PATCH 16/16] chore: deps and bump version unreleased --- Cargo.toml | 2 +- rpxy-bin/Cargo.toml | 2 +- rpxy-certs/Cargo.toml | 2 +- rpxy-lib/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f51c4db3..1c86d59c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.8.0" +version = "0.8.0-alpha.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 6cbbb488..6545eefa 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -31,7 +31,7 @@ mimalloc = { version = "*", default-features = false } anyhow = "1.0.86" rustc-hash = "1.1.0" serde = { version = "1.0.203", default-features = false, features = ["derive"] } -tokio = { version = "1.37.0", default-features = false, features = [ +tokio = { version = "1.38.0", default-features = false, features = [ "net", "rt-multi-thread", "time", diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index 2205ebba..61e1bf1f 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -33,7 +33,7 @@ rustls-webpki = { version = "0.102.4", default-features = false, features = [ x509-parser = { version = "0.16.0" } [dev-dependencies] -tokio = { version = "1.37.0", default-features = false, features = [ +tokio = { version = "1.38.0", default-features = false, features = [ "rt-multi-thread", "macros", ] } diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index a746052d..696119ca 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -36,7 +36,7 @@ rustc-hash = "1.1.0" bytes = "1.6.0" derive_builder = "0.20.0" futures = { version = "0.3.30", features = ["alloc", "async-await"] } -tokio = { version = "1.37.0", default-features = false, features = [ +tokio = { version = "1.38.0", default-features = false, features = [ "net", "rt-multi-thread", "time",