diff --git a/Cargo.lock b/Cargo.lock index f823f798..440b86e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,27 +1085,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-extra" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef252edff26ddba56bbcdf2ee3307b8129acb86f5749b68990c168a6fcc9c76" -dependencies = [ - "axum", - "axum-core", - "bytes", - "futures-core", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "axum-server" version = "0.8.0" @@ -4588,9 +4567,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ "memchr", "serde", @@ -5529,9 +5508,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" [[package]] name = "opentelemetry" @@ -7037,7 +7016,6 @@ dependencies = [ "atoi", "atomic_enum", "axum", - "axum-extra", "axum-server", "base64", "base64-simd", @@ -7091,6 +7069,7 @@ dependencies = [ "rustfs-utils", "rustfs-zip", "rustls 0.23.35", + "rustls-pemfile", "s3s", "serde", "serde_json", @@ -7576,6 +7555,7 @@ dependencies = [ "pin-project-lite", "rand 0.10.0-rc.5", "reqwest", + "rustfs-config", "rustfs-utils", "s3s", "serde", @@ -7821,9 +7801,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -7898,7 +7878,7 @@ checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "s3s" version = "0.13.0-alpha" -source = "git+https://github.com/s3s-project/s3s.git?branch=main#f6198bbf49abe60066fe47cbbefcb7078863b3e9" +source = "git+https://github.com/s3s-project/s3s.git?branch=main#9e41304ed549b89cfb03ede98e9c0d2ac7522051" dependencies = [ "arrayvec", "async-trait", @@ -10381,9 +10361,9 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", @@ -10459,9 +10439,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] name = "zmij" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5858cd3a46fff31e77adea2935e357e3a2538d870741617bfb7c943e218fee6" +checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index 6eed163c..107c6cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,7 +100,6 @@ async-compression = { version = "0.4.19" } async-recursion = "1.1.1" async-trait = "0.1.89" axum = "0.8.8" -axum-extra = "0.12.5" axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false } futures = "0.3.31" futures-core = "0.3.31" diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index e0f6a38a..d616f15f 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -25,18 +25,42 @@ pub static GLOBAL_RUSTFS_PORT: LazyLock> = LazyLock::new(|| RwLoc pub static GLOBAL_RUSTFS_ADDR: LazyLock> = LazyLock::new(|| RwLock::new("".to_string())); pub static GLOBAL_CONN_MAP: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); +pub static GLOBAL_MTLS_IDENTITY: LazyLock>> = LazyLock::new(|| RwLock::new(None)); +/// Set the global RustFS address used for gRPC connections. +/// +/// # Arguments +/// * `addr` - A string slice representing the RustFS address (e.g., "https://node1:9000"). pub async fn set_global_addr(addr: &str) { *GLOBAL_RUSTFS_ADDR.write().await = addr.to_string(); } +/// Set the global root CA certificate for outbound gRPC clients. +/// This certificate is used to validate server TLS certificates. +/// When set to None, clients use the system default root CAs. +/// +/// # Arguments +/// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate. pub async fn set_global_root_cert(cert: Vec) { *GLOBAL_ROOT_CERT.write().await = Some(cert); } +/// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients. +/// When set, clients will present this identity to servers requesting/requiring mTLS. +/// When None, clients proceed with standard server-authenticated TLS. +/// +/// # Arguments +/// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM. +pub async fn set_global_mtls_identity(identity: Option) { + *GLOBAL_MTLS_IDENTITY.write().await = identity; +} + /// Evict a stale/dead connection from the global connection cache. /// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off). /// By removing the cached connection, subsequent requests will establish a fresh connection. +/// +/// # Arguments +/// * `addr` - The address of the connection to evict. pub async fn evict_connection(addr: &str) { let removed = GLOBAL_CONN_MAP.write().await.remove(addr); if removed.is_some() { @@ -45,6 +69,12 @@ pub async fn evict_connection(addr: &str) { } /// Check if a connection exists in the cache for the given address. +/// +/// # Arguments +/// * `addr` - The address to check. +/// +/// # Returns +/// * `bool` - True if a cached connection exists, false otherwise. pub async fn has_cached_connection(addr: &str) -> bool { GLOBAL_CONN_MAP.read().await.contains_key(addr) } @@ -58,3 +88,12 @@ pub async fn clear_all_connections() { tracing::warn!("Cleared {} cached connections from global map", count); } } +/// Optional client identity (cert+key PEM) for outbound mTLS. +/// +/// When present, gRPC clients will present this identity to servers requesting/requiring mTLS. +/// When absent, clients proceed with standard server-authenticated TLS. +#[derive(Clone, Debug)] +pub struct MtlsIdentityPem { + pub cert_pem: Vec, + pub key_pem: Vec, +} diff --git a/crates/config/src/constants/tls.rs b/crates/config/src/constants/tls.rs index 6cbebcd4..7772abff 100644 --- a/crates/config/src/constants/tls.rs +++ b/crates/config/src/constants/tls.rs @@ -35,3 +35,52 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA"; /// By default, RustFS does not trust system CA certificates. /// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1 pub const DEFAULT_TRUST_SYSTEM_CA: bool = false; + +/// Environment variable to trust leaf certificates as CA +/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation. +/// By default, this is disabled. +/// To enable, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1 +pub const ENV_TRUST_LEAF_CERT_AS_CA: &str = "RUSTFS_TRUST_LEAF_CERT_AS_CA"; + +/// Default value for trusting leaf certificates as CA +/// By default, RustFS does not trust leaf certificates as CA. +/// To change this behavior, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1 +pub const DEFAULT_TRUST_LEAF_CERT_AS_CA: bool = false; + +/// Default filename for client CA certificate +/// client_ca.crt (CA bundle for verifying client certificates in server mTLS) +pub const RUSTFS_CLIENT_CA_CERT_FILENAME: &str = "client_ca.crt"; + +/// Environment variable for client certificate file path +/// RUSTFS_MTLS_CLIENT_CERT +/// Specifies the file path to the client certificate used for mTLS authentication. +/// If not set, RustFS will look for the default filename "client_cert.pem" in the current directory. +/// To set, use the environment variable RUSTFS_MTLS_CLIENT_CERT=/path/to/client_cert.pem +pub const ENV_MTLS_CLIENT_CERT: &str = "RUSTFS_MTLS_CLIENT_CERT"; + +/// Default filename for client certificate +/// client_cert.pem +pub const RUSTFS_CLIENT_CERT_FILENAME: &str = "client_cert.pem"; + +/// Environment variable for client private key file path +/// RUSTFS_MTLS_CLIENT_KEY +/// Specifies the file path to the client private key used for mTLS authentication. +/// If not set, RustFS will look for the default filename "client_key.pem" in the current directory. +/// To set, use the environment variable RUSTFS_MTLS_CLIENT_KEY=/path/to/client_key.pem +pub const ENV_MTLS_CLIENT_KEY: &str = "RUSTFS_MTLS_CLIENT_KEY"; + +/// Default filename for client private key +/// client_key.pem +pub const RUSTFS_CLIENT_KEY_FILENAME: &str = "client_key.pem"; + +/// RUSTFS_SERVER_MTLS_ENABLE +/// Environment variable to enable server mTLS +/// When set to "1", RustFS server will require client certificates for authentication. +/// By default, this is disabled. +/// To enable, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1 +pub const ENV_SERVER_MTLS_ENABLE: &str = "RUSTFS_SERVER_MTLS_ENABLE"; + +/// Default value for enabling server mTLS +/// By default, RustFS server mTLS is disabled. +/// To change this behavior, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1 +pub const DEFAULT_SERVER_MTLS_ENABLE: bool = false; diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index 2be5d7c2..58b17f13 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -132,6 +132,25 @@ pub enum BucketLookupType { BucketLookupPath, } +fn load_root_store_from_tls_path() -> Option { + // Load the root certificate bundle from the path specified by the + // RUSTFS_TLS_PATH environment variable. + let tp = std::env::var("RUSTFS_TLS_PATH").ok()?; + let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT); + if !ca.exists() { + return None; + } + + let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?; + let mut store = rustls::RootCertStore::empty(); + for der in der_list { + if let Err(e) = store.add(der.into()) { + warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display()); + } + } + Some(store) +} + impl TransitionClient { pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result { let clnt = Self::private_new(endpoint, opts, tier_type).await?; @@ -142,18 +161,22 @@ impl TransitionClient { async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result { let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; - //#[cfg(feature = "ring")] let _ = rustls::crypto::ring::default_provider().install_default(); - //#[cfg(feature = "aws-lc-rs")] - // let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let scheme = endpoint_url.scheme(); let client; - let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth(); + let tls = if let Some(store) = load_root_store_from_tls_path() { + rustls::ClientConfig::builder() + .with_root_certificates(store) + .with_no_client_auth() + } else { + rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth() + }; + let https = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls) .https_or_http() .enable_http1() + .enable_http2() .build(); client = Client::builder(TokioExecutor::new()).build(https); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 607eb1b2..f9f4fe9c 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -16,7 +16,7 @@ mod generated; use proto_gen::node_service::node_service_client::NodeServiceClient; -use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_ROOT_CERT, evict_connection}; +use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection}; use std::{error::Error, time::Duration}; use tonic::{ Request, Status, @@ -83,6 +83,11 @@ async fn create_new_channel(addr: &str) -> Result> { let root_cert = GLOBAL_ROOT_CERT.read().await; if addr.starts_with(RUSTFS_HTTPS_PREFIX) { + if root_cert.is_none() { + debug!("No custom root certificate configured; using system roots for TLS: {}", addr); + // If no custom root cert is configured, try to use system roots. + connector = connector.tls_config(ClientTlsConfig::new())?; + } if let Some(cert_pem) = root_cert.as_ref() { let ca = Certificate::from_pem(cert_pem); // Derive the hostname from the HTTPS URL for TLS hostname verification. @@ -95,7 +100,13 @@ async fn create_new_channel(addr: &str) -> Result> { .next() .unwrap_or(""); let tls = if !domain.is_empty() { - ClientTlsConfig::new().ca_certificate(ca).domain_name(domain) + let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain); + let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await; + if let Some(id) = mtls_identity.as_ref() { + let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone()); + cfg = cfg.identity(identity); + } + cfg } else { // Fallback: configure TLS without explicit domain if parsing fails. ClientTlsConfig::new().ca_certificate(ca) @@ -103,12 +114,9 @@ async fn create_new_channel(addr: &str) -> Result> { connector = connector.tls_config(tls)?; debug!("Configured TLS with custom root certificate for: {}", addr); } else { - debug!("Using system root certificates for TLS: {}", addr); - } - } else { - // Custom root certificates are configured but will be ignored for non-HTTPS addresses. - if root_cert.is_some() { - warn!("Custom root certificates are configured but not used because the address does not use HTTPS: {addr}"); + return Err(std::io::Error::other( + "HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)." + ).into()); } } diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index 4017c62c..3b50e199 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -41,7 +41,8 @@ reqwest.workspace = true tokio-util.workspace = true faster-hex.workspace = true futures.workspace = true -rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] } +rustfs-config = { workspace = true, features = ["constants"] } +rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] } serde_json.workspace = true md-5 = { workspace = true } tracing.workspace = true diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index a2b8d33a..33b88bba 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::{EtagResolvable, HashReaderDetector, HashReaderMut}; use bytes::Bytes; use futures::{Stream, TryStreamExt as _}; use http::HeaderMap; use pin_project_lite::pin_project; -use reqwest::{Client, Method, RequestBuilder}; +use reqwest::{Certificate, Client, Identity, Method, RequestBuilder}; use std::error::Error as _; use std::io::{self, Error}; use std::ops::Not as _; @@ -26,21 +27,88 @@ use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::sync::mpsc; use tokio_util::io::StreamReader; +use tracing::error; -use crate::{EtagResolvable, HashReaderDetector, HashReaderMut}; +/// Get the TLS path from the RUSTFS_TLS_PATH environment variable. +/// If the variable is not set, return None. +fn tls_path() -> Option<&'static std::path::PathBuf> { + static TLS_PATH: LazyLock> = LazyLock::new(|| { + std::env::var("RUSTFS_TLS_PATH") + .ok() + .and_then(|s| if s.is_empty() { None } else { Some(s.into()) }) + }); + TLS_PATH.as_ref() +} + +/// Load CA root certificates from the RUSTFS_TLS_PATH directory. +/// The CA certificates should be in PEM format and stored in the file +/// specified by the RUSTFS_CA_CERT constant. +/// If the file does not exist or cannot be read, return the builder unchanged. +fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + let Some(tp) = tls_path() else { + return builder; + }; + let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT); + if !ca_path.exists() { + return builder; + } + + let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else { + return builder; + }; + + let mut b = builder; + for der in certs_der { + if let Ok(cert) = Certificate::from_der(&der) { + b = b.add_root_certificate(cert); + } + } + b +} + +/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory. +/// The client certificate and private key should be in PEM format and stored in the files +/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants. +/// If the files do not exist or cannot be read, return None. +fn load_optional_mtls_identity_from_tls_path() -> Option { + let tp = tls_path()?; + let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?; + let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?; + + let mut pem = Vec::with_capacity(cert.len() + key.len() + 1); + pem.extend_from_slice(&cert); + if !pem.ends_with(b"\n") { + pem.push(b'\n'); + } + pem.extend_from_slice(&key); + + match Identity::from_pem(&pem) { + Ok(id) => Some(id), + Err(e) => { + error!("Failed to load mTLS identity from PEM: {e}"); + None + } + } +} fn get_http_client() -> Client { // Reuse the HTTP connection pool in the global `reqwest::Client` instance // TODO: interact with load balancing? static CLIENT: LazyLock = LazyLock::new(|| { - Client::builder() + let mut builder = Client::builder() .connect_timeout(std::time::Duration::from_secs(5)) .tcp_keepalive(std::time::Duration::from_secs(10)) .http2_keep_alive_interval(std::time::Duration::from_secs(5)) .http2_keep_alive_timeout(std::time::Duration::from_secs(3)) - .http2_keep_alive_while_idle(true) - .build() - .expect("Failed to create global HTTP client") + .http2_keep_alive_while_idle(true); + + // HTTPS root trust + optional mTLS identity from RUSTFS_TLS_PATH + builder = load_ca_roots_from_tls_path(builder); + if let Some(id) = load_optional_mtls_identity_from_tls_path() { + builder = builder.identity(id); + } + + builder.build().expect("Failed to create global HTTP client") }); CLIENT.clone() } diff --git a/crates/utils/src/certs.rs b/crates/utils/src/certs.rs index 463874ed..8615375a 100644 --- a/crates/utils/src/certs.rs +++ b/crates/utils/src/certs.rs @@ -12,8 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::get_env_bool; use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; -use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; +use rustls::RootCertStore; +use rustls::server::danger::ClientCertVerifier; +use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni, WebPkiClientVerifier}; use rustls::sign::CertifiedKey; use rustls_pemfile::{certs, private_key}; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; @@ -48,6 +51,79 @@ pub fn load_certs(filename: &str) -> io::Result>> { Ok(certs) } +/// Load a PEM certificate bundle and return each certificate as DER bytes. +/// +/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that +/// need to add root certificates one-by-one. +/// +/// - Input: a PEM file that may contain multiple cert blocks. +/// - Output: Vec of DER-encoded cert bytes, one per cert. +/// +/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates +/// to depend on rustls types. +pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result>> { + let pem = fs::read(path)?; + let mut reader = io::BufReader::new(&pem[..]); + + let certs = certs(&mut reader) + .collect::, _>>() + .map_err(|e| certs_error(format!("Failed to parse PEM certs from {path}: {e}")))?; + + Ok(certs.into_iter().map(|c| c.to_vec()).collect()) +} + +/// Builds a WebPkiClientVerifier for mTLS if enabled via environment variable. +/// +/// # Arguments +/// * `tls_path` - Directory containing client CA certificates +/// +/// # Returns +/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found +/// * `Ok(None)` if mTLS is disabled +/// * `Err` if mTLS is enabled but configuration is invalid +pub fn build_webpki_client_verifier(tls_path: &str) -> io::Result>> { + if !get_env_bool(rustfs_config::ENV_SERVER_MTLS_ENABLE, rustfs_config::DEFAULT_SERVER_MTLS_ENABLE) { + return Ok(None); + } + + let ca_path = mtls_ca_bundle_path(tls_path).ok_or_else(|| { + Error::other(format!( + "RUSTFS_SERVER_MTLS_ENABLE=true but missing {}/client_ca.crt (or fallback {}/ca.crt)", + tls_path, tls_path + )) + })?; + + let der_list = load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default())?; + + let mut store = RootCertStore::empty(); + for der in der_list { + store + .add(der.into()) + .map_err(|e| Error::other(format!("Invalid client CA cert: {e}")))?; + } + + let verifier = WebPkiClientVerifier::builder(Arc::new(store)) + .build() + .map_err(|e| Error::other(format!("Build client cert verifier failed: {e}")))?; + + Ok(Some(verifier)) +} + +/// Locate the mTLS client CA bundle in the specified TLS path +fn mtls_ca_bundle_path(tls_path: &str) -> Option { + use std::path::Path; + + let p1 = Path::new(tls_path).join(rustfs_config::RUSTFS_CLIENT_CA_CERT_FILENAME); + if p1.exists() { + return Some(p1); + } + let p2 = Path::new(tls_path).join(rustfs_config::RUSTFS_CA_CERT); + if p2.exists() { + return Some(p2); + } + None +} + /// Load private key from file. /// This function loads a private key from the specified file. /// diff --git a/docs/examples/mnmd/docker-compose.mtls.yml b/docs/examples/mnmd/docker-compose.mtls.yml new file mode 100644 index 00000000..088a167d --- /dev/null +++ b/docs/examples/mnmd/docker-compose.mtls.yml @@ -0,0 +1,32 @@ +services: + mnmd: + image: ghcr.io/your-org/mnmd:latest + container_name: mnmd + ports: + - "8443:8443" + volumes: + - ./tls:/tls:ro + environment: + # Example mnmd settings (adapt to your image) + - MNMD_LISTEN_ADDR=0.0.0.0:8443 + - MNMD_TLS_CERT=/tls/server_cert.pem + - MNMD_TLS_KEY=/tls/server_key.pem + - MNMD_TLS_CLIENT_CA=/tls/ca.crt + + rustfs: + image: ghcr.io/rustfs/rustfs:latest + container_name: rustfs + depends_on: + - mnmd + environment: + - RUSTFS_TLS_PATH=/tls + - RUSTFS_TRUST_SYSTEM_CA=false + - RUSTFS_TRUST_LEAF_CERT_AS_CA=false + # Enable outbound mTLS (client identity) for MNMD + - RUSTFS_MTLS_CLIENT_CERT=/tls/client_cert.pem + - RUSTFS_MTLS_CLIENT_KEY=/tls/client_key.pem + # MNMD address configured to https + - RUSTFS_MNMD_ADDR=https://mnmd:8443 + - RUSTFS_MNMD_DOMAIN=mnmd + volumes: + - ./tls:/tls:ro diff --git a/docs/tls.md b/docs/tls.md new file mode 100644 index 00000000..404cae87 --- /dev/null +++ b/docs/tls.md @@ -0,0 +1,63 @@ +# TLS / mTLS configuration + +RustFS supports TLS for serving HTTPS and for outbound gRPC connections (MNMD). +It also supports optional client certificate authentication (mTLS) for outbound gRPC: +if a client identity is configured, RustFS will present it; otherwise it will use +server-authenticated TLS only. + +## Recommended `tls/` directory layout + +Place these files in a directory (default: `./tls`, configurable via `RUSTFS_TLS_PATH`). + +``` +TLS_DIR/ + ca.crt # PEM bundle of CA/root certificates to trust (recommended) + public.crt # optional extra root bundle (PEM) + rustfs_cert.pem # server leaf certificate (PEM) used by the RustFS server + rustfs_key.pem # server private key (PEM) used by the RustFS server + + # Optional: outbound mTLS client identity for MNMD + client_cert.pem # client certificate chain (PEM) + client_key.pem # client private key (PEM) + + # Optional: server-side mTLS (inbound client certificate verification) + client_ca.crt # PEM bundle of CA certificates to verify client certificates +``` + +## Environment variables + +### Root trust + +- `RUSTFS_TLS_PATH` (default: `tls`): TLS directory. +- `RUSTFS_TRUST_SYSTEM_CA` (default: `false`): When `true`, include the platform/system + trust store as additional roots. When `false`, system roots are not used. +- `RUSTFS_TRUST_LEAF_CERT_AS_CA` (default: `false`): Compatibility switch. If `true`, + RustFS will also load `rustfs_cert.pem` into the root store (treating leaf certificates + as trusted roots). Prefer providing `ca.crt` instead. + +### Outbound mTLS identity + +- `RUSTFS_MTLS_CLIENT_CERT` (default: `${RUSTFS_TLS_PATH}/client_cert.pem`): path to PEM client cert/chain. +- `RUSTFS_MTLS_CLIENT_KEY` (default: `${RUSTFS_TLS_PATH}/client_key.pem`): path to PEM private key. + +If both files exist, RustFS enables outbound mTLS. If either is missing, RustFS proceeds +with server-only TLS. + +### Server-side mTLS (inbound client certificate verification) + +- `RUSTFS_SERVER_MTLS_ENABLE` (default: `false`): When `true`, the RustFS server requires + clients to present valid certificates signed by a trusted CA for authentication. + +When enabled, RustFS loads client CA certificates from: +1. `${RUSTFS_TLS_PATH}/client_ca.crt` (preferred) +2. `${RUSTFS_TLS_PATH}/ca.crt` (fallback if `client_ca.crt` does not exist) + +**Important**: Server mTLS is disabled by default. When enabled but no valid CA bundle is +found, RustFS will fail to start with a clear error message. This ensures that server mTLS +cannot be accidentally enabled without proper client CA configuration. + +## Failure mode for HTTPS without roots + +When connecting to an `https://` MNMD address, RustFS requires at least one configured +trusted root. If none are loaded (no `ca.crt`/`public.crt` and system roots disabled), +RustFS fails fast with a clear error message. diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 3e2da8e1..6e6d66ec 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -65,7 +65,6 @@ rustfs-zip = { workspace = true } # Async Runtime and Networking async-trait = { workspace = true } axum.workspace = true -axum-extra = { workspace = true } axum-server = { workspace = true } futures.workspace = true futures-util.workspace = true @@ -95,6 +94,7 @@ serde_urlencoded = { workspace = true } # Cryptography and Security rustls = { workspace = true } subtle = { workspace = true } +rustls-pemfile = { workspace = true } # Time and Date chrono = { workspace = true } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 438181e1..9a993d00 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -95,7 +95,9 @@ async fn async_main() -> Result<()> { // Store in global storage match set_global_guard(guard).map_err(Error::other) { - Ok(_) => (), + Ok(_) => { + info!(target: "rustfs::main", "Global observability guard set successfully."); + } Err(e) => { error!("Failed to set global observability guard: {}", e); return Err(e); @@ -110,7 +112,15 @@ async fn async_main() -> Result<()> { // Initialize TLS if a certificate path is provided if let Some(tls_path) = &opt.tls_path { - init_cert(tls_path).await + match init_cert(tls_path).await { + Ok(_) => { + info!(target: "rustfs::main", "TLS initialized successfully with certs from {}", tls_path); + } + Err(e) => { + error!("Failed to initialize TLS from {}: {}", tls_path, e); + return Err(Error::other(e)); + } + } } // Run parameters diff --git a/rustfs/src/server/cert.rs b/rustfs/src/server/cert.rs index 93013be0..5c57e7f1 100644 --- a/rustfs/src/server/cert.rs +++ b/rustfs/src/server/cert.rs @@ -12,34 +12,129 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_common::set_global_root_cert; +use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert}; use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_PUBLIC_CERT, RUSTFS_TLS_CERT}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::path::{Path, PathBuf}; use tracing::{debug, info}; -/// Initialize TLS certificates for inter-node communication. -/// This function attempts to load certificates from the specified `tls_path`. -/// It looks for `rustfs_cert.pem`, `public.crt`, and `ca.crt` files. -/// Additionally, it tries to load system root certificates from common locations -/// to ensure trust for public CAs when mixing self-signed and public certificates. -/// If any certificates are found, they are set as the global root certificates. -pub(crate) async fn init_cert(tls_path: &str) { +#[derive(Debug)] +pub enum RustFSError { + Cert(String), +} + +impl std::fmt::Display for RustFSError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RustFSError::Cert(msg) => write!(f, "Certificate error: {}", msg), + } + } +} + +impl std::error::Error for RustFSError {} + +/// Parse PEM-encoded certificates into DER format. +/// Returns a vector of DER-encoded certificates. +/// +/// # Arguments +/// * `pem` - A byte slice containing the PEM-encoded certificates. +/// +/// # Returns +/// A vector of `CertificateDer` containing the DER-encoded certificates. +/// +/// # Errors +/// Returns `RustFSError` if parsing fails. +fn parse_pem_certs(pem: &[u8]) -> Result>, RustFSError> { + let mut out = Vec::new(); + let mut reader = std::io::Cursor::new(pem); + for item in rustls_pemfile::certs(&mut reader) { + let c = item.map_err(|e| RustFSError::Cert(format!("parse cert pem: {e}")))?; + out.push(c); + } + Ok(out) +} + +/// Parse a PEM-encoded private key into DER format. +/// Supports PKCS#8 and RSA private keys. +/// +/// # Arguments +/// * `pem` - A byte slice containing the PEM-encoded private key. +/// +/// # Returns +/// A `PrivateKeyDer` containing the DER-encoded private key. +/// +/// # Errors +/// Returns `RustFSError` if parsing fails or no key is found. +fn parse_pem_private_key(pem: &[u8]) -> Result, RustFSError> { + let mut reader = std::io::Cursor::new(pem); + let key = rustls_pemfile::private_key(&mut reader).map_err(|e| RustFSError::Cert(format!("parse private key pem: {e}")))?; + key.ok_or_else(|| RustFSError::Cert("no private key found in PEM".into())) +} + +/// Helper function to read a file and return its contents. +/// Returns the file contents as a vector of bytes. +/// # Errors +/// Returns `RustFSError` if reading fails. +async fn read_file(path: &PathBuf, desc: &str) -> Result, RustFSError> { + tokio::fs::read(path) + .await + .map_err(|e| RustFSError::Cert(format!("read {} {:?}: {e}", desc, path))) +} + +/// Initialize TLS material for both server and outbound client connections. +/// +/// Loads roots from: +/// - `${RUSTFS_TLS_PATH}/ca.crt` (or `tls/ca.crt`) +/// - `${RUSTFS_TLS_PATH}/public.crt` (optional additional root bundle) +/// - system roots if `RUSTFS_TRUST_SYSTEM_CA=true` (default: false) +/// - if `RUSTFS_TRUST_LEAF_CERT_AS_CA=true`, also loads leaf cert(s) from +/// `${RUSTFS_TLS_PATH}/rustfs_cert.pem` into the root store. +/// +/// Loads mTLS client identity (optional) from: +/// - `${RUSTFS_TLS_PATH}/client_cert.pem` +/// - `${RUSTFS_TLS_PATH}/client_key.pem` +/// +/// Environment overrides: +/// - RUSTFS_TLS_PATH +/// - RUSTFS_MTLS_CLIENT_CERT +/// - RUSTFS_MTLS_CLIENT_KEY +pub(crate) async fn init_cert(tls_path: &str) -> Result<(), RustFSError> { + if tls_path.is_empty() { + info!("No TLS path configured; skipping certificate initialization"); + return Ok(()); + } + let tls_dir = PathBuf::from(tls_path); + + // Load root certificates + load_root_certs(&tls_dir).await?; + + // Load optional mTLS identity + load_mtls_identity(&tls_dir).await?; + + Ok(()) +} + +/// Load root certificates from various sources. +async fn load_root_certs(tls_dir: &Path) -> Result<(), RustFSError> { let mut cert_data = Vec::new(); - // Try rustfs_cert.pem (custom cert name) - walk_dir(std::path::PathBuf::from(tls_path), RUSTFS_TLS_CERT, &mut cert_data).await; + let trust_leaf_as_ca = + rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA); + if trust_leaf_as_ca { + walk_dir(tls_dir.to_path_buf(), RUSTFS_TLS_CERT, &mut cert_data).await; + info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA"); + } - // Try public.crt (common CA name) - let public_cert_path = std::path::Path::new(tls_path).join(RUSTFS_PUBLIC_CERT); + // Try public.crt and ca.crt + let public_cert_path = tls_dir.join(RUSTFS_PUBLIC_CERT); load_cert_file(public_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await; - // Try ca.crt (common CA name) - let ca_cert_path = std::path::Path::new(tls_path).join(RUSTFS_CA_CERT); + let ca_cert_path = tls_dir.join(RUSTFS_CA_CERT); load_cert_file(ca_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await; + // Load system root certificates if enabled let trust_system_ca = rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_SYSTEM_CA, rustfs_config::DEFAULT_TRUST_SYSTEM_CA); - if !trust_system_ca { - // Attempt to load system root certificates to maintain trust for public CAs - // This is important when mixing self-signed internal certs with public external certs + if trust_system_ca { let system_ca_paths = [ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Alpine "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL/CentOS @@ -57,7 +152,7 @@ pub(crate) async fn init_cert(tls_path: &str) { if load_cert_file(path, &mut cert_data, "system root certificates").await { system_cert_loaded = true; info!("Loaded system root certificates from {}", path); - break; // Stop after finding the first valid bundle + break; } } @@ -67,10 +162,51 @@ pub(crate) async fn init_cert(tls_path: &str) { } else { info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA"); } + if !cert_data.is_empty() { set_global_root_cert(cert_data).await; info!("Configured custom root certificates for inter-node communication"); } + + Ok(()) +} + +/// Load optional mTLS identity. +async fn load_mtls_identity(tls_dir: &Path) -> Result<(), RustFSError> { + let client_cert_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_CERT) { + Some(p) => PathBuf::from(p), + None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME), + }; + + let client_key_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_KEY) { + Some(p) => PathBuf::from(p), + None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME), + }; + + if client_cert_path.exists() && client_key_path.exists() { + let cert_bytes = read_file(&client_cert_path, "client cert").await?; + let key_bytes = read_file(&client_key_path, "client key").await?; + + // Validate parse-ability early; store as PEM bytes for tonic. + parse_pem_certs(&cert_bytes)?; + parse_pem_private_key(&key_bytes)?; + + let identity_pem = MtlsIdentityPem { + cert_pem: cert_bytes, + key_pem: key_bytes, + }; + + set_global_mtls_identity(Some(identity_pem)).await; + info!("Loaded mTLS client identity cert={:?} key={:?}", client_cert_path, client_key_path); + } else { + set_global_mtls_identity(None).await; + info!( + "mTLS client identity not configured (missing {:?} and/or {:?}); proceeding with server-only TLS", + client_cert_path, client_key_path + ); + } + + Ok(()) } /// Helper function to load a certificate file and append to cert_data. @@ -114,7 +250,7 @@ async fn load_if_matches(entry: &tokio::fs::DirEntry, cert_name: &str, cert_data /// - `path`: The starting directory path to search for certificates. /// - `cert_name`: The name of the certificate file to look for. /// - `cert_data`: A mutable vector to append loaded certificate data. -async fn walk_dir(path: std::path::PathBuf, cert_name: &str, cert_data: &mut Vec) { +async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec) { if let Ok(mut rd) = tokio::fs::read_dir(&path).await { while let Ok(Some(entry)) = rd.next_entry().await { if let Ok(ft) = entry.file_type().await { diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index efe4c56a..d1753019 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -431,11 +431,11 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result> { debug!("TLS path is not provided or does not exist, starting with HTTP"); return Ok(None); } - debug!("Found TLS directory, checking for certificates"); // Make sure to use a modern encryption suite let _ = rustls::crypto::ring::default_provider().install_default(); + let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?; // 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI) if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) { @@ -446,9 +446,15 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result> { let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?; // Configure the server to enable SNI support - let mut server_config = ServerConfig::builder() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver)); + let mut server_config = if let Some(verifier) = mtls_verifier.clone() { + ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_cert_resolver(Arc::new(resolver)) + } else { + ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver)) + }; // Configure ALPN protocol priority server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; @@ -470,10 +476,17 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result> { let certs = rustfs_utils::load_certs(&cert_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?; let key = rustfs_utils::load_private_key(&key_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?; - let mut server_config = ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| rustfs_utils::certs_error(e.to_string()))?; + let mut server_config = if let Some(verifier) = mtls_verifier { + ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(certs, key) + .map_err(|e| rustfs_utils::certs_error(e.to_string()))? + } else { + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|e| rustfs_utils::certs_error(e.to_string()))? + }; // Configure ALPN protocol priority server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];