sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-16 17:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Restore globals and add unified TLS/mTLS loading from RUSTFS_TLS_PATH (#1309)

Browse Source 
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2025-12-30 21:55:43 +08:00
committed by GitHub
parent b4ba62fa33
commit 2924b4e463
15 changed files with 582 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
Cargo.lock generated
View File

@@ -1085,27 +1085,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "axum-extra"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fef252edff26ddba56bbcdf2ee3307b8129acb86f5749b68990c168a6fcc9c76"
dependencies = [
"axum",
"axum-core",
"bytes",
"futures-core",
"futures-util",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"mime",
"pin-project-lite",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-server"
version = "0.8.0"
@@ -4588,9 +4567,9 @@ dependencies = [
[[package]]
name = "iri-string"
version = "0.7.9"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397"
checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
dependencies = [
"memchr",
"serde",
@@ -5529,9 +5508,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "openssl-probe"
version = "0.1.6"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391"
[[package]]
name = "opentelemetry"
@@ -7037,7 +7016,6 @@ dependencies = [
"atoi",
"atomic_enum",
"axum",
"axum-extra",
"axum-server",
"base64",
"base64-simd",
@@ -7091,6 +7069,7 @@ dependencies = [
"rustfs-utils",
"rustfs-zip",
"rustls 0.23.35",
"rustls-pemfile",
"s3s",
"serde",
"serde_json",
@@ -7576,6 +7555,7 @@ dependencies = [
"pin-project-lite",
"rand 0.10.0-rc.5",
"reqwest",
"rustfs-config",
"rustfs-utils",
"s3s",
"serde",
@@ -7821,9 +7801,9 @@ dependencies = [
[[package]]
name = "rustls-native-certs"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923"
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
dependencies = [
"openssl-probe",
"rustls-pki-types",
@@ -7898,7 +7878,7 @@ checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984"
[[package]]
name = "s3s"
version = "0.13.0-alpha"
source = "git+https://github.com/s3s-project/s3s.git?branch=main#f6198bbf49abe60066fe47cbbefcb7078863b3e9"
source = "git+https://github.com/s3s-project/s3s.git?branch=main#9e41304ed549b89cfb03ede98e9c0d2ac7522051"
dependencies = [
"arrayvec",
"async-trait",
@@ -10381,9 +10361,9 @@ dependencies = [
[[package]]
name = "zeroize_derive"
version = "1.4.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
dependencies = [
"proc-macro2",
"quote",
@@ -10459,9 +10439,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
[[package]]
name = "zmij"
version = "1.0.1"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5858cd3a46fff31e77adea2935e357e3a2538d870741617bfb7c943e218fee6"
checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06"
[[package]]
name = "zopfli"

1
Cargo.toml
View File

@@ -100,7 +100,6 @@ async-compression = { version = "0.4.19" }
async-recursion = "1.1.1"
async-trait = "0.1.89"
axum = "0.8.8"
axum-extra = "0.12.5"
axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false }
futures = "0.3.31"
futures-core = "0.3.31"

39
crates/common/src/globals.rs
View File

@@ -25,18 +25,42 @@ pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLoc
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
/// Set the global RustFS address used for gRPC connections.
///
/// # Arguments
/// * `addr` - A string slice representing the RustFS address (e.g., "https://node1:9000").
pub async fn set_global_addr(addr: &str) {
*GLOBAL_RUSTFS_ADDR.write().await = addr.to_string();
}
/// Set the global root CA certificate for outbound gRPC clients.
/// This certificate is used to validate server TLS certificates.
/// When set to None, clients use the system default root CAs.
///
/// # Arguments
/// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate.
pub async fn set_global_root_cert(cert: Vec<u8>) {
*GLOBAL_ROOT_CERT.write().await = Some(cert);
}
/// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients.
/// When set, clients will present this identity to servers requesting/requiring mTLS.
/// When None, clients proceed with standard server-authenticated TLS.
///
/// # Arguments
/// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM.
pub async fn set_global_mtls_identity(identity: Option<MtlsIdentityPem>) {
*GLOBAL_MTLS_IDENTITY.write().await = identity;
}
/// Evict a stale/dead connection from the global connection cache.
/// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off).
/// By removing the cached connection, subsequent requests will establish a fresh connection.
///
/// # Arguments
/// * `addr` - The address of the connection to evict.
pub async fn evict_connection(addr: &str) {
let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
if removed.is_some() {
@@ -45,6 +69,12 @@ pub async fn evict_connection(addr: &str) {
}
/// Check if a connection exists in the cache for the given address.
///
/// # Arguments
/// * `addr` - The address to check.
///
/// # Returns
/// * `bool` - True if a cached connection exists, false otherwise.
pub async fn has_cached_connection(addr: &str) -> bool {
GLOBAL_CONN_MAP.read().await.contains_key(addr)
}
@@ -58,3 +88,12 @@ pub async fn clear_all_connections() {
tracing::warn!("Cleared {} cached connections from global map", count);
}
}
/// Optional client identity (cert+key PEM) for outbound mTLS.
///
/// When present, gRPC clients will present this identity to servers requesting/requiring mTLS.
/// When absent, clients proceed with standard server-authenticated TLS.
#[derive(Clone, Debug)]
pub struct MtlsIdentityPem {
pub cert_pem: Vec<u8>,
pub key_pem: Vec<u8>,
}

49
crates/config/src/constants/tls.rs
View File

@@ -35,3 +35,52 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
/// By default, RustFS does not trust system CA certificates.
/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;
/// Environment variable to trust leaf certificates as CA
/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation.
/// By default, this is disabled.
/// To enable, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1
pub const ENV_TRUST_LEAF_CERT_AS_CA: &str = "RUSTFS_TRUST_LEAF_CERT_AS_CA";
/// Default value for trusting leaf certificates as CA
/// By default, RustFS does not trust leaf certificates as CA.
/// To change this behavior, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1
pub const DEFAULT_TRUST_LEAF_CERT_AS_CA: bool = false;
/// Default filename for client CA certificate
/// client_ca.crt (CA bundle for verifying client certificates in server mTLS)
pub const RUSTFS_CLIENT_CA_CERT_FILENAME: &str = "client_ca.crt";
/// Environment variable for client certificate file path
/// RUSTFS_MTLS_CLIENT_CERT
/// Specifies the file path to the client certificate used for mTLS authentication.
/// If not set, RustFS will look for the default filename "client_cert.pem" in the current directory.
/// To set, use the environment variable RUSTFS_MTLS_CLIENT_CERT=/path/to/client_cert.pem
pub const ENV_MTLS_CLIENT_CERT: &str = "RUSTFS_MTLS_CLIENT_CERT";
/// Default filename for client certificate
/// client_cert.pem
pub const RUSTFS_CLIENT_CERT_FILENAME: &str = "client_cert.pem";
/// Environment variable for client private key file path
/// RUSTFS_MTLS_CLIENT_KEY
/// Specifies the file path to the client private key used for mTLS authentication.
/// If not set, RustFS will look for the default filename "client_key.pem" in the current directory.
/// To set, use the environment variable RUSTFS_MTLS_CLIENT_KEY=/path/to/client_key.pem
pub const ENV_MTLS_CLIENT_KEY: &str = "RUSTFS_MTLS_CLIENT_KEY";
/// Default filename for client private key
/// client_key.pem
pub const RUSTFS_CLIENT_KEY_FILENAME: &str = "client_key.pem";
/// RUSTFS_SERVER_MTLS_ENABLE
/// Environment variable to enable server mTLS
/// When set to "1", RustFS server will require client certificates for authentication.
/// By default, this is disabled.
/// To enable, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
pub const ENV_SERVER_MTLS_ENABLE: &str = "RUSTFS_SERVER_MTLS_ENABLE";
/// Default value for enabling server mTLS
/// By default, RustFS server mTLS is disabled.
/// To change this behavior, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
pub const DEFAULT_SERVER_MTLS_ENABLE: bool = false;

33
crates/ecstore/src/client/transition_api.rs
View File

@@ -132,6 +132,25 @@ pub enum BucketLookupType {
BucketLookupPath,
}
fn load_root_store_from_tls_path() -> Option<rustls::RootCertStore> {
// Load the root certificate bundle from the path specified by the
// RUSTFS_TLS_PATH environment variable.
let tp = std::env::var("RUSTFS_TLS_PATH").ok()?;
let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT);
if !ca.exists() {
return None;
}
let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?;
let mut store = rustls::RootCertStore::empty();
for der in der_list {
if let Err(e) = store.add(der.into()) {
warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display());
}
}
Some(store)
}
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
let clnt = Self::private_new(endpoint, opts, tier_type).await?;
@@ -142,18 +161,22 @@ impl TransitionClient {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
//#[cfg(feature = "ring")]
let _ = rustls::crypto::ring::default_provider().install_default();
//#[cfg(feature = "aws-lc-rs")]
// let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let scheme = endpoint_url.scheme();
let client;
let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth();
let tls = if let Some(store) = load_root_store_from_tls_path() {
rustls::ClientConfig::builder()
.with_root_certificates(store)
.with_no_client_auth()
} else {
rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth()
};
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
.https_or_http()
.enable_http1()
.enable_http2()
.build();
client = Client::builder(TokioExecutor::new()).build(https);

24
crates/protos/src/lib.rs
View File

@@ -16,7 +16,7 @@
mod generated;
use proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_ROOT_CERT, evict_connection};
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection};
use std::{error::Error, time::Duration};
use tonic::{
Request, Status,
@@ -83,6 +83,11 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
let root_cert = GLOBAL_ROOT_CERT.read().await;
if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
if root_cert.is_none() {
debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
// If no custom root cert is configured, try to use system roots.
connector = connector.tls_config(ClientTlsConfig::new())?;
}
if let Some(cert_pem) = root_cert.as_ref() {
let ca = Certificate::from_pem(cert_pem);
// Derive the hostname from the HTTPS URL for TLS hostname verification.
@@ -95,7 +100,13 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
.next()
.unwrap_or("");
let tls = if !domain.is_empty() {
ClientTlsConfig::new().ca_certificate(ca).domain_name(domain)
let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain);
let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await;
if let Some(id) = mtls_identity.as_ref() {
let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
cfg = cfg.identity(identity);
}
cfg
} else {
// Fallback: configure TLS without explicit domain if parsing fails.
ClientTlsConfig::new().ca_certificate(ca)
@@ -103,12 +114,9 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
connector = connector.tls_config(tls)?;
debug!("Configured TLS with custom root certificate for: {}", addr);
} else {
debug!("Using system root certificates for TLS: {}", addr);
}
} else {
// Custom root certificates are configured but will be ignored for non-HTTPS addresses.
if root_cert.is_some() {
warn!("Custom root certificates are configured but not used because the address does not use HTTPS: {addr}");
return Err(std::io::Error::other(
"HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)."
).into());
}
}

3
crates/rio/Cargo.toml
View File

@@ -41,7 +41,8 @@ reqwest.workspace = true
tokio-util.workspace = true
faster-hex.workspace = true
futures.workspace = true
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] }
serde_json.workspace = true
md-5 = { workspace = true }
tracing.workspace = true

80
crates/rio/src/http_reader.rs
View File

@@ -12,11 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
use bytes::Bytes;
use futures::{Stream, TryStreamExt as _};
use http::HeaderMap;
use pin_project_lite::pin_project;
use reqwest::{Client, Method, RequestBuilder};
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
use std::error::Error as _;
use std::io::{self, Error};
use std::ops::Not as _;
@@ -26,21 +27,88 @@ use std::task::{Context, Poll};
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio::sync::mpsc;
use tokio_util::io::StreamReader;
use tracing::error;
use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
/// Get the TLS path from the RUSTFS_TLS_PATH environment variable.
/// If the variable is not set, return None.
fn tls_path() -> Option<&'static std::path::PathBuf> {
static TLS_PATH: LazyLock<Option<std::path::PathBuf>> = LazyLock::new(|| {
std::env::var("RUSTFS_TLS_PATH")
.ok()
.and_then(|s| if s.is_empty() { None } else { Some(s.into()) })
});
TLS_PATH.as_ref()
}
/// Load CA root certificates from the RUSTFS_TLS_PATH directory.
/// The CA certificates should be in PEM format and stored in the file
/// specified by the RUSTFS_CA_CERT constant.
/// If the file does not exist or cannot be read, return the builder unchanged.
fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
let Some(tp) = tls_path() else {
return builder;
};
let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
if !ca_path.exists() {
return builder;
}
let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else {
return builder;
};
let mut b = builder;
for der in certs_der {
if let Ok(cert) = Certificate::from_der(&der) {
b = b.add_root_certificate(cert);
}
}
b
}
/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory.
/// The client certificate and private key should be in PEM format and stored in the files
/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants.
/// If the files do not exist or cannot be read, return None.
fn load_optional_mtls_identity_from_tls_path() -> Option<Identity> {
let tp = tls_path()?;
let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?;
let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?;
let mut pem = Vec::with_capacity(cert.len() + key.len() + 1);
pem.extend_from_slice(&cert);
if !pem.ends_with(b"\n") {
pem.push(b'\n');
}
pem.extend_from_slice(&key);
match Identity::from_pem(&pem) {
Ok(id) => Some(id),
Err(e) => {
error!("Failed to load mTLS identity from PEM: {e}");
None
}
}
}
fn get_http_client() -> Client {
// Reuse the HTTP connection pool in the global `reqwest::Client` instance
// TODO: interact with load balancing?
static CLIENT: LazyLock<Client> = LazyLock::new(|| {
Client::builder()
let mut builder = Client::builder()
.connect_timeout(std::time::Duration::from_secs(5))
.tcp_keepalive(std::time::Duration::from_secs(10))
.http2_keep_alive_interval(std::time::Duration::from_secs(5))
.http2_keep_alive_timeout(std::time::Duration::from_secs(3))
.http2_keep_alive_while_idle(true)
.build()
.expect("Failed to create global HTTP client")
.http2_keep_alive_while_idle(true);
// HTTPS root trust + optional mTLS identity from RUSTFS_TLS_PATH
builder = load_ca_roots_from_tls_path(builder);
if let Some(id) = load_optional_mtls_identity_from_tls_path() {
builder = builder.identity(id);
}
builder.build().expect("Failed to create global HTTP client")
});
CLIENT.clone()
}

78
crates/utils/src/certs.rs
View File

@@ -12,8 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::get_env_bool;
use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
use rustls::RootCertStore;
use rustls::server::danger::ClientCertVerifier;
use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni, WebPkiClientVerifier};
use rustls::sign::CertifiedKey;
use rustls_pemfile::{certs, private_key};
use rustls_pki_types::{CertificateDer, PrivateKeyDer};
@@ -48,6 +51,79 @@ pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
Ok(certs)
}
/// Load a PEM certificate bundle and return each certificate as DER bytes.
///
/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that
/// need to add root certificates one-by-one.
///
/// - Input: a PEM file that may contain multiple cert blocks.
/// - Output: Vec of DER-encoded cert bytes, one per cert.
///
/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates
/// to depend on rustls types.
pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
let pem = fs::read(path)?;
let mut reader = io::BufReader::new(&pem[..]);
let certs = certs(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| certs_error(format!("Failed to parse PEM certs from {path}: {e}")))?;
Ok(certs.into_iter().map(|c| c.to_vec()).collect())
}
/// Builds a WebPkiClientVerifier for mTLS if enabled via environment variable.
///
/// # Arguments
/// * `tls_path` - Directory containing client CA certificates
///
/// # Returns
/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found
/// * `Ok(None)` if mTLS is disabled
/// * `Err` if mTLS is enabled but configuration is invalid
pub fn build_webpki_client_verifier(tls_path: &str) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
if !get_env_bool(rustfs_config::ENV_SERVER_MTLS_ENABLE, rustfs_config::DEFAULT_SERVER_MTLS_ENABLE) {
return Ok(None);
}
let ca_path = mtls_ca_bundle_path(tls_path).ok_or_else(|| {
Error::other(format!(
"RUSTFS_SERVER_MTLS_ENABLE=true but missing {}/client_ca.crt (or fallback {}/ca.crt)",
tls_path, tls_path
))
})?;
let der_list = load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default())?;
let mut store = RootCertStore::empty();
for der in der_list {
store
.add(der.into())
.map_err(|e| Error::other(format!("Invalid client CA cert: {e}")))?;
}
let verifier = WebPkiClientVerifier::builder(Arc::new(store))
.build()
.map_err(|e| Error::other(format!("Build client cert verifier failed: {e}")))?;
Ok(Some(verifier))
}
/// Locate the mTLS client CA bundle in the specified TLS path
fn mtls_ca_bundle_path(tls_path: &str) -> Option<std::path::PathBuf> {
use std::path::Path;
let p1 = Path::new(tls_path).join(rustfs_config::RUSTFS_CLIENT_CA_CERT_FILENAME);
if p1.exists() {
return Some(p1);
}
let p2 = Path::new(tls_path).join(rustfs_config::RUSTFS_CA_CERT);
if p2.exists() {
return Some(p2);
}
None
}
/// Load private key from file.
/// This function loads a private key from the specified file.
///

32
docs/examples/mnmd/docker-compose.mtls.yml Normal file
View File

@@ -0,0 +1,32 @@
services:
mnmd:
image: ghcr.io/your-org/mnmd:latest
container_name: mnmd
ports:
- "8443:8443"
volumes:
- ./tls:/tls:ro
environment:
# Example mnmd settings (adapt to your image)
- MNMD_LISTEN_ADDR=0.0.0.0:8443
- MNMD_TLS_CERT=/tls/server_cert.pem
- MNMD_TLS_KEY=/tls/server_key.pem
- MNMD_TLS_CLIENT_CA=/tls/ca.crt
rustfs:
image: ghcr.io/rustfs/rustfs:latest
container_name: rustfs
depends_on:
- mnmd
environment:
- RUSTFS_TLS_PATH=/tls
- RUSTFS_TRUST_SYSTEM_CA=false
- RUSTFS_TRUST_LEAF_CERT_AS_CA=false
# Enable outbound mTLS (client identity) for MNMD
- RUSTFS_MTLS_CLIENT_CERT=/tls/client_cert.pem
- RUSTFS_MTLS_CLIENT_KEY=/tls/client_key.pem
# MNMD address configured to https
- RUSTFS_MNMD_ADDR=https://mnmd:8443
- RUSTFS_MNMD_DOMAIN=mnmd
volumes:
- ./tls:/tls:ro

63
docs/tls.md Normal file
View File

@@ -0,0 +1,63 @@
# TLS / mTLS configuration
RustFS supports TLS for serving HTTPS and for outbound gRPC connections (MNMD).
It also supports optional client certificate authentication (mTLS) for outbound gRPC:
if a client identity is configured, RustFS will present it; otherwise it will use
server-authenticated TLS only.
## Recommended `tls/` directory layout
Place these files in a directory (default: `./tls`, configurable via `RUSTFS_TLS_PATH`).
```
TLS_DIR/
ca.crt # PEM bundle of CA/root certificates to trust (recommended)
public.crt # optional extra root bundle (PEM)
rustfs_cert.pem # server leaf certificate (PEM) used by the RustFS server
rustfs_key.pem # server private key (PEM) used by the RustFS server
# Optional: outbound mTLS client identity for MNMD
client_cert.pem # client certificate chain (PEM)
client_key.pem # client private key (PEM)
# Optional: server-side mTLS (inbound client certificate verification)
client_ca.crt # PEM bundle of CA certificates to verify client certificates
```
## Environment variables
### Root trust
- `RUSTFS_TLS_PATH` (default: `tls`): TLS directory.
- `RUSTFS_TRUST_SYSTEM_CA` (default: `false`): When `true`, include the platform/system
trust store as additional roots. When `false`, system roots are not used.
- `RUSTFS_TRUST_LEAF_CERT_AS_CA` (default: `false`): Compatibility switch. If `true`,
RustFS will also load `rustfs_cert.pem` into the root store (treating leaf certificates
as trusted roots). Prefer providing `ca.crt` instead.
### Outbound mTLS identity
- `RUSTFS_MTLS_CLIENT_CERT` (default: `${RUSTFS_TLS_PATH}/client_cert.pem`): path to PEM client cert/chain.
- `RUSTFS_MTLS_CLIENT_KEY` (default: `${RUSTFS_TLS_PATH}/client_key.pem`): path to PEM private key.
If both files exist, RustFS enables outbound mTLS. If either is missing, RustFS proceeds
with server-only TLS.
### Server-side mTLS (inbound client certificate verification)
- `RUSTFS_SERVER_MTLS_ENABLE` (default: `false`): When `true`, the RustFS server requires
clients to present valid certificates signed by a trusted CA for authentication.
When enabled, RustFS loads client CA certificates from:
1. `${RUSTFS_TLS_PATH}/client_ca.crt` (preferred)
2. `${RUSTFS_TLS_PATH}/ca.crt` (fallback if `client_ca.crt` does not exist)
**Important**: Server mTLS is disabled by default. When enabled but no valid CA bundle is
found, RustFS will fail to start with a clear error message. This ensures that server mTLS
cannot be accidentally enabled without proper client CA configuration.
## Failure mode for HTTPS without roots
When connecting to an `https://` MNMD address, RustFS requires at least one configured
trusted root. If none are loaded (no `ca.crt`/`public.crt` and system roots disabled),
RustFS fails fast with a clear error message.

2
rustfs/Cargo.toml
View File

@@ -65,7 +65,6 @@ rustfs-zip = { workspace = true }
# Async Runtime and Networking
async-trait = { workspace = true }
axum.workspace = true
axum-extra = { workspace = true }
axum-server = { workspace = true }
futures.workspace = true
futures-util.workspace = true
@@ -95,6 +94,7 @@ serde_urlencoded = { workspace = true }
# Cryptography and Security
rustls = { workspace = true }
subtle = { workspace = true }
rustls-pemfile = { workspace = true }
# Time and Date
chrono = { workspace = true }

14
rustfs/src/main.rs
View File

@@ -95,7 +95,9 @@ async fn async_main() -> Result<()> {
// Store in global storage
match set_global_guard(guard).map_err(Error::other) {
Ok(_) => (),
Ok(_) => {
info!(target: "rustfs::main", "Global observability guard set successfully.");
}
Err(e) => {
error!("Failed to set global observability guard: {}", e);
return Err(e);
@@ -110,7 +112,15 @@ async fn async_main() -> Result<()> {
// Initialize TLS if a certificate path is provided
if let Some(tls_path) = &opt.tls_path {
init_cert(tls_path).await
match init_cert(tls_path).await {
Ok(_) => {
info!(target: "rustfs::main", "TLS initialized successfully with certs from {}", tls_path);
}
Err(e) => {
error!("Failed to initialize TLS from {}: {}", tls_path, e);
return Err(Error::other(e));
}
}
}
// Run parameters

174
rustfs/src/server/cert.rs
View File

@@ -12,34 +12,129 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_common::set_global_root_cert;
use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert};
use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_PUBLIC_CERT, RUSTFS_TLS_CERT};
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use std::path::{Path, PathBuf};
use tracing::{debug, info};
/// Initialize TLS certificates for inter-node communication.
/// This function attempts to load certificates from the specified `tls_path`.
/// It looks for `rustfs_cert.pem`, `public.crt`, and `ca.crt` files.
/// Additionally, it tries to load system root certificates from common locations
/// to ensure trust for public CAs when mixing self-signed and public certificates.
/// If any certificates are found, they are set as the global root certificates.
pub(crate) async fn init_cert(tls_path: &str) {
#[derive(Debug)]
pub enum RustFSError {
Cert(String),
}
impl std::fmt::Display for RustFSError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RustFSError::Cert(msg) => write!(f, "Certificate error: {}", msg),
}
}
}
impl std::error::Error for RustFSError {}
/// Parse PEM-encoded certificates into DER format.
/// Returns a vector of DER-encoded certificates.
///
/// # Arguments
/// * `pem` - A byte slice containing the PEM-encoded certificates.
///
/// # Returns
/// A vector of `CertificateDer` containing the DER-encoded certificates.
///
/// # Errors
/// Returns `RustFSError` if parsing fails.
fn parse_pem_certs(pem: &[u8]) -> Result<Vec<CertificateDer<'static>>, RustFSError> {
let mut out = Vec::new();
let mut reader = std::io::Cursor::new(pem);
for item in rustls_pemfile::certs(&mut reader) {
let c = item.map_err(|e| RustFSError::Cert(format!("parse cert pem: {e}")))?;
out.push(c);
}
Ok(out)
}
/// Parse a PEM-encoded private key into DER format.
/// Supports PKCS#8 and RSA private keys.
///
/// # Arguments
/// * `pem` - A byte slice containing the PEM-encoded private key.
///
/// # Returns
/// A `PrivateKeyDer` containing the DER-encoded private key.
///
/// # Errors
/// Returns `RustFSError` if parsing fails or no key is found.
fn parse_pem_private_key(pem: &[u8]) -> Result<PrivateKeyDer<'static>, RustFSError> {
let mut reader = std::io::Cursor::new(pem);
let key = rustls_pemfile::private_key(&mut reader).map_err(|e| RustFSError::Cert(format!("parse private key pem: {e}")))?;
key.ok_or_else(|| RustFSError::Cert("no private key found in PEM".into()))
}
/// Helper function to read a file and return its contents.
/// Returns the file contents as a vector of bytes.
/// # Errors
/// Returns `RustFSError` if reading fails.
async fn read_file(path: &PathBuf, desc: &str) -> Result<Vec<u8>, RustFSError> {
tokio::fs::read(path)
.await
.map_err(|e| RustFSError::Cert(format!("read {} {:?}: {e}", desc, path)))
}
/// Initialize TLS material for both server and outbound client connections.
///
/// Loads roots from:
/// - `${RUSTFS_TLS_PATH}/ca.crt` (or `tls/ca.crt`)
/// - `${RUSTFS_TLS_PATH}/public.crt` (optional additional root bundle)
/// - system roots if `RUSTFS_TRUST_SYSTEM_CA=true` (default: false)
/// - if `RUSTFS_TRUST_LEAF_CERT_AS_CA=true`, also loads leaf cert(s) from
/// `${RUSTFS_TLS_PATH}/rustfs_cert.pem` into the root store.
///
/// Loads mTLS client identity (optional) from:
/// - `${RUSTFS_TLS_PATH}/client_cert.pem`
/// - `${RUSTFS_TLS_PATH}/client_key.pem`
///
/// Environment overrides:
/// - RUSTFS_TLS_PATH
/// - RUSTFS_MTLS_CLIENT_CERT
/// - RUSTFS_MTLS_CLIENT_KEY
pub(crate) async fn init_cert(tls_path: &str) -> Result<(), RustFSError> {
if tls_path.is_empty() {
info!("No TLS path configured; skipping certificate initialization");
return Ok(());
}
let tls_dir = PathBuf::from(tls_path);
// Load root certificates
load_root_certs(&tls_dir).await?;
// Load optional mTLS identity
load_mtls_identity(&tls_dir).await?;
Ok(())
}
/// Load root certificates from various sources.
async fn load_root_certs(tls_dir: &Path) -> Result<(), RustFSError> {
let mut cert_data = Vec::new();
// Try rustfs_cert.pem (custom cert name)
walk_dir(std::path::PathBuf::from(tls_path), RUSTFS_TLS_CERT, &mut cert_data).await;
let trust_leaf_as_ca =
rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA);
if trust_leaf_as_ca {
walk_dir(tls_dir.to_path_buf(), RUSTFS_TLS_CERT, &mut cert_data).await;
info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA");
}
// Try public.crt (common CA name)
let public_cert_path = std::path::Path::new(tls_path).join(RUSTFS_PUBLIC_CERT);
// Try public.crt and ca.crt
let public_cert_path = tls_dir.join(RUSTFS_PUBLIC_CERT);
load_cert_file(public_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
// Try ca.crt (common CA name)
let ca_cert_path = std::path::Path::new(tls_path).join(RUSTFS_CA_CERT);
let ca_cert_path = tls_dir.join(RUSTFS_CA_CERT);
load_cert_file(ca_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
// Load system root certificates if enabled
let trust_system_ca = rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_SYSTEM_CA, rustfs_config::DEFAULT_TRUST_SYSTEM_CA);
if !trust_system_ca {
// Attempt to load system root certificates to maintain trust for public CAs
// This is important when mixing self-signed internal certs with public external certs
if trust_system_ca {
let system_ca_paths = [
"/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Alpine
"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL/CentOS
@@ -57,7 +152,7 @@ pub(crate) async fn init_cert(tls_path: &str) {
if load_cert_file(path, &mut cert_data, "system root certificates").await {
system_cert_loaded = true;
info!("Loaded system root certificates from {}", path);
break; // Stop after finding the first valid bundle
break;
}
}
@@ -67,10 +162,51 @@ pub(crate) async fn init_cert(tls_path: &str) {
} else {
info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA");
}
if !cert_data.is_empty() {
set_global_root_cert(cert_data).await;
info!("Configured custom root certificates for inter-node communication");
}
Ok(())
}
/// Load optional mTLS identity.
async fn load_mtls_identity(tls_dir: &Path) -> Result<(), RustFSError> {
let client_cert_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_CERT) {
Some(p) => PathBuf::from(p),
None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME),
};
let client_key_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_KEY) {
Some(p) => PathBuf::from(p),
None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME),
};
if client_cert_path.exists() && client_key_path.exists() {
let cert_bytes = read_file(&client_cert_path, "client cert").await?;
let key_bytes = read_file(&client_key_path, "client key").await?;
// Validate parse-ability early; store as PEM bytes for tonic.
parse_pem_certs(&cert_bytes)?;
parse_pem_private_key(&key_bytes)?;
let identity_pem = MtlsIdentityPem {
cert_pem: cert_bytes,
key_pem: key_bytes,
};
set_global_mtls_identity(Some(identity_pem)).await;
info!("Loaded mTLS client identity cert={:?} key={:?}", client_cert_path, client_key_path);
} else {
set_global_mtls_identity(None).await;
info!(
"mTLS client identity not configured (missing {:?} and/or {:?}); proceeding with server-only TLS",
client_cert_path, client_key_path
);
}
Ok(())
}
/// Helper function to load a certificate file and append to cert_data.
@@ -114,7 +250,7 @@ async fn load_if_matches(entry: &tokio::fs::DirEntry, cert_name: &str, cert_data
/// - `path`: The starting directory path to search for certificates.
/// - `cert_name`: The name of the certificate file to look for.
/// - `cert_data`: A mutable vector to append loaded certificate data.
async fn walk_dir(path: std::path::PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
if let Ok(mut rd) = tokio::fs::read_dir(&path).await {
while let Ok(Some(entry)) = rd.next_entry().await {
if let Ok(ft) = entry.file_type().await {

29
rustfs/src/server/http.rs
View File

@@ -431,11 +431,11 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
debug!("TLS path is not provided or does not exist, starting with HTTP");
return Ok(None);
}
debug!("Found TLS directory, checking for certificates");
// Make sure to use a modern encryption suite
let _ = rustls::crypto::ring::default_provider().install_default();
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?;
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) {
@@ -446,9 +446,15 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?;
// Configure the server to enable SNI support
let mut server_config = ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver));
let mut server_config = if let Some(verifier) = mtls_verifier.clone() {
ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_cert_resolver(Arc::new(resolver))
} else {
ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver))
};
// Configure ALPN protocol priority
server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
@@ -470,10 +476,17 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
let certs = rustfs_utils::load_certs(&cert_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
let key = rustfs_utils::load_private_key(&key_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
let mut server_config = ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
let mut server_config = if let Some(verifier) = mtls_verifier {
ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(certs, key)
.map_err(|e| rustfs_utils::certs_error(e.to_string()))?
} else {
ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| rustfs_utils::certs_error(e.to_string()))?
};
// Configure ALPN protocol priority
server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];