From f3a1431fa57ea8c2103bd2da67e9c86919ffe88f Mon Sep 17 00:00:00 2001 From: loverustfs Date: Sun, 21 Dec 2025 16:11:55 +0800 Subject: [PATCH] fix: resolve TLS handshake failure in inter-node communication (#1201) (#1222) Co-authored-by: houseme --- Cargo.lock | 109 +++++++------------- Cargo.toml | 12 +-- crates/common/src/globals.rs | 5 + crates/config/src/constants/app.rs | 24 +++++ crates/config/src/constants/tls.rs | 22 ++++ crates/protos/src/lib.rs | 49 +++++++-- crates/utils/src/certs.rs | 14 +-- rustfs/src/main.rs | 17 +-- rustfs/src/server/cert.rs | 160 +++++++++++++++++++++++++++++ rustfs/src/server/mod.rs | 2 + scripts/run.sh | 3 + 11 files changed, 313 insertions(+), 104 deletions(-) create mode 100644 rustfs/src/server/cert.rs diff --git a/Cargo.lock b/Cargo.lock index 7ada333c..4f7d153b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,9 +1032,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ "axum-core", "bytes", @@ -1084,9 +1084,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfe9f610fe4e99cf0cfcd03ccf8c63c28c616fe714d80475ef731f3b13dd21b" +checksum = "6dfbd6109d91702d55fc56df06aae7ed85c465a7a451db6c0e54a4b9ca5983d1" dependencies = [ "axum", "axum-core", @@ -1434,31 +1434,14 @@ dependencies = [ "serde_core", ] -[[package]] -name = "cargo-util-schemas" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dc1a6f7b5651af85774ae5a34b4e8be397d9cf4bc063b7e6dbd99a841837830" -dependencies = [ - "semver", - "serde", - "serde-untagged", - "serde-value", - "thiserror 2.0.17", - "toml", - "unicode-xid", - "url", -] - [[package]] name = "cargo_metadata" -version = "0.22.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3f56c207c76c07652489840ff98687dcf213de178ac0974660d6fefeaf5ec6" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" dependencies = [ "camino", "cargo-platform", - "cargo-util-schemas", "semver", "serde", "serde_json", @@ -1473,9 +1456,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.49" +version = "1.2.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" dependencies = [ "find-msvc-tools", "jobserver", @@ -1576,7 +1559,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -1798,9 +1781,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.4.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -1965,9 +1948,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -2997,7 +2980,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3405,9 +3388,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.9.23" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags 2.10.0", "rustc_version", @@ -3607,9 +3590,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4641,9 +4624,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" [[package]] name = "jemalloc_pprof" @@ -4972,9 +4955,9 @@ dependencies = [ [[package]] name = "lzma-rust2" -version = "0.13.0" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +checksum = "48172246aa7c3ea28e423295dd1ca2589a24617cc4e588bb8cfe177cb2c54d95" dependencies = [ "crc", "sha2 0.10.9", @@ -5134,9 +5117,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.11" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" +checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" dependencies = [ "async-lock", "crossbeam-channel", @@ -5147,7 +5130,6 @@ dependencies = [ "futures-util", "parking_lot", "portable-atomic", - "rustc_version", "smallvec", "tagptr", "uuid", @@ -5281,9 +5263,9 @@ checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" [[package]] name = "ntapi" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" dependencies = [ "winapi", ] @@ -6113,9 +6095,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd" [[package]] name = "potential_utf" @@ -7879,9 +7861,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" [[package]] name = "s3s" @@ -8096,28 +8078,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-untagged" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" -dependencies = [ - "erased-serde", - "serde", - "serde_core", - "typeid", -] - -[[package]] -name = "serde-value" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" -dependencies = [ - "ordered-float", - "serde", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -8315,9 +8275,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d18183cef626bce22836103349c7050d73db799be0171386b80947d157ae32" +checksum = "ff351910f271e7065781b6b4f0f43cb515d474d812f31176a0246d9058e47d5d" dependencies = [ "cargo_metadata", "const_format", @@ -10434,9 +10394,9 @@ dependencies = [ [[package]] name = "zip" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037" dependencies = [ "aes 0.8.4", "arbitrary", @@ -10445,6 +10405,7 @@ dependencies = [ "crc32fast", "deflate64", "flate2", + "generic-array", "getrandom 0.3.4", "hmac 0.12.1", "indexmap 2.12.1", diff --git a/Cargo.toml b/Cargo.toml index a93368d1..6f0d3a32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,8 +97,8 @@ async-channel = "2.5.0" async-compression = { version = "0.4.19" } async-recursion = "1.1.1" async-trait = "0.1.89" -axum = "0.8.7" -axum-extra = "0.12.2" +axum = "0.8.8" +axum-extra = "0.12.3" axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false } futures = "0.3.31" futures-core = "0.3.31" @@ -126,7 +126,7 @@ tower-http = { version = "0.6.8", features = ["cors"] } bytes = { version = "1.11.0", features = ["serde"] } bytesize = "2.3.1" byteorder = "1.5.0" -flatbuffers = "25.9.23" +flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.1" quick-xml = "0.38.4" @@ -203,7 +203,7 @@ matchit = "0.9.0" md-5 = "0.11.0-rc.3" md5 = "0.8.0" mime_guess = "2.0.5" -moka = { version = "0.12.11", features = ["future"] } +moka = { version = "0.12.12", features = ["future"] } netif = "0.1.6" nix = { version = "0.30.1", features = ["fs"] } nu-ansi-term = "0.50.3" @@ -224,7 +224,7 @@ rust-embed = { version = "8.9.0" } rustc-hash = { version = "2.1.1" } s3s = { version = "0.12.0-rc.6", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", branch = "main" } serial_test = "3.2.0" -shadow-rs = { version = "1.4.0", default-features = false } +shadow-rs = { version = "1.5.0", default-features = false } siphasher = "1.0.1" smallvec = { version = "1.15.1", features = ["serde"] } smartstring = "1.0.1" @@ -252,7 +252,7 @@ walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } winapi = { version = "0.3.9" } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } -zip = "6.0.0" +zip = "7.0.0" zstd = "0.13.3" # Observability and Metrics diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index 6bcc7e29..e0f6a38a 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -24,11 +24,16 @@ pub static GLOBAL_RUSTFS_HOST: LazyLock> = LazyLock::new(|| RwLoc pub static GLOBAL_RUSTFS_PORT: LazyLock> = LazyLock::new(|| RwLock::new("9000".to_string())); pub static GLOBAL_RUSTFS_ADDR: LazyLock> = LazyLock::new(|| RwLock::new("".to_string())); pub static GLOBAL_CONN_MAP: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); +pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); pub async fn set_global_addr(addr: &str) { *GLOBAL_RUSTFS_ADDR.write().await = addr.to_string(); } +pub async fn set_global_root_cert(cert: Vec) { + *GLOBAL_ROOT_CERT.write().await = Some(cert); +} + /// Evict a stale/dead connection from the global connection cache. /// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off). /// By removing the cached connection, subsequent requests will establish a fresh connection. diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index f62b6407..0610319e 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -89,6 +89,30 @@ pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem"; /// This is the default cert for TLS. pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem"; +/// Default public certificate filename for rustfs +/// This is the default public certificate filename for rustfs. +/// It is used to store the public certificate of the application. +/// Default value: public.crt +pub const RUSTFS_PUBLIC_CERT: &str = "public.crt"; + +/// Default CA certificate filename for rustfs +/// This is the default CA certificate filename for rustfs. +/// It is used to store the CA certificate of the application. +/// Default value: ca.crt +pub const RUSTFS_CA_CERT: &str = "ca.crt"; + +/// Default HTTP prefix for rustfs +/// This is the default HTTP prefix for rustfs. +/// It is used to identify HTTP URLs. +/// Default value: http:// +pub const RUSTFS_HTTP_PREFIX: &str = "http://"; + +/// Default HTTPS prefix for rustfs +/// This is the default HTTPS prefix for rustfs. +/// It is used to identify HTTPS URLs. +/// Default value: https:// +pub const RUSTFS_HTTPS_PREFIX: &str = "https://"; + /// Default port for rustfs /// This is the default port for rustfs. /// This is used to bind the server to a specific port. diff --git a/crates/config/src/constants/tls.rs b/crates/config/src/constants/tls.rs index cfda42e2..6cbebcd4 100644 --- a/crates/config/src/constants/tls.rs +++ b/crates/config/src/constants/tls.rs @@ -12,4 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. +/// TLS related environment variable names and default values +/// Environment variable to enable TLS key logging +/// When set to "1", RustFS will log TLS keys to the specified file for debugging purposes. +/// By default, this is disabled. +/// To enable, set the environment variable RUSTFS_TLS_KEYLOG=1 pub const ENV_TLS_KEYLOG: &str = "RUSTFS_TLS_KEYLOG"; + +/// Default value for TLS key logging +/// By default, RustFS does not log TLS keys. +/// To change this behavior, set the environment variable RUSTFS_TLS_KEYLOG=1 +pub const DEFAULT_TLS_KEYLOG: bool = false; + +/// Environment variable to trust system CA certificates +/// When set to "1", RustFS will trust system CA certificates in addition to any +/// custom CA certificates provided in the configuration. +/// By default, this is disabled. +/// To enable, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1 +pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA"; + +/// Default value for trusting system CA certificates +/// By default, RustFS does not trust system CA certificates. +/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1 +pub const DEFAULT_TRUST_SYSTEM_CA: bool = false; diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 305d67a5..9b3a2aa4 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -15,19 +15,19 @@ #[allow(unsafe_code)] mod generated; -use std::{error::Error, time::Duration}; - -pub use generated::*; use proto_gen::node_service::node_service_client::NodeServiceClient; -use rustfs_common::globals::{GLOBAL_CONN_MAP, evict_connection}; +use rustfs_common::globals::{GLOBAL_CONN_MAP, GLOBAL_ROOT_CERT, evict_connection}; +use std::{error::Error, time::Duration}; use tonic::{ Request, Status, metadata::MetadataValue, service::interceptor::InterceptedService, - transport::{Channel, Endpoint}, + transport::{Certificate, Channel, ClientTlsConfig, Endpoint}, }; use tracing::{debug, warn}; +pub use generated::*; + // Default 100 MB pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024; @@ -46,6 +46,12 @@ const HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3; /// Overall RPC timeout - maximum time for any single RPC operation const RPC_TIMEOUT_SECS: u64 = 30; +/// Default HTTPS prefix for rustfs +/// This is the default HTTPS prefix for rustfs. +/// It is used to identify HTTPS URLs. +/// Default value: https:// +const RUSTFS_HTTPS_PREFIX: &str = "https://"; + /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience. /// /// This function is designed to detect dead peers quickly: @@ -56,7 +62,7 @@ const RPC_TIMEOUT_SECS: u64 = 30; async fn create_new_channel(addr: &str) -> Result> { debug!("Creating new gRPC channel to: {}", addr); - let connector = Endpoint::from_shared(addr.to_string())? + let mut connector = Endpoint::from_shared(addr.to_string())? // Fast connection timeout for dead peer detection .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS)) // TCP-level keepalive - OS will probe connection @@ -70,6 +76,37 @@ async fn create_new_channel(addr: &str) -> Result> { // Overall timeout for any RPC - fail fast on unresponsive peers .timeout(Duration::from_secs(RPC_TIMEOUT_SECS)); + let root_cert = GLOBAL_ROOT_CERT.read().await; + if addr.starts_with(RUSTFS_HTTPS_PREFIX) { + if let Some(cert_pem) = root_cert.as_ref() { + let ca = Certificate::from_pem(cert_pem); + // Derive the hostname from the HTTPS URL for TLS hostname verification. + let domain = addr + .trim_start_matches(RUSTFS_HTTPS_PREFIX) + .split('/') + .next() + .unwrap_or("") + .split(':') + .next() + .unwrap_or(""); + let tls = if !domain.is_empty() { + ClientTlsConfig::new().ca_certificate(ca).domain_name(domain) + } else { + // Fallback: configure TLS without explicit domain if parsing fails. + ClientTlsConfig::new().ca_certificate(ca) + }; + connector = connector.tls_config(tls)?; + debug!("Configured TLS with custom root certificate for: {}", addr); + } else { + debug!("Using system root certificates for TLS: {}", addr); + } + } else { + // Custom root certificates are configured but will be ignored for non-HTTPS addresses. + if root_cert.is_some() { + warn!("Custom root certificates are configured but not used because the address does not use HTTPS: {addr}"); + } + } + let channel = connector.connect().await?; // Cache the new connection diff --git a/crates/utils/src/certs.rs b/crates/utils/src/certs.rs index 24657f7a..463874ed 100644 --- a/crates/utils/src/certs.rs +++ b/crates/utils/src/certs.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::io::Error; use std::path::Path; use std::sync::Arc; -use std::{env, fs, io}; +use std::{fs, io}; use tracing::{debug, warn}; /// Load public certificate from file. @@ -243,17 +243,7 @@ pub fn create_multi_cert_resolver( /// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable. /// pub fn tls_key_log() -> bool { - env::var("RUSTFS_TLS_KEYLOG") - .map(|v| { - let v = v.trim(); - v.eq_ignore_ascii_case("1") - || v.eq_ignore_ascii_case("on") - || v.eq_ignore_ascii_case("true") - || v.eq_ignore_ascii_case("yes") - || v.eq_ignore_ascii_case("enabled") - || v.eq_ignore_ascii_case("t") - }) - .unwrap_or(false) + crate::get_env_bool(rustfs_config::ENV_TLS_KEYLOG, rustfs_config::DEFAULT_TLS_KEYLOG) } #[cfg(test)] diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index d62777bb..fbc946cb 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -27,7 +27,7 @@ mod version; // Ensure the correct path for parse_license is imported use crate::init::{add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system, init_update_check}; use crate::server::{ - SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_event_notifier, shutdown_event_notifier, + SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_cert, init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server, stop_audit_system, wait_for_shutdown, }; use chrono::Datelike; @@ -38,19 +38,19 @@ use rustfs_ahm::{ scanner::data_scanner::ScannerConfig, shutdown_ahm_services, }; use rustfs_common::globals::set_global_addr; -use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys; -use rustfs_ecstore::bucket::replication::{GLOBAL_REPLICATION_POOL, init_background_replication}; -use rustfs_ecstore::config as ecconfig; -use rustfs_ecstore::config::GLOBAL_CONFIG_SYS; -use rustfs_ecstore::store_api::BucketOptions; use rustfs_ecstore::{ StorageAPI, + bucket::metadata_sys::init_bucket_metadata_sys, + bucket::replication::{GLOBAL_REPLICATION_POOL, init_background_replication}, + config as ecconfig, + config::GLOBAL_CONFIG_SYS, endpoints::EndpointServerPools, global::{set_global_rustfs_port, shutdown_background_services}, notification_sys::new_global_notification_sys, set_global_endpoints, store::ECStore, store::init_local_disks, + store_api::BucketOptions, update_erasure_type, }; use rustfs_iam::init_iam_sys; @@ -125,6 +125,11 @@ async fn async_main() -> Result<()> { // Initialize performance profiling if enabled profiling::init_from_env().await; + // Initialize TLS if a certificate path is provided + if let Some(tls_path) = &opt.tls_path { + init_cert(tls_path).await + } + // Run parameters match run(opt).await { Ok(_) => Ok(()), diff --git a/rustfs/src/server/cert.rs b/rustfs/src/server/cert.rs new file mode 100644 index 00000000..6dba5c05 --- /dev/null +++ b/rustfs/src/server/cert.rs @@ -0,0 +1,160 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_common::globals::set_global_root_cert; +use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_PUBLIC_CERT, RUSTFS_TLS_CERT}; +use tracing::{debug, info}; + +/// Initialize TLS certificates for inter-node communication. +/// This function attempts to load certificates from the specified `tls_path`. +/// It looks for `rustfs_cert.pem`, `public.crt`, and `ca.crt` files. +/// Additionally, it tries to load system root certificates from common locations +/// to ensure trust for public CAs when mixing self-signed and public certificates. +/// If any certificates are found, they are set as the global root certificates. +pub(crate) async fn init_cert(tls_path: &str) { + let mut cert_data = Vec::new(); + + // Try rustfs_cert.pem (custom cert name) + walk_dir(std::path::PathBuf::from(tls_path), RUSTFS_TLS_CERT, &mut cert_data).await; + + // Try public.crt (common CA name) + let public_cert_path = std::path::Path::new(tls_path).join(RUSTFS_PUBLIC_CERT); + load_cert_file(public_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await; + + // Try ca.crt (common CA name) + let ca_cert_path = std::path::Path::new(tls_path).join(RUSTFS_CA_CERT); + load_cert_file(ca_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await; + + let trust_system_ca = rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_SYSTEM_CA, rustfs_config::DEFAULT_TRUST_SYSTEM_CA); + if !trust_system_ca { + // Attempt to load system root certificates to maintain trust for public CAs + // This is important when mixing self-signed internal certs with public external certs + let system_ca_paths = [ + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Alpine + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL/CentOS + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + "/etc/ssl/cert.pem", // macOS/FreeBSD + "/usr/local/etc/openssl/cert.pem", // macOS/Homebrew OpenSSL + "/usr/local/share/certs/ca-root-nss.crt", // FreeBSD + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // RHEL + "/usr/share/pki/ca-trust-legacy/ca-bundle.legacy.crt", // RHEL legacy + ]; + + let mut system_cert_loaded = false; + for path in system_ca_paths { + if load_cert_file(path, &mut cert_data, "system root certificates").await { + system_cert_loaded = true; + info!("Loaded system root certificates from {}", path); + break; // Stop after finding the first valid bundle + } + } + + if !system_cert_loaded { + debug!("Could not find system root certificates in common locations."); + } + } else { + info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA"); + } + if !cert_data.is_empty() { + set_global_root_cert(cert_data).await; + info!("Configured custom root certificates for inter-node communication"); + } +} + +/// Helper function to load a certificate file and append to cert_data. +/// Returns true if the file was successfully loaded. +async fn load_cert_file(path: &str, cert_data: &mut Vec, desc: &str) -> bool { + if tokio::fs::metadata(path).await.is_ok() { + if let Ok(data) = tokio::fs::read(path).await { + cert_data.extend(data); + cert_data.push(b'\n'); + info!("Loaded {} from {}", desc, path); + true + } else { + debug!("Failed to read {} from {}", desc, path); + false + } + } else { + debug!("{} file not found at {}", desc, path); + false + } +} + +/// Load the certificate file if its name matches `cert_name`. +/// If it matches, the certificate data is appended to `cert_data`. +/// +/// # Parameters +/// - `entry`: The directory entry to check. +/// - `cert_name`: The name of the certificate file to match. +/// - `cert_data`: A mutable vector to append loaded certificate data. +async fn load_if_matches(entry: &tokio::fs::DirEntry, cert_name: &str, cert_data: &mut Vec) { + let fname = entry.file_name().to_string_lossy().to_string(); + if fname == cert_name { + let p = entry.path(); + load_cert_file(&p.to_string_lossy(), cert_data, "certificate").await; + } +} + +/// Search the directory at `path` and one level of subdirectories to find and load +/// certificates matching `cert_name`. Loaded certificate data is appended to +/// `cert_data`. +/// # Parameters +/// - `path`: The starting directory path to search for certificates. +/// - `cert_name`: The name of the certificate file to look for. +/// - `cert_data`: A mutable vector to append loaded certificate data. +async fn walk_dir(path: std::path::PathBuf, cert_name: &str, cert_data: &mut Vec) { + if let Ok(mut rd) = tokio::fs::read_dir(&path).await { + while let Ok(Some(entry)) = rd.next_entry().await { + if let Ok(ft) = entry.file_type().await { + if ft.is_file() { + load_if_matches(&entry, cert_name, cert_data).await; + } else if ft.is_dir() { + // Only check direct subdirectories, no deeper recursion + if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await { + while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { + if let Ok(sub_ft) = sub_entry.file_type().await { + if sub_ft.is_file() { + load_if_matches(&sub_entry, cert_name, cert_data).await; + } + // Ignore subdirectories and symlinks in subdirs to limit to one level + } + } + } + } else if ft.is_symlink() { + // Follow symlink and treat target as file or directory, but limit to one level + if let Ok(meta) = tokio::fs::metadata(&entry.path()).await { + if meta.is_file() { + load_if_matches(&entry, cert_name, cert_data).await; + } else if meta.is_dir() { + // Treat as directory but only check its direct contents + if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await { + while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { + if let Ok(sub_ft) = sub_entry.file_type().await { + if sub_ft.is_file() { + load_if_matches(&sub_entry, cert_name, cert_data).await; + } + // Ignore deeper levels + } + } + } + } + } + } + } + } + } else { + debug!("Certificate directory not found: {}", path.display()); + } +} diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index df6b04a5..630f6f94 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod audit; +mod cert; mod compress; mod event; mod http; @@ -22,6 +23,7 @@ mod runtime; mod service_state; pub(crate) use audit::{start_audit_system, stop_audit_system}; +pub(crate) use cert::init_cert; pub(crate) use event::{init_event_notifier, shutdown_event_notifier}; pub(crate) use http::start_http_server; pub(crate) use runtime::get_tokio_runtime_builder; diff --git a/scripts/run.sh b/scripts/run.sh index 762215c6..6c268c37 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -183,6 +183,9 @@ export RUSTFS_ENABLE_PROFILING=false # Heal configuration queue size export RUSTFS_HEAL_QUEUE_SIZE=10000 +# rustfs trust system CA certificates +export RUSTFS_TRUST_SYSTEM_CA=true + if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi