sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-17 01:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: resolve TLS handshake failure in inter-node communication (#1201) (#1222)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
loverustfs
2025-12-21 16:11:55 +08:00
committed by GitHub
parent 3bd96bcf10
commit f3a1431fa5
11 changed files with 313 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
Cargo.lock generated
View File

@@ -1032,9 +1032,9 @@ dependencies = [
[[package]]
name = "axum"
version = "0.8.7"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425"
checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
dependencies = [
"axum-core",
"bytes",
@@ -1084,9 +1084,9 @@ dependencies = [
[[package]]
name = "axum-extra"
version = "0.12.2"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbfe9f610fe4e99cf0cfcd03ccf8c63c28c616fe714d80475ef731f3b13dd21b"
checksum = "6dfbd6109d91702d55fc56df06aae7ed85c465a7a451db6c0e54a4b9ca5983d1"
dependencies = [
"axum",
"axum-core",
@@ -1434,31 +1434,14 @@ dependencies = [
"serde_core",
]
[[package]]
name = "cargo-util-schemas"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dc1a6f7b5651af85774ae5a34b4e8be397d9cf4bc063b7e6dbd99a841837830"
dependencies = [
"semver",
"serde",
"serde-untagged",
"serde-value",
"thiserror 2.0.17",
"toml",
"unicode-xid",
"url",
]
[[package]]
name = "cargo_metadata"
version = "0.22.0"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c3f56c207c76c07652489840ff98687dcf213de178ac0974660d6fefeaf5ec6"
checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9"
dependencies = [
"camino",
"cargo-platform",
"cargo-util-schemas",
"semver",
"serde",
"serde_json",
@@ -1473,9 +1456,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.2.49"
version = "1.2.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1576,7 +1559,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"inout 0.1.4",
]
@@ -1798,9 +1781,9 @@ dependencies = [
[[package]]
name = "crc"
version = "3.4.0"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
dependencies = [
"crc-catalog",
]
@@ -1965,9 +1948,9 @@ dependencies = [
[[package]]
name = "crypto-common"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
@@ -2997,7 +2980,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid 0.9.6",
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -3405,9 +3388,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flatbuffers"
version = "25.9.23"
version = "25.12.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5"
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
dependencies = [
"bitflags 2.10.0",
"rustc_version",
@@ -3607,9 +3590,9 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.14.9"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -4641,9 +4624,9 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.15"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010"
[[package]]
name = "jemalloc_pprof"
@@ -4972,9 +4955,9 @@ dependencies = [
[[package]]
name = "lzma-rust2"
version = "0.13.0"
version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a"
checksum = "48172246aa7c3ea28e423295dd1ca2589a24617cc4e588bb8cfe177cb2c54d95"
dependencies = [
"crc",
"sha2 0.10.9",
@@ -5134,9 +5117,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.11"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077"
checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a"
dependencies = [
"async-lock",
"crossbeam-channel",
@@ -5147,7 +5130,6 @@ dependencies = [
"futures-util",
"parking_lot",
"portable-atomic",
"rustc_version",
"smallvec",
"tagptr",
"uuid",
@@ -5281,9 +5263,9 @@ checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d"
[[package]]
name = "ntapi"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081"
dependencies = [
"winapi",
]
@@ -6113,9 +6095,9 @@ dependencies = [
[[package]]
name = "portable-atomic"
version = "1.11.1"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd"
[[package]]
name = "potential_utf"
@@ -7879,9 +7861,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.20"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea"
[[package]]
name = "s3s"
@@ -8096,28 +8078,6 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-untagged"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058"
dependencies = [
"erased-serde",
"serde",
"serde_core",
"typeid",
]
[[package]]
name = "serde-value"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"
dependencies = [
"ordered-float",
"serde",
]
[[package]]
name = "serde_core"
version = "1.0.228"
@@ -8315,9 +8275,9 @@ dependencies = [
[[package]]
name = "shadow-rs"
version = "1.4.0"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72d18183cef626bce22836103349c7050d73db799be0171386b80947d157ae32"
checksum = "ff351910f271e7065781b6b4f0f43cb515d474d812f31176a0246d9058e47d5d"
dependencies = [
"cargo_metadata",
"const_format",
@@ -10434,9 +10394,9 @@ dependencies = [
[[package]]
name = "zip"
version = "6.0.0"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037"
dependencies = [
"aes 0.8.4",
"arbitrary",
@@ -10445,6 +10405,7 @@ dependencies = [
"crc32fast",
"deflate64",
"flate2",
"generic-array",
"getrandom 0.3.4",
"hmac 0.12.1",
"indexmap 2.12.1",

12
Cargo.toml
View File

@@ -97,8 +97,8 @@ async-channel = "2.5.0"
async-compression = { version = "0.4.19" }
async-recursion = "1.1.1"
async-trait = "0.1.89"
axum = "0.8.7"
axum-extra = "0.12.2"
axum = "0.8.8"
axum-extra = "0.12.3"
axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false }
futures = "0.3.31"
futures-core = "0.3.31"
@@ -126,7 +126,7 @@ tower-http = { version = "0.6.8", features = ["cors"] }
bytes = { version = "1.11.0", features = ["serde"] }
bytesize = "2.3.1"
byteorder = "1.5.0"
flatbuffers = "25.9.23"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.1"
quick-xml = "0.38.4"
@@ -203,7 +203,7 @@ matchit = "0.9.0"
md-5 = "0.11.0-rc.3"
md5 = "0.8.0"
mime_guess = "2.0.5"
moka = { version = "0.12.11", features = ["future"] }
moka = { version = "0.12.12", features = ["future"] }
netif = "0.1.6"
nix = { version = "0.30.1", features = ["fs"] }
nu-ansi-term = "0.50.3"
@@ -224,7 +224,7 @@ rust-embed = { version = "8.9.0" }
rustc-hash = { version = "2.1.1" }
s3s = { version = "0.12.0-rc.6", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", branch = "main" }
serial_test = "3.2.0"
shadow-rs = { version = "1.4.0", default-features = false }
shadow-rs = { version = "1.5.0", default-features = false }
siphasher = "1.0.1"
smallvec = { version = "1.15.1", features = ["serde"] }
smartstring = "1.0.1"
@@ -252,7 +252,7 @@ walkdir = "2.5.0"
wildmatch = { version = "2.6.1", features = ["serde"] }
winapi = { version = "0.3.9" }
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
zip = "6.0.0"
zip = "7.0.0"
zstd = "0.13.3"
# Observability and Metrics

5
crates/common/src/globals.rs
View File

@@ -24,11 +24,16 @@ pub static GLOBAL_RUSTFS_HOST: LazyLock<RwLock<String>> = LazyLock::new(|| RwLoc
pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("9000".to_string()));
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
pub async fn set_global_addr(addr: &str) {
*GLOBAL_RUSTFS_ADDR.write().await = addr.to_string();
}
pub async fn set_global_root_cert(cert: Vec<u8>) {
*GLOBAL_ROOT_CERT.write().await = Some(cert);
}
/// Evict a stale/dead connection from the global connection cache.
/// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off).
/// By removing the cached connection, subsequent requests will establish a fresh connection.

24
crates/config/src/constants/app.rs
View File

@@ -89,6 +89,30 @@ pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem";
/// This is the default cert for TLS.
pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem";
/// Default public certificate filename for rustfs
/// This is the default public certificate filename for rustfs.
/// It is used to store the public certificate of the application.
/// Default value: public.crt
pub const RUSTFS_PUBLIC_CERT: &str = "public.crt";
/// Default CA certificate filename for rustfs
/// This is the default CA certificate filename for rustfs.
/// It is used to store the CA certificate of the application.
/// Default value: ca.crt
pub const RUSTFS_CA_CERT: &str = "ca.crt";
/// Default HTTP prefix for rustfs
/// This is the default HTTP prefix for rustfs.
/// It is used to identify HTTP URLs.
/// Default value: http://
pub const RUSTFS_HTTP_PREFIX: &str = "http://";
/// Default HTTPS prefix for rustfs
/// This is the default HTTPS prefix for rustfs.
/// It is used to identify HTTPS URLs.
/// Default value: https://
pub const RUSTFS_HTTPS_PREFIX: &str = "https://";
/// Default port for rustfs
/// This is the default port for rustfs.
/// This is used to bind the server to a specific port.

22
crates/config/src/constants/tls.rs
View File

@@ -12,4 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
/// TLS related environment variable names and default values
/// Environment variable to enable TLS key logging
/// When set to "1", RustFS will log TLS keys to the specified file for debugging purposes.
/// By default, this is disabled.
/// To enable, set the environment variable RUSTFS_TLS_KEYLOG=1
pub const ENV_TLS_KEYLOG: &str = "RUSTFS_TLS_KEYLOG";
/// Default value for TLS key logging
/// By default, RustFS does not log TLS keys.
/// To change this behavior, set the environment variable RUSTFS_TLS_KEYLOG=1
pub const DEFAULT_TLS_KEYLOG: bool = false;
/// Environment variable to trust system CA certificates
/// When set to "1", RustFS will trust system CA certificates in addition to any
/// custom CA certificates provided in the configuration.
/// By default, this is disabled.
/// To enable, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
/// Default value for trusting system CA certificates
/// By default, RustFS does not trust system CA certificates.
/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;

49
crates/protos/src/lib.rs
View File

@@ -15,19 +15,19 @@
#[allow(unsafe_code)]
mod generated;
use std::{error::Error, time::Duration};
pub use generated::*;
use proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_common::globals::{GLOBAL_CONN_MAP, evict_connection};
use rustfs_common::globals::{GLOBAL_CONN_MAP, GLOBAL_ROOT_CERT, evict_connection};
use std::{error::Error, time::Duration};
use tonic::{
Request, Status,
metadata::MetadataValue,
service::interceptor::InterceptedService,
transport::{Channel, Endpoint},
transport::{Certificate, Channel, ClientTlsConfig, Endpoint},
};
use tracing::{debug, warn};
pub use generated::*;
// Default 100 MB
pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
@@ -46,6 +46,12 @@ const HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
/// Overall RPC timeout - maximum time for any single RPC operation
const RPC_TIMEOUT_SECS: u64 = 30;
/// Default HTTPS prefix for rustfs
/// This is the default HTTPS prefix for rustfs.
/// It is used to identify HTTPS URLs.
/// Default value: https://
const RUSTFS_HTTPS_PREFIX: &str = "https://";
/// Creates a new gRPC channel with optimized keepalive settings for cluster resilience.
///
/// This function is designed to detect dead peers quickly:
@@ -56,7 +62,7 @@ const RPC_TIMEOUT_SECS: u64 = 30;
async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
debug!("Creating new gRPC channel to: {}", addr);
let connector = Endpoint::from_shared(addr.to_string())?
let mut connector = Endpoint::from_shared(addr.to_string())?
// Fast connection timeout for dead peer detection
.connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
// TCP-level keepalive - OS will probe connection
@@ -70,6 +76,37 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
// Overall timeout for any RPC - fail fast on unresponsive peers
.timeout(Duration::from_secs(RPC_TIMEOUT_SECS));
let root_cert = GLOBAL_ROOT_CERT.read().await;
if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
if let Some(cert_pem) = root_cert.as_ref() {
let ca = Certificate::from_pem(cert_pem);
// Derive the hostname from the HTTPS URL for TLS hostname verification.
let domain = addr
.trim_start_matches(RUSTFS_HTTPS_PREFIX)
.split('/')
.next()
.unwrap_or("")
.split(':')
.next()
.unwrap_or("");
let tls = if !domain.is_empty() {
ClientTlsConfig::new().ca_certificate(ca).domain_name(domain)
} else {
// Fallback: configure TLS without explicit domain if parsing fails.
ClientTlsConfig::new().ca_certificate(ca)
};
connector = connector.tls_config(tls)?;
debug!("Configured TLS with custom root certificate for: {}", addr);
} else {
debug!("Using system root certificates for TLS: {}", addr);
}
} else {
// Custom root certificates are configured but will be ignored for non-HTTPS addresses.
if root_cert.is_some() {
warn!("Custom root certificates are configured but not used because the address does not use HTTPS: {addr}");
}
}
let channel = connector.connect().await?;
// Cache the new connection

14
crates/utils/src/certs.rs
View File

@@ -21,7 +21,7 @@ use std::collections::HashMap;
use std::io::Error;
use std::path::Path;
use std::sync::Arc;
use std::{env, fs, io};
use std::{fs, io};
use tracing::{debug, warn};
/// Load public certificate from file.
@@ -243,17 +243,7 @@ pub fn create_multi_cert_resolver(
/// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable.
///
pub fn tls_key_log() -> bool {
env::var("RUSTFS_TLS_KEYLOG")
.map(|v| {
let v = v.trim();
v.eq_ignore_ascii_case("1")
|| v.eq_ignore_ascii_case("on")
|| v.eq_ignore_ascii_case("true")
|| v.eq_ignore_ascii_case("yes")
|| v.eq_ignore_ascii_case("enabled")
|| v.eq_ignore_ascii_case("t")
})
.unwrap_or(false)
crate::get_env_bool(rustfs_config::ENV_TLS_KEYLOG, rustfs_config::DEFAULT_TLS_KEYLOG)
}
#[cfg(test)]

17
rustfs/src/main.rs
View File

@@ -27,7 +27,7 @@ mod version;
// Ensure the correct path for parse_license is imported
use crate::init::{add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system, init_update_check};
use crate::server::{
SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_event_notifier, shutdown_event_notifier,
SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_cert, init_event_notifier, shutdown_event_notifier,
start_audit_system, start_http_server, stop_audit_system, wait_for_shutdown,
};
use chrono::Datelike;
@@ -38,19 +38,19 @@ use rustfs_ahm::{
scanner::data_scanner::ScannerConfig, shutdown_ahm_services,
};
use rustfs_common::globals::set_global_addr;
use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
use rustfs_ecstore::bucket::replication::{GLOBAL_REPLICATION_POOL, init_background_replication};
use rustfs_ecstore::config as ecconfig;
use rustfs_ecstore::config::GLOBAL_CONFIG_SYS;
use rustfs_ecstore::store_api::BucketOptions;
use rustfs_ecstore::{
StorageAPI,
bucket::metadata_sys::init_bucket_metadata_sys,
bucket::replication::{GLOBAL_REPLICATION_POOL, init_background_replication},
config as ecconfig,
config::GLOBAL_CONFIG_SYS,
endpoints::EndpointServerPools,
global::{set_global_rustfs_port, shutdown_background_services},
notification_sys::new_global_notification_sys,
set_global_endpoints,
store::ECStore,
store::init_local_disks,
store_api::BucketOptions,
update_erasure_type,
};
use rustfs_iam::init_iam_sys;
@@ -125,6 +125,11 @@ async fn async_main() -> Result<()> {
// Initialize performance profiling if enabled
profiling::init_from_env().await;
// Initialize TLS if a certificate path is provided
if let Some(tls_path) = &opt.tls_path {
init_cert(tls_path).await
}
// Run parameters
match run(opt).await {
Ok(_) => Ok(()),

160
rustfs/src/server/cert.rs Normal file
View File

@@ -0,0 +1,160 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_common::globals::set_global_root_cert;
use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_PUBLIC_CERT, RUSTFS_TLS_CERT};
use tracing::{debug, info};
/// Initialize TLS certificates for inter-node communication.
/// This function attempts to load certificates from the specified `tls_path`.
/// It looks for `rustfs_cert.pem`, `public.crt`, and `ca.crt` files.
/// Additionally, it tries to load system root certificates from common locations
/// to ensure trust for public CAs when mixing self-signed and public certificates.
/// If any certificates are found, they are set as the global root certificates.
pub(crate) async fn init_cert(tls_path: &str) {
let mut cert_data = Vec::new();
// Try rustfs_cert.pem (custom cert name)
walk_dir(std::path::PathBuf::from(tls_path), RUSTFS_TLS_CERT, &mut cert_data).await;
// Try public.crt (common CA name)
let public_cert_path = std::path::Path::new(tls_path).join(RUSTFS_PUBLIC_CERT);
load_cert_file(public_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
// Try ca.crt (common CA name)
let ca_cert_path = std::path::Path::new(tls_path).join(RUSTFS_CA_CERT);
load_cert_file(ca_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
let trust_system_ca = rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_SYSTEM_CA, rustfs_config::DEFAULT_TRUST_SYSTEM_CA);
if !trust_system_ca {
// Attempt to load system root certificates to maintain trust for public CAs
// This is important when mixing self-signed internal certs with public external certs
let system_ca_paths = [
"/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Alpine
"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL/CentOS
"/etc/ssl/ca-bundle.pem", // OpenSUSE
"/etc/pki/tls/cacert.pem", // OpenELEC
"/etc/ssl/cert.pem", // macOS/FreeBSD
"/usr/local/etc/openssl/cert.pem", // macOS/Homebrew OpenSSL
"/usr/local/share/certs/ca-root-nss.crt", // FreeBSD
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // RHEL
"/usr/share/pki/ca-trust-legacy/ca-bundle.legacy.crt", // RHEL legacy
];
let mut system_cert_loaded = false;
for path in system_ca_paths {
if load_cert_file(path, &mut cert_data, "system root certificates").await {
system_cert_loaded = true;
info!("Loaded system root certificates from {}", path);
break; // Stop after finding the first valid bundle
}
}
if !system_cert_loaded {
debug!("Could not find system root certificates in common locations.");
}
} else {
info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA");
}
if !cert_data.is_empty() {
set_global_root_cert(cert_data).await;
info!("Configured custom root certificates for inter-node communication");
}
}
/// Helper function to load a certificate file and append to cert_data.
/// Returns true if the file was successfully loaded.
async fn load_cert_file(path: &str, cert_data: &mut Vec<u8>, desc: &str) -> bool {
if tokio::fs::metadata(path).await.is_ok() {
if let Ok(data) = tokio::fs::read(path).await {
cert_data.extend(data);
cert_data.push(b'\n');
info!("Loaded {} from {}", desc, path);
true
} else {
debug!("Failed to read {} from {}", desc, path);
false
}
} else {
debug!("{} file not found at {}", desc, path);
false
}
}
/// Load the certificate file if its name matches `cert_name`.
/// If it matches, the certificate data is appended to `cert_data`.
///
/// # Parameters
/// - `entry`: The directory entry to check.
/// - `cert_name`: The name of the certificate file to match.
/// - `cert_data`: A mutable vector to append loaded certificate data.
async fn load_if_matches(entry: &tokio::fs::DirEntry, cert_name: &str, cert_data: &mut Vec<u8>) {
let fname = entry.file_name().to_string_lossy().to_string();
if fname == cert_name {
let p = entry.path();
load_cert_file(&p.to_string_lossy(), cert_data, "certificate").await;
}
}
/// Search the directory at `path` and one level of subdirectories to find and load
/// certificates matching `cert_name`. Loaded certificate data is appended to
/// `cert_data`.
/// # Parameters
/// - `path`: The starting directory path to search for certificates.
/// - `cert_name`: The name of the certificate file to look for.
/// - `cert_data`: A mutable vector to append loaded certificate data.
async fn walk_dir(path: std::path::PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
if let Ok(mut rd) = tokio::fs::read_dir(&path).await {
while let Ok(Some(entry)) = rd.next_entry().await {
if let Ok(ft) = entry.file_type().await {
if ft.is_file() {
load_if_matches(&entry, cert_name, cert_data).await;
} else if ft.is_dir() {
// Only check direct subdirectories, no deeper recursion
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await {
if sub_ft.is_file() {
load_if_matches(&sub_entry, cert_name, cert_data).await;
}
// Ignore subdirectories and symlinks in subdirs to limit to one level
}
}
}
} else if ft.is_symlink() {
// Follow symlink and treat target as file or directory, but limit to one level
if let Ok(meta) = tokio::fs::metadata(&entry.path()).await {
if meta.is_file() {
load_if_matches(&entry, cert_name, cert_data).await;
} else if meta.is_dir() {
// Treat as directory but only check its direct contents
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await {
if sub_ft.is_file() {
load_if_matches(&sub_entry, cert_name, cert_data).await;
}
// Ignore deeper levels
}
}
}
}
}
}
}
}
} else {
debug!("Certificate directory not found: {}", path.display());
}
}

2
rustfs/src/server/mod.rs
View File

@@ -13,6 +13,7 @@
// limitations under the License.
mod audit;
mod cert;
mod compress;
mod event;
mod http;
@@ -22,6 +23,7 @@ mod runtime;
mod service_state;
pub(crate) use audit::{start_audit_system, stop_audit_system};
pub(crate) use cert::init_cert;
pub(crate) use event::{init_event_notifier, shutdown_event_notifier};
pub(crate) use http::start_http_server;
pub(crate) use runtime::get_tokio_runtime_builder;

3
scripts/run.sh
View File

@@ -183,6 +183,9 @@ export RUSTFS_ENABLE_PROFILING=false
# Heal configuration queue size
export RUSTFS_HEAL_QUEUE_SIZE=10000
# rustfs trust system CA certificates
export RUSTFS_TRUST_SYSTEM_CA=true
if [ -n "$1" ]; then
export RUSTFS_VOLUMES="$1"
fi