From b142563127ce1fe0e4b84ade403155c4a6928229 Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 5 Jan 2026 21:52:04 +0800 Subject: [PATCH 01/12] fix rpc client (#1393) --- crates/ecstore/src/admin_server_info.rs | 6 +- crates/ecstore/src/disk/disk_store.rs | 4 +- crates/ecstore/src/rpc/peer_s3_client.rs | 14 +- crates/ecstore/src/rpc/remote_disk.rs | 6 +- crates/lock/src/client/remote.rs | 403 ----------------------- rustfs/src/init.rs | 4 +- rustfs/src/server/cert.rs | 4 +- rustfs/src/storage/tonic_service.rs | 4 +- 8 files changed, 19 insertions(+), 426 deletions(-) delete mode 100644 crates/lock/src/client/remote.rs diff --git a/crates/ecstore/src/admin_server_info.rs b/crates/ecstore/src/admin_server_info.rs index 13187790..54f9b981 100644 --- a/crates/ecstore/src/admin_server_info.rs +++ b/crates/ecstore/src/admin_server_info.rs @@ -14,7 +14,7 @@ use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend}; use crate::error::{Error, Result}; -use crate::rpc::node_service_time_out_client_no_auth; +use crate::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::{ disk::endpoint::Endpoint, global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints}, @@ -101,9 +101,9 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> { let decoded_payload = flatbuffers::root::(finished_data); assert!(decoded_payload.is_ok()); - let mut client = node_service_time_out_client_no_auth(&addr) + let mut client = node_service_time_out_client(&addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())) .await - .map_err(|err| Error::other(err.to_string()))?; + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; let request = Request::new(PingRequest { version: 1, diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 0f8fbf4d..baab323f 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -384,7 +384,7 @@ impl LocalDiskWrapper { let stored_disk_id = self.disk.get_disk_id().await?; if stored_disk_id != want_id { - return Err(Error::other(format!("Disk ID mismatch wanted {:?}, got {:?}", want_id, stored_disk_id))); + return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}"))); } Ok(()) @@ -468,7 +468,7 @@ impl LocalDiskWrapper { // Timeout occurred, mark disk as potentially faulty and decrement waiting counter self.health.decrement_waiting(); warn!("disk operation timeout after {:?}", timeout_duration); - Err(DiskError::other(format!("disk operation timeout after {:?}", timeout_duration))) + Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}"))) } } } diff --git a/crates/ecstore/src/rpc/peer_s3_client.rs b/crates/ecstore/src/rpc/peer_s3_client.rs index 90d684eb..16522548 100644 --- a/crates/ecstore/src/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/rpc/peer_s3_client.rs @@ -18,9 +18,7 @@ use crate::disk::error::{Error, Result}; use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs}; use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration}; use crate::global::GLOBAL_LOCAL_DISK_MAP; -use crate::rpc::client::{ - TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, -}; +use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::store::all_local_disk; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{ @@ -675,7 +673,7 @@ impl RemotePeerS3Client { async fn perform_connectivity_check(addr: &str) -> Result<()> { use tokio::time::timeout; - let url = url::Url::parse(addr).map_err(|e| Error::other(format!("Invalid URL: {}", e)))?; + let url = url::Url::parse(addr).map_err(|e| Error::other(format!("Invalid URL: {e}")))?; let Some(host) = url.host_str() else { return Err(Error::other("No host in URL".to_string())); @@ -686,7 +684,7 @@ impl RemotePeerS3Client { // Try to establish TCP connection match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await { Ok(Ok(_)) => Ok(()), - _ => Err(Error::other(format!("Cannot connect to {}:{}", host, port))), + _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))), } } @@ -725,7 +723,7 @@ impl RemotePeerS3Client { // Timeout occurred, mark peer as potentially faulty self.health.decrement_waiting(); warn!("Remote peer operation timeout after {:?}", timeout_duration); - Err(Error::other(format!("Remote peer operation timeout after {:?}", timeout_duration))) + Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}"))) } } } @@ -823,9 +821,7 @@ impl PeerS3Client for RemotePeerS3Client { self.execute_with_timeout( || async { let options = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let mut client = self.get_client().await?; let request = Request::new(GetBucketInfoRequest { bucket: bucket.to_string(), options, diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index 78f39f55..f4f03468 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -206,7 +206,7 @@ impl RemoteDisk { /// Perform basic connectivity check for remote disk async fn perform_connectivity_check(addr: &str) -> Result<()> { - let url = url::Url::parse(addr).map_err(|e| Error::other(format!("Invalid URL: {}", e)))?; + let url = url::Url::parse(addr).map_err(|e| Error::other(format!("Invalid URL: {e}")))?; let Some(host) = url.host_str() else { return Err(Error::other("No host in URL".to_string())); @@ -220,7 +220,7 @@ impl RemoteDisk { drop(stream); Ok(()) } - _ => Err(Error::other(format!("Cannot connect to {}:{}", host, port))), + _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))), } } @@ -260,7 +260,7 @@ impl RemoteDisk { // Timeout occurred, mark disk as potentially faulty self.health.decrement_waiting(); warn!("Remote disk operation timeout after {:?}", timeout_duration); - Err(Error::other(format!("Remote disk operation timeout after {:?}", timeout_duration))) + Err(Error::other(format!("Remote disk operation timeout after {timeout_duration:?}"))) } } } diff --git a/crates/lock/src/client/remote.rs b/crates/lock/src/client/remote.rs deleted file mode 100644 index e69a82f4..00000000 --- a/crates/lock/src/client/remote.rs +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use async_trait::async_trait; -use rustfs_protos::{ - node_service_time_out_client, node_service_time_out_client_no_auth, - proto_gen::node_service::{GenerallyLockRequest, PingRequest}, -}; -use std::collections::HashMap; -use std::sync::Arc; -use tokio::sync::RwLock; -use tonic::Request; -use tracing::info; - -use crate::{ - error::{LockError, Result}, - types::{LockId, LockInfo, LockRequest, LockResponse, LockStats}, -}; - -use super::LockClient; - -/// Remote lock client implementation -#[derive(Debug)] -pub struct RemoteClient { - addr: String, - // Track active locks with their original owner information - active_locks: Arc>>, // lock_id -> owner -} - -impl Clone for RemoteClient { - fn clone(&self) -> Self { - Self { - addr: self.addr.clone(), - active_locks: self.active_locks.clone(), - } - } -} - -impl RemoteClient { - pub fn new(endpoint: String) -> Self { - Self { - addr: endpoint, - active_locks: Arc::new(RwLock::new(HashMap::new())), - } - } - - pub fn from_url(url: url::Url) -> Self { - Self { - addr: url.to_string(), - active_locks: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Create a minimal LockRequest for unlock operations - fn create_unlock_request(&self, lock_id: &LockId, owner: &str) -> LockRequest { - LockRequest { - lock_id: lock_id.clone(), - resource: lock_id.resource.clone(), - lock_type: crate::types::LockType::Exclusive, // Type doesn't matter for unlock - owner: owner.to_string(), - acquire_timeout: std::time::Duration::from_secs(30), - ttl: std::time::Duration::from_secs(300), - metadata: crate::types::LockMetadata::default(), - priority: crate::types::LockPriority::Normal, - deadlock_detection: false, - } - } -} - -#[async_trait] -impl LockClient for RemoteClient { - async fn acquire_exclusive(&self, request: &LockRequest) -> Result { - info!("remote acquire_exclusive for {}", request.resource); - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - let req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let resp = client - .lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? - .into_inner(); - - // Check for explicit error first - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - - // Check if the lock acquisition was successful - if resp.success { - // Save the lock information for later release - let mut locks = self.active_locks.write().await; - locks.insert(request.lock_id.clone(), request.owner.clone()); - - Ok(LockResponse::success( - LockInfo { - id: request.lock_id.clone(), - resource: request.resource.clone(), - lock_type: request.lock_type, - status: crate::types::LockStatus::Acquired, - owner: request.owner.clone(), - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + request.ttl, - last_refreshed: std::time::SystemTime::now(), - metadata: request.metadata.clone(), - priority: request.priority, - wait_start_time: None, - }, - std::time::Duration::ZERO, - )) - } else { - // Lock acquisition failed - Ok(LockResponse::failure( - "Lock acquisition failed on remote server".to_string(), - std::time::Duration::ZERO, - )) - } - } - - async fn acquire_shared(&self, request: &LockRequest) -> Result { - info!("remote acquire_shared for {}", request.resource); - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - let req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let resp = client - .r_lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? - .into_inner(); - - // Check for explicit error first - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - - // Check if the lock acquisition was successful - if resp.success { - // Save the lock information for later release - let mut locks = self.active_locks.write().await; - locks.insert(request.lock_id.clone(), request.owner.clone()); - - Ok(LockResponse::success( - LockInfo { - id: request.lock_id.clone(), - resource: request.resource.clone(), - lock_type: request.lock_type, - status: crate::types::LockStatus::Acquired, - owner: request.owner.clone(), - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + request.ttl, - last_refreshed: std::time::SystemTime::now(), - metadata: request.metadata.clone(), - priority: request.priority, - wait_start_time: None, - }, - std::time::Duration::ZERO, - )) - } else { - // Lock acquisition failed - Ok(LockResponse::failure( - "Shared lock acquisition failed on remote server".to_string(), - std::time::Duration::ZERO, - )) - } - } - - async fn release(&self, lock_id: &LockId) -> Result { - info!("remote release for {}", lock_id); - - // Get the original owner for this lock - let owner = { - let locks = self.active_locks.read().await; - locks.get(lock_id).cloned().unwrap_or_else(|| "remote".to_string()) - }; - - let unlock_request = self.create_unlock_request(lock_id, &owner); - - let request_string = serde_json::to_string(&unlock_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?; - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - - // Try UnLock first (for exclusive locks) - let req = Request::new(GenerallyLockRequest { - args: request_string.clone(), - }); - let resp = client.un_lock(req).await; - - let success = if resp.is_err() { - // If that fails, try RUnLock (for shared locks) - let req = Request::new(GenerallyLockRequest { args: request_string }); - let resp = client - .r_un_lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? - .into_inner(); - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - resp.success - } else { - let resp = resp.map_err(|e| LockError::internal(e.to_string()))?.into_inner(); - - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - resp.success - }; - - // Remove the lock from our tracking if successful - if success { - let mut locks = self.active_locks.write().await; - locks.remove(lock_id); - } - - Ok(success) - } - - async fn refresh(&self, lock_id: &LockId) -> Result { - info!("remote refresh for {}", lock_id); - let refresh_request = self.create_unlock_request(lock_id, "remote"); - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - let req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&refresh_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let resp = client - .refresh(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? - .into_inner(); - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - Ok(resp.success) - } - - async fn force_release(&self, lock_id: &LockId) -> Result { - info!("remote force_release for {}", lock_id); - let force_request = self.create_unlock_request(lock_id, "remote"); - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - let req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&force_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let resp = client - .force_un_lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? - .into_inner(); - if let Some(error_info) = resp.error_info { - return Err(LockError::internal(error_info)); - } - Ok(resp.success) - } - - async fn check_status(&self, lock_id: &LockId) -> Result> { - info!("remote check_status for {}", lock_id); - - // Since there's no direct status query in the gRPC service, - // we attempt a non-blocking lock acquisition to check if the resource is available - let status_request = self.create_unlock_request(lock_id, "remote"); - let mut client = node_service_time_out_client_no_auth(&self.addr) - .await - .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; - - // Try to acquire a very short-lived lock to test availability - let req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&status_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - - // Try exclusive lock first with very short timeout - let resp = client.lock(req).await; - - match resp { - Ok(response) => { - let resp = response.into_inner(); - if resp.success { - // If we successfully acquired the lock, the resource was free - // Immediately release it - let release_req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&status_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let _ = client.un_lock(release_req).await; // Best effort release - - // Return None since no one was holding the lock - Ok(None) - } else { - // Lock acquisition failed, meaning someone is holding it - // We can't determine the exact details remotely, so return a generic status - Ok(Some(LockInfo { - id: lock_id.clone(), - resource: lock_id.as_str().to_string(), - lock_type: crate::types::LockType::Exclusive, // We can't know the exact type - status: crate::types::LockStatus::Acquired, - owner: "unknown".to_string(), // Remote client can't determine owner - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), - last_refreshed: std::time::SystemTime::now(), - metadata: crate::types::LockMetadata::default(), - priority: crate::types::LockPriority::Normal, - wait_start_time: None, - })) - } - } - Err(_) => { - // Communication error or lock is held - Ok(Some(LockInfo { - id: lock_id.clone(), - resource: lock_id.as_str().to_string(), - lock_type: crate::types::LockType::Exclusive, - status: crate::types::LockStatus::Acquired, - owner: "unknown".to_string(), - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), - last_refreshed: std::time::SystemTime::now(), - metadata: crate::types::LockMetadata::default(), - priority: crate::types::LockPriority::Normal, - wait_start_time: None, - })) - } - } - } - - async fn get_stats(&self) -> Result { - info!("remote get_stats from {}", self.addr); - - // Since there's no direct statistics endpoint in the gRPC service, - // we return basic stats indicating this is a remote client - let stats = LockStats { - last_updated: std::time::SystemTime::now(), - ..Default::default() - }; - - // We could potentially enhance this by: - // 1. Keeping local counters of operations performed - // 2. Adding a stats gRPC method to the service - // 3. Querying server health endpoints - - // For now, return minimal stats indicating remote connectivity - Ok(stats) - } - - async fn close(&self) -> Result<()> { - Ok(()) - } - - async fn is_online(&self) -> bool { - // Use Ping interface to test if remote service is online - let mut client = match self.get_client().await { - Ok(client) => client, - Err(_) => { - info!("remote client {} connection failed", self.addr); - return false; - } - }; - - let ping_req = Request::new(PingRequest { - version: 1, - body: bytes::Bytes::new(), - }); - - match client.ping(ping_req).await { - Ok(_) => { - info!("remote client {} is online", self.addr); - true - } - Err(_) => { - info!("remote client {} ping failed", self.addr); - false - } - } - } - - async fn is_local(&self) -> bool { - false - } -} diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index b24088a0..66a016b9 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -336,7 +336,7 @@ pub async fn init_ftp_system( let ftps_address_str = rustfs_utils::get_env_str(rustfs_config::ENV_FTPS_ADDRESS, rustfs_config::DEFAULT_FTPS_ADDRESS); let addr: SocketAddr = ftps_address_str .parse() - .map_err(|e| format!("Invalid FTPS address '{}': {}", ftps_address_str, e))?; + .map_err(|e| format!("Invalid FTPS address '{ftps_address_str}': {e}"))?; // Get FTPS configuration from environment variables let cert_file = rustfs_utils::get_env_opt_str(rustfs_config::ENV_FTPS_CERTS_FILE); @@ -402,7 +402,7 @@ pub async fn init_sftp_system( let sftp_address_str = rustfs_utils::get_env_str(rustfs_config::ENV_SFTP_ADDRESS, rustfs_config::DEFAULT_SFTP_ADDRESS); let addr: SocketAddr = sftp_address_str .parse() - .map_err(|e| format!("Invalid SFTP address '{}': {}", sftp_address_str, e))?; + .map_err(|e| format!("Invalid SFTP address '{sftp_address_str}': {e}"))?; // Get SFTP configuration from environment variables let host_key = rustfs_utils::get_env_opt_str(rustfs_config::ENV_SFTP_HOST_KEY); diff --git a/rustfs/src/server/cert.rs b/rustfs/src/server/cert.rs index 18f8656a..68b9bc14 100644 --- a/rustfs/src/server/cert.rs +++ b/rustfs/src/server/cert.rs @@ -26,7 +26,7 @@ pub enum RustFSError { impl std::fmt::Display for RustFSError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - RustFSError::Cert(msg) => write!(f, "Certificate error: {}", msg), + RustFSError::Cert(msg) => write!(f, "Certificate error: {msg}"), } } } @@ -78,7 +78,7 @@ fn parse_pem_private_key(pem: &[u8]) -> Result, RustFSErr async fn read_file(path: &PathBuf, desc: &str) -> Result, RustFSError> { tokio::fs::read(path) .await - .map_err(|e| RustFSError::Cert(format!("read {} {:?}: {e}", desc, path))) + .map_err(|e| RustFSError::Cert(format!("read {desc} {path:?}: {e}"))) } /// Initialize TLS material for both server and outbound client connections. diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index 1f173da5..4787aeb5 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -1784,7 +1784,7 @@ impl Node for NodeService { return Ok(Response::new(GetMetricsResponse { success: false, realtime_metrics: Bytes::new(), - error_info: Some(format!("Invalid metric_type: {}", err)), + error_info: Some(format!("Invalid metric_type: {err}")), })); } }; @@ -1798,7 +1798,7 @@ impl Node for NodeService { return Ok(Response::new(GetMetricsResponse { success: false, realtime_metrics: Bytes::new(), - error_info: Some(format!("Invalid opts: {}", err)), + error_info: Some(format!("Invalid opts: {err}")), })); } }; From e7a3129be4e80db2df15849b6d7895d4bcc2746c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Mon, 5 Jan 2026 22:24:35 +0800 Subject: [PATCH 02/12] feat: s3 tests classification (#1392) Co-authored-by: houseme --- _typos.toml | 4 +- crates/policy/src/policy/action.rs | 88 +++- crates/policy/src/policy/id.rs | 7 + crates/policy/src/policy/policy.rs | 106 ++++- crates/policy/src/policy/principal.rs | 54 ++- crates/policy/src/policy/resource.rs | 5 + crates/policy/src/policy/statement.rs | 10 +- scripts/s3-tests/implemented_tests.txt | 130 ++++++ scripts/s3-tests/non_standard_tests.txt | 505 +++++++++++++++++++++++ scripts/s3-tests/run.sh | 271 ++++++------ scripts/s3-tests/unimplemented_tests.txt | 191 +++++++++ 11 files changed, 1233 insertions(+), 138 deletions(-) create mode 100644 scripts/s3-tests/implemented_tests.txt create mode 100644 scripts/s3-tests/non_standard_tests.txt create mode 100644 scripts/s3-tests/unimplemented_tests.txt diff --git a/_typos.toml b/_typos.toml index b79e2226..231928f6 100644 --- a/_typos.toml +++ b/_typos.toml @@ -37,6 +37,8 @@ datas = "datas" bre = "bre" abd = "abd" mak = "mak" +# s3-tests original test names (cannot be changed) +nonexisted = "nonexisted" [files] -extend-exclude = [] \ No newline at end of file +extend-exclude = [] diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index 16f0e12b..e6ac3f3d 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -22,10 +22,42 @@ use strum::{EnumString, IntoStaticStr}; use super::{Error as IamError, Validator, utils::wildcard}; -#[derive(Serialize, Clone, Default, Debug)] +/// A set of policy actions that serializes as a single string when containing one item, +/// or as an array when containing multiple items (matching AWS S3 API format). +#[derive(Clone, Default, Debug)] pub struct ActionSet(pub HashSet); +impl Serialize for ActionSet { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeSeq; + + if self.0.len() == 1 { + // Serialize single action as string (not array) + if let Some(action) = self.0.iter().next() { + let action_str: &str = action.into(); + return serializer.serialize_str(action_str); + } + } + + // Serialize multiple actions as array + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for action in &self.0 { + let action_str: &str = action.into(); + seq.serialize_element(action_str)?; + } + seq.end() + } +} + impl ActionSet { + /// Returns true if the action set is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn is_match(&self, action: &Action) -> bool { for act in self.0.iter() { if act.is_match(action) { @@ -150,6 +182,10 @@ impl Action { impl TryFrom<&str> for Action { type Error = Error; fn try_from(value: &str) -> std::result::Result { + // Support wildcard "*" which matches all S3 actions (AWS S3 standard) + if value == "*" { + return Ok(Self::S3Action(S3Action::AllActions)); + } if value.starts_with(Self::S3_PREFIX) { Ok(Self::S3Action( S3Action::try_from(value).map_err(|_| IamError::InvalidAction(value.into()))?, @@ -559,3 +595,53 @@ pub enum KmsAction { #[strum(serialize = "kms:*")] AllActions, } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn test_action_wildcard_parsing() { + // Test that "*" parses to S3Action::AllActions + let action = Action::try_from("*").expect("Should parse wildcard"); + assert!(matches!(action, Action::S3Action(S3Action::AllActions))); + } + + #[test] + fn test_actionset_serialize_single_element() { + // Single element should serialize as string + let mut set = HashSet::new(); + set.insert(Action::S3Action(S3Action::GetObjectAction)); + let actionset = ActionSet(set); + + let json = serde_json::to_string(&actionset).expect("Should serialize"); + assert_eq!(json, "\"s3:GetObject\""); + } + + #[test] + fn test_actionset_serialize_multiple_elements() { + // Multiple elements should serialize as array + let mut set = HashSet::new(); + set.insert(Action::S3Action(S3Action::GetObjectAction)); + set.insert(Action::S3Action(S3Action::PutObjectAction)); + let actionset = ActionSet(set); + + let json = serde_json::to_string(&actionset).expect("Should serialize"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); + assert!(parsed.is_array()); + let arr = parsed.as_array().expect("Should be array"); + assert_eq!(arr.len(), 2); + } + + #[test] + fn test_actionset_wildcard_serialization() { + // Wildcard action should serialize correctly + let mut set = HashSet::new(); + set.insert(Action::try_from("*").expect("Should parse wildcard")); + let actionset = ActionSet(set); + + let json = serde_json::to_string(&actionset).expect("Should serialize"); + assert_eq!(json, "\"s3:*\""); + } +} diff --git a/crates/policy/src/policy/id.rs b/crates/policy/src/policy/id.rs index a915eaa4..ea373fc4 100644 --- a/crates/policy/src/policy/id.rs +++ b/crates/policy/src/policy/id.rs @@ -21,6 +21,13 @@ use super::Validator; #[derive(Serialize, Deserialize, Clone, Default, Debug)] pub struct ID(pub String); +impl ID { + /// Returns true if the ID is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + impl Validator for ID { type Error = Error; /// if id is a valid utf string, then it is valid. diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index 1e763dfa..6cde4e9d 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -177,9 +177,11 @@ pub struct BucketPolicyArgs<'a> { pub object: &'a str, } +/// Bucket Policy with AWS S3-compatible JSON serialization. +/// Empty optional fields are omitted from output to match AWS format. #[derive(Serialize, Deserialize, Clone, Default, Debug)] pub struct BucketPolicy { - #[serde(default, rename = "ID")] + #[serde(default, rename = "ID", skip_serializing_if = "ID::is_empty")] pub id: ID, #[serde(rename = "Version")] pub version: String, @@ -950,4 +952,106 @@ mod test { ); } } + + #[test] + fn test_bucket_policy_serialize_omits_empty_fields() { + use crate::policy::action::{Action, ActionSet, S3Action}; + use crate::policy::resource::{Resource, ResourceSet}; + use crate::policy::{Effect, Functions, Principal}; + + // Create a BucketPolicy with empty optional fields + // Use JSON deserialization to create Principal (since aws field is private) + let principal: Principal = serde_json::from_str(r#"{"AWS": "*"}"#).expect("Should parse principal"); + + let mut policy = BucketPolicy { + id: ID::default(), // Empty ID + version: "2012-10-17".to_string(), + statements: vec![BPStatement { + sid: ID::default(), // Empty Sid + effect: Effect::Allow, + principal, + actions: ActionSet::default(), + not_actions: ActionSet::default(), // Empty NotAction + resources: ResourceSet::default(), + not_resources: ResourceSet::default(), // Empty NotResource + conditions: Functions::default(), // Empty Condition + }], + }; + + // Set actions and resources (required fields) + policy.statements[0] + .actions + .0 + .insert(Action::S3Action(S3Action::ListBucketAction)); + policy.statements[0] + .resources + .0 + .insert(Resource::try_from("arn:aws:s3:::test/*").unwrap()); + + let json = serde_json::to_string(&policy).expect("Should serialize"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); + + // Verify empty fields are omitted + assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted"); + + let statement = &parsed["Statement"][0]; + assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted"); + assert!( + !statement.as_object().unwrap().contains_key("NotAction"), + "Empty NotAction should be omitted" + ); + assert!( + !statement.as_object().unwrap().contains_key("NotResource"), + "Empty NotResource should be omitted" + ); + assert!( + !statement.as_object().unwrap().contains_key("Condition"), + "Empty Condition should be omitted" + ); + + // Verify required fields are present + assert_eq!(parsed["Version"], "2012-10-17"); + assert_eq!(statement["Effect"], "Allow"); + assert_eq!(statement["Principal"]["AWS"], "*"); + } + + #[test] + fn test_bucket_policy_serialize_single_action_as_string() { + use crate::policy::action::{Action, ActionSet, S3Action}; + use crate::policy::resource::{Resource, ResourceSet}; + use crate::policy::{Effect, Principal}; + + // Use JSON deserialization to create Principal (since aws field is private) + let principal: Principal = serde_json::from_str(r#"{"AWS": "*"}"#).expect("Should parse principal"); + + let mut policy = BucketPolicy { + version: "2012-10-17".to_string(), + statements: vec![BPStatement { + effect: Effect::Allow, + principal, + actions: ActionSet::default(), + resources: ResourceSet::default(), + ..Default::default() + }], + ..Default::default() + }; + + // Single action + policy.statements[0] + .actions + .0 + .insert(Action::S3Action(S3Action::ListBucketAction)); + policy.statements[0] + .resources + .0 + .insert(Resource::try_from("arn:aws:s3:::test/*").unwrap()); + + let json = serde_json::to_string(&policy).expect("Should serialize"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); + let action = &parsed["Statement"][0]["Action"]; + + // Single action should be serialized as string + assert!(action.is_string(), "Single action should serialize as string"); + assert_eq!(action.as_str().unwrap(), "s3:ListBucket"); + } } diff --git a/crates/policy/src/policy/principal.rs b/crates/policy/src/policy/principal.rs index 8c12ef9f..85689e07 100644 --- a/crates/policy/src/policy/principal.rs +++ b/crates/policy/src/policy/principal.rs @@ -17,13 +17,35 @@ use crate::error::Error; use serde::Serialize; use std::collections::HashSet; -#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)] -#[serde(rename_all = "PascalCase", default)] +/// Principal that serializes AWS field as single string when containing only "*", +/// or as an array otherwise (matching AWS S3 API format). +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Principal { - #[serde(rename = "AWS")] aws: HashSet, } +impl Serialize for Principal { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + + let mut map = serializer.serialize_map(Some(1))?; + + // If single element, serialize as string; otherwise as array + if self.aws.len() == 1 { + if let Some(val) = self.aws.iter().next() { + map.serialize_entry("AWS", val)?; + } + } else { + map.serialize_entry("AWS", &self.aws)?; + } + + map.end() + } +} + #[derive(serde::Deserialize)] #[serde(untagged)] enum PrincipalFormat { @@ -118,4 +140,30 @@ mod test { }; assert!(result); } + + #[test] + fn test_principal_serialize_single_element() { + // Single element should serialize as string (AWS format) + let principal = Principal { + aws: HashSet::from(["*".to_string()]), + }; + + let json = serde_json::to_string(&principal).expect("Should serialize"); + assert_eq!(json, r#"{"AWS":"*"}"#); + } + + #[test] + fn test_principal_serialize_multiple_elements() { + // Multiple elements should serialize as array + let principal = Principal { + aws: HashSet::from(["*".to_string(), "arn:aws:iam::123456789012:root".to_string()]), + }; + + let json = serde_json::to_string(&principal).expect("Should serialize"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); + let aws_value = parsed.get("AWS").expect("Should have AWS field"); + assert!(aws_value.is_array()); + let arr = aws_value.as_array().expect("Should be array"); + assert_eq!(arr.len(), 2); + } } diff --git a/crates/policy/src/policy/resource.rs b/crates/policy/src/policy/resource.rs index 0d7ff9eb..12398c00 100644 --- a/crates/policy/src/policy/resource.rs +++ b/crates/policy/src/policy/resource.rs @@ -35,6 +35,11 @@ use super::{ pub struct ResourceSet(pub HashSet); impl ResourceSet { + /// Returns true if the resource set is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub async fn is_match(&self, resource: &str, conditions: &HashMap>) -> bool { self.is_match_with_resolver(resource, conditions, None).await } diff --git a/crates/policy/src/policy/statement.rs b/crates/policy/src/policy/statement.rs index 6412e1c8..7d69a99e 100644 --- a/crates/policy/src/policy/statement.rs +++ b/crates/policy/src/policy/statement.rs @@ -179,10 +179,12 @@ impl PartialEq for Statement { } } +/// Bucket Policy Statement with AWS S3-compatible JSON serialization. +/// Empty optional fields are omitted from output to match AWS format. #[derive(Debug, Deserialize, Serialize, Default, Clone)] #[serde(rename_all = "PascalCase", default)] pub struct BPStatement { - #[serde(rename = "Sid", default)] + #[serde(rename = "Sid", default, skip_serializing_if = "ID::is_empty")] pub sid: ID, #[serde(rename = "Effect")] pub effect: Effect, @@ -190,13 +192,13 @@ pub struct BPStatement { pub principal: Principal, #[serde(rename = "Action")] pub actions: ActionSet, - #[serde(rename = "NotAction", default)] + #[serde(rename = "NotAction", default, skip_serializing_if = "ActionSet::is_empty")] pub not_actions: ActionSet, #[serde(rename = "Resource", default)] pub resources: ResourceSet, - #[serde(rename = "NotResource", default)] + #[serde(rename = "NotResource", default, skip_serializing_if = "ResourceSet::is_empty")] pub not_resources: ResourceSet, - #[serde(rename = "Condition", default)] + #[serde(rename = "Condition", default, skip_serializing_if = "Functions::is_empty")] pub conditions: Functions, } diff --git a/scripts/s3-tests/implemented_tests.txt b/scripts/s3-tests/implemented_tests.txt new file mode 100644 index 00000000..e88688de --- /dev/null +++ b/scripts/s3-tests/implemented_tests.txt @@ -0,0 +1,130 @@ +# Implemented S3 feature tests +# ============================ +# +# These tests SHOULD PASS on RustFS for standard S3 API compatibility. +# Run these tests to verify RustFS S3 compatibility. +# +# Covered operations: +# - Bucket: Create, Delete, List, Head, GetLocation +# - Object: Put, Get, Delete, Copy, Head +# - ListObjects/ListObjectsV2: prefix, delimiter, marker, maxkeys +# - Multipart Upload: Create, Upload, Complete, Abort, List +# - Tagging: Bucket and Object tags +# - Bucket Policy: Put, Get, Delete +# - Public Access Block: Put, Get, Delete +# - Presigned URLs: GET and PUT operations +# - Range requests: Partial object retrieval +# - Metadata: User-defined metadata +# - Conditional GET: If-Match, If-None-Match, If-Modified-Since +# +# Total: 109 tests + +test_basic_key_count +test_bucket_create_naming_bad_short_one +test_bucket_create_naming_bad_short_two +test_bucket_create_naming_bad_starts_nonalpha +test_bucket_create_naming_dns_dash_at_end +test_bucket_create_naming_dns_dash_dot +test_bucket_create_naming_dns_dot_dash +test_bucket_create_naming_dns_dot_dot +test_bucket_create_naming_dns_underscore +test_bucket_create_naming_good_contains_hyphen +test_bucket_create_naming_good_contains_period +test_bucket_create_naming_good_long_60 +test_bucket_create_naming_good_long_61 +test_bucket_create_naming_good_long_62 +test_bucket_create_naming_good_long_63 +test_bucket_create_naming_good_starts_alpha +test_bucket_create_naming_good_starts_digit +test_bucket_delete_nonempty +test_bucket_delete_notexist +test_bucket_head +test_bucket_head_notexist +test_bucket_list_distinct +test_bucket_list_empty +test_bucket_list_long_name +test_bucket_list_marker_after_list +test_bucket_list_marker_empty +test_bucket_list_marker_none +test_bucket_list_marker_not_in_list +test_bucket_list_marker_unreadable +test_bucket_list_maxkeys_invalid +test_bucket_list_maxkeys_none +test_bucket_list_maxkeys_zero +test_bucket_list_prefix_alt +test_bucket_list_prefix_basic +test_bucket_list_prefix_delimiter_alt +test_bucket_list_prefix_delimiter_basic +test_bucket_list_prefix_delimiter_delimiter_not_exist +test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist +test_bucket_list_prefix_delimiter_prefix_not_exist +test_bucket_list_prefix_empty +test_bucket_list_prefix_none +test_bucket_list_prefix_not_exist +test_bucket_list_prefix_unreadable +test_bucket_list_special_prefix +test_bucket_listv2_continuationtoken +test_bucket_listv2_continuationtoken_empty +test_bucket_listv2_fetchowner_defaultempty +test_bucket_listv2_fetchowner_empty +test_bucket_listv2_fetchowner_notempty +test_bucket_listv2_maxkeys_none +test_bucket_listv2_maxkeys_zero +test_bucket_listv2_prefix_alt +test_bucket_listv2_prefix_basic +test_bucket_listv2_prefix_delimiter_alt +test_bucket_listv2_prefix_delimiter_basic +test_bucket_listv2_prefix_delimiter_delimiter_not_exist +test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist +test_bucket_listv2_prefix_delimiter_prefix_not_exist +test_bucket_listv2_prefix_empty +test_bucket_listv2_prefix_none +test_bucket_listv2_prefix_not_exist +test_bucket_listv2_prefix_unreadable +test_bucket_listv2_startafter_after_list +test_bucket_listv2_startafter_not_in_list +test_bucket_listv2_startafter_unreadable +test_bucket_notexist +test_buckets_create_then_list +test_buckets_list_ctime +test_bucketv2_notexist +test_bucketv2_policy_another_bucket +test_get_bucket_policy_status +test_get_nonpublicpolicy_principal_bucket_policy_status +test_get_object_ifmatch_good +test_get_object_ifmodifiedsince_good +test_get_object_ifunmodifiedsince_failed +test_list_buckets_bad_auth +test_multi_object_delete +test_multi_object_delete_key_limit +test_multi_objectv2_delete +test_multi_objectv2_delete_key_limit +test_multipart_copy_without_range +test_multipart_upload_empty +test_multipart_upload_incorrect_etag +test_multipart_upload_missing_part +test_multipart_upload_multiple_sizes +test_multipart_upload_on_a_bucket_with_policy +test_multipart_upload_overwrite_existing_object +test_multipart_upload_size_too_small +test_object_copy_bucket_not_found +test_object_copy_key_not_found +test_object_copy_not_owned_object_bucket +test_object_head_zero_bytes +test_object_metadata_replaced_on_put +test_object_put_authenticated +test_object_read_not_exist +test_object_set_get_metadata_none_to_empty +test_object_set_get_metadata_none_to_good +test_object_set_get_metadata_overwrite_to_empty +test_object_write_cache_control +test_object_write_check_etag +test_object_write_expires +test_object_write_file +test_object_write_read_update_read_delete +test_object_write_to_nonexist_bucket +test_put_max_kvsize_tags +test_ranged_request_empty_object +test_ranged_request_invalid_range +test_set_multipart_tagging +test_upload_part_copy_percent_encoded_key diff --git a/scripts/s3-tests/non_standard_tests.txt b/scripts/s3-tests/non_standard_tests.txt new file mode 100644 index 00000000..c4b01aea --- /dev/null +++ b/scripts/s3-tests/non_standard_tests.txt @@ -0,0 +1,505 @@ +# Non-standard S3 tests (Ceph/RGW/MinIO specific) +# ================================================ +# +# These tests use vendor-specific extensions not part of AWS S3 API. +# They are PERMANENTLY EXCLUDED from RustFS compatibility testing. +# +# Exclusion reasons: +# - fails_on_aws marker: Ceph-specific features +# - X-RGW-* headers: Ceph proprietary headers +# - allowUnordered: Ceph-specific query parameter +# - ACL tests: RustFS uses IAM policy-based access control +# - CORS tests: Not implemented +# - POST Object: HTML form upload not implemented +# - Error format differences: Minor response format variations +# +# Total: non-standard tests listed below + +test_100_continue +test_100_continue_error_retry +test_abort_multipart_upload_not_found +test_access_bucket_private_object_private +test_access_bucket_private_object_publicread +test_access_bucket_private_object_publicreadwrite +test_access_bucket_private_objectv2_private +test_access_bucket_private_objectv2_publicread +test_access_bucket_private_objectv2_publicreadwrite +test_access_bucket_publicread_object_private +test_access_bucket_publicread_object_publicread +test_access_bucket_publicread_object_publicreadwrite +test_access_bucket_publicreadwrite_object_private +test_access_bucket_publicreadwrite_object_publicread +test_access_bucket_publicreadwrite_object_publicreadwrite +test_account_usage +test_atomic_conditional_write_1mb +test_atomic_dual_conditional_write_1mb +test_atomic_write_bucket_gone +test_block_public_restrict_public_buckets +test_bucket_acl_canned +test_bucket_acl_canned_authenticatedread +test_bucket_acl_canned_during_create +test_bucket_acl_canned_private_to_private +test_bucket_acl_canned_publicreadwrite +test_bucket_acl_default +test_bucket_acl_grant_email +test_bucket_acl_grant_email_not_exist +test_bucket_acl_grant_nonexist_user +test_bucket_acl_grant_userid_fullcontrol +test_bucket_acl_grant_userid_read +test_bucket_acl_grant_userid_readacp +test_bucket_acl_grant_userid_write +test_bucket_acl_grant_userid_writeacp +test_bucket_acl_revoke_all +test_bucket_concurrent_set_canned_acl +test_bucket_create_exists +test_bucket_create_exists_nonowner +test_bucket_create_naming_bad_ip +test_bucket_create_naming_dns_long +test_bucket_create_special_key_names +test_bucket_get_location +test_bucket_head_extended +test_bucket_header_acl_grants +test_bucket_list_delimiter_not_skip_special +test_bucket_list_delimiter_prefix +test_bucket_list_delimiter_prefix_underscore +test_bucket_list_many +test_bucket_list_maxkeys_one +test_bucket_list_objects_anonymous +test_bucket_list_objects_anonymous_fail +test_bucket_list_return_data +test_bucket_list_return_data_versioning +test_bucket_list_unordered +test_bucket_listv2_both_continuationtoken_startafter +test_bucket_listv2_delimiter_prefix +test_bucket_listv2_delimiter_prefix_underscore +test_bucket_listv2_many +test_bucket_listv2_maxkeys_one +test_bucket_listv2_objects_anonymous +test_bucket_listv2_objects_anonymous_fail +test_bucket_listv2_unordered +test_bucket_logging_bucket_acl_required +test_bucket_logging_bucket_auth_type +test_bucket_logging_cleanup_bucket_concurrent_deletion_j +test_bucket_logging_cleanup_bucket_concurrent_deletion_j_single +test_bucket_logging_cleanup_bucket_concurrent_deletion_s +test_bucket_logging_cleanup_bucket_concurrent_deletion_s_single +test_bucket_logging_cleanup_bucket_deletion_j +test_bucket_logging_cleanup_bucket_deletion_j_single +test_bucket_logging_cleanup_bucket_deletion_s +test_bucket_logging_cleanup_bucket_deletion_s_single +test_bucket_logging_cleanup_concurrent_disabling_j +test_bucket_logging_cleanup_concurrent_disabling_j_single +test_bucket_logging_cleanup_concurrent_disabling_s +test_bucket_logging_cleanup_concurrent_disabling_s_single +test_bucket_logging_cleanup_concurrent_updating_j +test_bucket_logging_cleanup_concurrent_updating_j_single +test_bucket_logging_cleanup_concurrent_updating_s +test_bucket_logging_cleanup_concurrent_updating_s_single +test_bucket_logging_cleanup_disabling_j +test_bucket_logging_cleanup_disabling_j_single +test_bucket_logging_cleanup_disabling_s +test_bucket_logging_cleanup_disabling_s_single +test_bucket_logging_cleanup_updating_j +test_bucket_logging_cleanup_updating_j_single +test_bucket_logging_cleanup_updating_s +test_bucket_logging_cleanup_updating_s_single +test_bucket_logging_concurrent_flush_j +test_bucket_logging_concurrent_flush_j_single +test_bucket_logging_concurrent_flush_s +test_bucket_logging_concurrent_flush_s_single +test_bucket_logging_conf_concurrent_updating_pfx_j +test_bucket_logging_conf_concurrent_updating_pfx_s +test_bucket_logging_conf_concurrent_updating_roll_j +test_bucket_logging_conf_concurrent_updating_roll_s +test_bucket_logging_conf_updating_pfx_j +test_bucket_logging_conf_updating_pfx_s +test_bucket_logging_conf_updating_roll_j +test_bucket_logging_conf_updating_roll_s +test_bucket_logging_copy_objects +test_bucket_logging_copy_objects_bucket +test_bucket_logging_copy_objects_bucket_versioned +test_bucket_logging_copy_objects_versioned +test_bucket_logging_delete_objects +test_bucket_logging_delete_objects_versioned +test_bucket_logging_event_type_j +test_bucket_logging_event_type_s +test_bucket_logging_flush_empty +test_bucket_logging_flush_j +test_bucket_logging_flush_j_single +test_bucket_logging_flush_s +test_bucket_logging_flush_s_single +test_bucket_logging_get_objects +test_bucket_logging_get_objects_versioned +test_bucket_logging_head_objects +test_bucket_logging_head_objects_versioned +test_bucket_logging_key_filter_j +test_bucket_logging_key_filter_s +test_bucket_logging_mpu_copy +test_bucket_logging_mpu_copy_versioned +test_bucket_logging_mpu_j +test_bucket_logging_mpu_s +test_bucket_logging_mpu_versioned_j +test_bucket_logging_mpu_versioned_s +test_bucket_logging_mtime +test_bucket_logging_multi_delete +test_bucket_logging_multi_delete_versioned +test_bucket_logging_multiple_prefixes +test_bucket_logging_notupdating_j +test_bucket_logging_notupdating_j_single +test_bucket_logging_notupdating_s +test_bucket_logging_notupdating_s_single +test_bucket_logging_object_acl_required +test_bucket_logging_object_meta +test_bucket_logging_part_cleanup_concurrent_deletion_j +test_bucket_logging_part_cleanup_concurrent_deletion_s +test_bucket_logging_part_cleanup_concurrent_disabling_j +test_bucket_logging_part_cleanup_concurrent_disabling_s +test_bucket_logging_part_cleanup_concurrent_updating_j +test_bucket_logging_part_cleanup_concurrent_updating_s +test_bucket_logging_part_cleanup_deletion_j +test_bucket_logging_part_cleanup_deletion_s +test_bucket_logging_part_cleanup_disabling_j +test_bucket_logging_part_cleanup_disabling_s +test_bucket_logging_part_cleanup_updating_j +test_bucket_logging_part_cleanup_updating_s +test_bucket_logging_partitioned_key +test_bucket_logging_permission_change_j +test_bucket_logging_permission_change_s +test_bucket_logging_put_and_flush +test_bucket_logging_put_concurrency +test_bucket_logging_put_objects +test_bucket_logging_put_objects_versioned +test_bucket_logging_roll_time +test_bucket_logging_simple_key +test_bucket_logging_single_prefix +test_bucket_logging_target_cleanup_j +test_bucket_logging_target_cleanup_j_single +test_bucket_logging_target_cleanup_s +test_bucket_logging_target_cleanup_s_single +test_bucket_policy_get_obj_acl_existing_tag +test_bucket_policy_get_obj_existing_tag +test_bucket_policy_get_obj_tagging_existing_tag +test_bucket_policy_put_obj_copy_source +test_bucket_policy_put_obj_copy_source_meta +test_bucket_policy_put_obj_kms_noenc +test_bucket_policy_put_obj_request_obj_tag +test_bucket_policy_put_obj_s3_incorrect_algo_sse_s3 +test_bucket_policy_put_obj_s3_noenc +test_bucket_policy_put_obj_tagging_existing_tag +test_bucket_policy_set_condition_operator_end_with_IfExists +test_bucket_policy_upload_part_copy +test_bucket_recreate_new_acl +test_bucket_recreate_not_overriding +test_bucket_recreate_overwrite_acl +test_copy_object_ifmatch_failed +test_copy_object_ifmatch_good +test_copy_object_ifnonematch_failed +test_copy_object_ifnonematch_good +test_cors_header_option +test_cors_origin_response +test_cors_origin_wildcard +test_cors_presigned_get_object +test_cors_presigned_get_object_tenant +test_cors_presigned_get_object_tenant_v2 +test_cors_presigned_get_object_v2 +test_cors_presigned_put_object +test_cors_presigned_put_object_tenant +test_cors_presigned_put_object_tenant_v2 +test_cors_presigned_put_object_tenant_with_acl +test_cors_presigned_put_object_v2 +test_cors_presigned_put_object_with_acl +test_create_bucket_bucket_owner_enforced +test_create_bucket_bucket_owner_preferred +test_create_bucket_object_writer +test_delete_marker_expiration +test_delete_marker_nonversioned +test_delete_marker_suspended +test_delete_marker_versioned +test_delete_object_current_if_match +test_delete_object_current_if_match_last_modified_time +test_delete_object_current_if_match_size +test_delete_object_if_match +test_delete_object_if_match_last_modified_time +test_delete_object_if_match_size +test_delete_object_version_if_match +test_delete_object_version_if_match_last_modified_time +test_delete_object_version_if_match_size +test_delete_objects_current_if_match +test_delete_objects_current_if_match_last_modified_time +test_delete_objects_current_if_match_size +test_delete_objects_if_match +test_delete_objects_if_match_last_modified_time +test_delete_objects_if_match_size +test_delete_objects_version_if_match +test_delete_objects_version_if_match_last_modified_time +test_delete_objects_version_if_match_size +test_delete_tags_obj_public +test_encrypted_transfer_13b +test_encrypted_transfer_1MB +test_encrypted_transfer_1b +test_encrypted_transfer_1kb +test_encryption_sse_c_deny_algo_with_bucket_policy +test_encryption_sse_c_enforced_with_bucket_policy +test_encryption_sse_c_multipart_invalid_chunks_1 +test_encryption_sse_c_multipart_invalid_chunks_2 +test_encryption_sse_c_multipart_upload +test_encryption_sse_c_post_object_authenticated_request +test_encryption_sse_c_unaligned_multipart_upload +test_expected_bucket_owner +test_get_multipart_checksum_object_attributes +test_get_multipart_object_attributes +test_get_obj_tagging +test_get_object_attributes +test_get_object_ifmatch_failed +test_get_object_ifmodifiedsince_failed +test_get_object_ifnonematch_failed +test_get_object_ifnonematch_good +test_get_object_ifunmodifiedsince_good +test_get_paginated_multipart_object_attributes +test_get_single_multipart_object_attributes +test_get_sse_c_encrypted_object_attributes +test_get_tags_acl_public +test_head_bucket_usage +test_lifecycle_cloud_multiple_transition +test_lifecycle_cloud_transition +test_lifecycle_cloud_transition_large_obj +test_lifecycle_deletemarker_expiration +test_lifecycle_deletemarker_expiration_with_days_tag +test_lifecycle_expiration +test_lifecycle_expiration_date +test_lifecycle_expiration_header_and_tags_head +test_lifecycle_expiration_header_head +test_lifecycle_expiration_header_tags_head +test_lifecycle_expiration_newer_noncurrent +test_lifecycle_expiration_noncur_tags1 +test_lifecycle_expiration_size_gt +test_lifecycle_expiration_size_lt +test_lifecycle_expiration_tags1 +test_lifecycle_expiration_tags2 +test_lifecycle_expiration_versioned_tags2 +test_lifecycle_expiration_versioning_enabled +test_lifecycle_multipart_expiration +test_lifecycle_noncur_cloud_transition +test_lifecycle_noncur_expiration +test_lifecycle_noncur_transition +test_lifecycle_transition +test_lifecycle_transition_single_rule_multi_trans +test_lifecyclev2_expiration +test_list_buckets_anonymous +test_list_buckets_invalid_auth +test_list_buckets_paginated +test_list_multipart_upload +test_list_multipart_upload_owner +test_multipart_checksum_sha256 +test_multipart_copy_improper_range +test_multipart_copy_invalid_range +test_multipart_copy_multiple_sizes +test_multipart_copy_small +test_multipart_copy_special_names +test_multipart_copy_versioned +test_multipart_get_part +test_multipart_put_current_object_if_match +test_multipart_put_current_object_if_none_match +test_multipart_put_object_if_match +test_multipart_single_get_part +test_multipart_sse_c_get_part +test_multipart_upload +test_multipart_upload_contents +test_multipart_upload_resend_part +test_multipart_upload_small +test_multipart_use_cksum_helper_crc32 +test_multipart_use_cksum_helper_crc32c +test_multipart_use_cksum_helper_crc64nvme +test_multipart_use_cksum_helper_sha1 +test_multipart_use_cksum_helper_sha256 +test_non_multipart_get_part +test_non_multipart_sse_c_get_part +test_object_acl +test_object_acl_canned +test_object_acl_canned_authenticatedread +test_object_acl_canned_bucketownerfullcontrol +test_object_acl_canned_bucketownerread +test_object_acl_canned_during_create +test_object_acl_canned_publicreadwrite +test_object_acl_default +test_object_acl_full_control_verify_attributes +test_object_acl_full_control_verify_owner +test_object_acl_read +test_object_acl_readacp +test_object_acl_write +test_object_acl_writeacp +test_object_anon_put +test_object_anon_put_write_access +test_object_content_encoding_aws_chunked +test_object_copy_16m +test_object_copy_canned_acl +test_object_copy_diff_bucket +test_object_copy_not_owned_bucket +test_object_copy_replacing_metadata +test_object_copy_retaining_metadata +test_object_copy_same_bucket +test_object_copy_to_itself +test_object_copy_to_itself_with_metadata +test_object_copy_verify_contenttype +test_object_copy_versioned_bucket +test_object_copy_versioned_url_encoding +test_object_copy_versioning_multipart_upload +test_object_copy_zero_size +test_object_delete_key_bucket_gone +test_object_header_acl_grants +test_object_lock_changing_mode_from_compliance +test_object_lock_changing_mode_from_governance_with_bypass +test_object_lock_changing_mode_from_governance_without_bypass +test_object_lock_delete_multipart_object_with_legal_hold_on +test_object_lock_delete_multipart_object_with_retention +test_object_lock_delete_object_with_legal_hold_off +test_object_lock_delete_object_with_legal_hold_on +test_object_lock_delete_object_with_retention +test_object_lock_delete_object_with_retention_and_marker +test_object_lock_get_legal_hold +test_object_lock_get_obj_lock +test_object_lock_get_obj_metadata +test_object_lock_get_obj_retention +test_object_lock_get_obj_retention_iso8601 +test_object_lock_multi_delete_object_with_retention +test_object_lock_put_legal_hold +test_object_lock_put_legal_hold_invalid_status +test_object_lock_put_obj_lock +test_object_lock_put_obj_lock_invalid_days +test_object_lock_put_obj_lock_invalid_mode +test_object_lock_put_obj_lock_invalid_status +test_object_lock_put_obj_lock_invalid_years +test_object_lock_put_obj_lock_with_days_and_years +test_object_lock_put_obj_retention +test_object_lock_put_obj_retention_increase_period +test_object_lock_put_obj_retention_invalid_mode +test_object_lock_put_obj_retention_override_default_retention +test_object_lock_put_obj_retention_shorten_period +test_object_lock_put_obj_retention_shorten_period_bypass +test_object_lock_put_obj_retention_versionid +test_object_lock_suspend_versioning +test_object_lock_uploading_obj +test_object_raw_authenticated +test_object_raw_authenticated_bucket_acl +test_object_raw_authenticated_bucket_gone +test_object_raw_authenticated_object_acl +test_object_raw_authenticated_object_gone +test_object_raw_get +test_object_raw_get_bucket_acl +test_object_raw_get_bucket_gone +test_object_raw_get_object_acl +test_object_raw_get_object_gone +test_object_raw_get_x_amz_expires_not_expired +test_object_raw_get_x_amz_expires_not_expired_tenant +test_object_raw_get_x_amz_expires_out_max_range +test_object_raw_get_x_amz_expires_out_positive_range +test_object_raw_get_x_amz_expires_out_range_zero +test_object_raw_put_authenticated_expired +test_object_raw_response_headers +test_object_read_unreadable +test_object_requestid_matches_header_on_error +test_object_set_get_unicode_metadata +test_object_write_with_chunked_transfer_encoding +test_post_object_anonymous_request +test_post_object_authenticated_no_content_type +test_post_object_authenticated_request +test_post_object_authenticated_request_bad_access_key +test_post_object_case_insensitive_condition_fields +test_post_object_condition_is_case_sensitive +test_post_object_empty_conditions +test_post_object_escaped_field_values +test_post_object_expired_policy +test_post_object_expires_is_case_sensitive +test_post_object_ignored_header +test_post_object_invalid_access_key +test_post_object_invalid_content_length_argument +test_post_object_invalid_date_format +test_post_object_invalid_request_field_value +test_post_object_invalid_signature +test_post_object_missing_conditions_list +test_post_object_missing_content_length_argument +test_post_object_missing_expires_condition +test_post_object_missing_policy_condition +test_post_object_missing_signature +test_post_object_no_key_specified +test_post_object_request_missing_policy_specified_field +test_post_object_set_invalid_success_code +test_post_object_set_key_from_filename +test_post_object_set_success_code +test_post_object_success_redirect_action +test_post_object_tags_anonymous_request +test_post_object_tags_authenticated_request +test_post_object_upload_larger_than_chunk +test_post_object_upload_size_below_minimum +test_post_object_upload_size_limit_exceeded +test_post_object_upload_size_rgw_chunk_size_bug +test_post_object_user_specified_header +test_post_object_wrong_bucket +test_put_bucket_acl_grant_group_read +test_put_bucket_logging_account_j +test_put_bucket_logging_account_s +test_put_bucket_logging_extensions +test_put_bucket_logging_policy_wildcard_objects +test_put_bucket_logging_tenant_j +test_put_bucket_logging_tenant_s +test_put_bucket_ownership_bucket_owner_enforced +test_put_bucket_ownership_bucket_owner_preferred +test_put_bucket_ownership_object_writer +test_put_current_object_if_match +test_put_current_object_if_none_match +test_put_delete_tags +test_put_max_tags +test_put_modify_tags +test_put_obj_with_tags +test_put_object_current_if_match +test_put_object_if_match +test_put_object_ifmatch_failed +test_put_object_ifmatch_good +test_put_object_ifmatch_nonexisted_failed +test_put_object_ifmatch_overwrite_existed_good +test_put_object_ifnonmatch_failed +test_put_object_ifnonmatch_good +test_put_object_ifnonmatch_nonexisted_good +test_put_object_ifnonmatch_overwrite_existed_failed +test_put_tags_acl_public +test_ranged_big_request_response_code +test_ranged_request_response_code +test_ranged_request_return_trailing_bytes_response_code +test_ranged_request_skip_leading_bytes_response_code +test_read_through +test_restore_noncur_obj +test_restore_object_permanent +test_restore_object_temporary +test_set_cors +test_sse_kms_default_post_object_authenticated_request +test_sse_kms_default_upload_1b +test_sse_kms_default_upload_1kb +test_sse_kms_default_upload_1mb +test_sse_kms_default_upload_8mb +test_sse_kms_method_head +test_sse_kms_multipart_invalid_chunks_1 +test_sse_kms_multipart_invalid_chunks_2 +test_sse_kms_multipart_upload +test_sse_kms_post_object_authenticated_request +test_sse_kms_present +test_sse_kms_transfer_13b +test_sse_kms_transfer_1MB +test_sse_kms_transfer_1b +test_sse_kms_transfer_1kb +test_sse_s3_default_method_head +test_sse_s3_default_multipart_upload +test_sse_s3_default_post_object_authenticated_request +test_sse_s3_default_upload_1b +test_sse_s3_default_upload_1kb +test_sse_s3_default_upload_1mb +test_sse_s3_default_upload_8mb +test_sse_s3_encrypted_upload_1b +test_sse_s3_encrypted_upload_1kb +test_sse_s3_encrypted_upload_1mb +test_sse_s3_encrypted_upload_8mb +test_versioned_object_acl_no_version_specified +test_versioning_copy_obj_version +test_versioning_multi_object_delete_with_marker_create +test_versioning_obj_create_overwrite_multipart +test_versioning_obj_suspended_copy +test_versioning_stack_delete_merkers diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index 8fe03390..8bf9c304 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -34,132 +34,9 @@ TEST_MODE="${TEST_MODE:-single}" MAXFAIL="${MAXFAIL:-1}" XDIST="${XDIST:-0}" -# ============================================================================= -# MARKEXPR: pytest marker expression to exclude test categories -# ============================================================================= -# These markers exclude entire test categories via pytest's -m option. -# Use MARKEXPR env var to override the default exclusions. -# -# Excluded categories: -# - Unimplemented S3 features: lifecycle, versioning, s3website, bucket_logging, encryption -# - Ceph/RGW specific tests: fails_on_aws, fails_on_rgw, fails_on_dbstore -# - IAM features: iam_account, iam_tenant, iam_role, iam_user, iam_cross_account -# - Other unimplemented: sns, sse_s3, storage_class, test_of_sts, webidentity_test -# ============================================================================= -if [[ -z "${MARKEXPR:-}" ]]; then - EXCLUDED_MARKERS=( - # Unimplemented S3 features - "lifecycle" - "versioning" - "s3website" - "bucket_logging" - "encryption" - # Ceph/RGW specific tests (not standard S3) - "fails_on_aws" # Tests for Ceph/RGW specific features (X-RGW-* headers, etc.) - "fails_on_rgw" # Known RGW issues we don't need to replicate - "fails_on_dbstore" # Ceph dbstore backend specific - # IAM features requiring additional setup - "iam_account" - "iam_tenant" - "iam_role" - "iam_user" - "iam_cross_account" - # Other unimplemented features - "sns" # SNS notification - "sse_s3" # Server-side encryption with S3-managed keys - "storage_class" # Storage class features - "test_of_sts" # STS token service - "webidentity_test" # Web Identity federation - ) - # Build MARKEXPR from array: "not marker1 and not marker2 and ..." - MARKEXPR="" - for marker in "${EXCLUDED_MARKERS[@]}"; do - if [[ -n "${MARKEXPR}" ]]; then - MARKEXPR+=" and " - fi - MARKEXPR+="not ${marker}" - done -fi - -# ============================================================================= -# TESTEXPR: pytest -k expression to exclude specific tests by name -# ============================================================================= -# These patterns exclude specific tests via pytest's -k option (name matching). -# Use TESTEXPR env var to override the default exclusions. -# -# Exclusion reasons are documented inline below. -# ============================================================================= -if [[ -z "${TESTEXPR:-}" ]]; then - EXCLUDED_TESTS=( - # POST Object (HTML form upload) - not implemented - "test_post_object" - # ACL-dependent tests - ACL not implemented - "test_bucket_list_objects_anonymous" # requires PutBucketAcl - "test_bucket_listv2_objects_anonymous" # requires PutBucketAcl - "test_bucket_concurrent_set_canned_acl" # ACL not implemented - "test_expected_bucket_owner" # requires PutBucketAcl - "test_bucket_acl" # Bucket ACL not implemented - "test_object_acl" # Object ACL not implemented - "test_put_bucket_acl" # PutBucketAcl not implemented - "test_object_anon" # Anonymous access requires ACL - "test_access_bucket" # Access control requires ACL - "test_100_continue" # requires ACL - # Chunked encoding - not supported - "test_object_write_with_chunked_transfer_encoding" - "test_object_content_encoding_aws_chunked" - # CORS - not implemented - "test_cors" - "test_set_cors" - # Presigned URL edge cases - "test_object_raw" # Raw presigned URL tests - # Error response format differences - "test_bucket_create_exists" # Error format issue - "test_bucket_recreate_not_overriding" # Error format issue - "test_list_buckets_invalid_auth" # 401 vs 403 - "test_object_delete_key_bucket_gone" # 403 vs 404 - "test_abort_multipart_upload_not_found" # Error code issue - # ETag conditional request edge cases - "test_get_object_ifmatch_failed" - "test_get_object_ifnonematch" - # Copy operation edge cases - "test_object_copy_to_itself" # Copy validation - "test_object_copy_not_owned_bucket" # Cross-account access - "test_multipart_copy_invalid_range" # Multipart validation - # Timing-sensitive tests - "test_versioning_concurrent_multi_object_delete" - ) - # Build TESTEXPR from array: "not test1 and not test2 and ..." - TESTEXPR="" - for pattern in "${EXCLUDED_TESTS[@]}"; do - if [[ -n "${TESTEXPR}" ]]; then - TESTEXPR+=" and " - fi - TESTEXPR+="not ${pattern}" - done -fi - -# Configuration file paths -S3TESTS_CONF_TEMPLATE="${S3TESTS_CONF_TEMPLATE:-.github/s3tests/s3tests.conf}" -S3TESTS_CONF="${S3TESTS_CONF:-s3tests.conf}" - -# Service deployment mode: "build", "binary", "docker", or "existing" -# - "build": Compile with cargo build --release and run (default) -# - "binary": Use pre-compiled binary (RUSTFS_BINARY path or default) -# - "docker": Build Docker image and run in container -# - "existing": Use already running service (skip start, use S3_HOST and S3_PORT) -DEPLOY_MODE="${DEPLOY_MODE:-build}" -RUSTFS_BINARY="${RUSTFS_BINARY:-}" -NO_CACHE="${NO_CACHE:-false}" - -# Directories +# Directories (define early for use in test list loading) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}" -CONTAINER_NAME="rustfs-${TEST_MODE}" -NETWORK_NAME="rustfs-net" -DATA_ROOT="${DATA_ROOT:-target}" -DATA_DIR="${PROJECT_ROOT}/${DATA_ROOT}/test-data/${CONTAINER_NAME}" -RUSTFS_PID="" # Colors for output RED='\033[0;31m' @@ -180,6 +57,137 @@ log_error() { echo -e "${RED}[ERROR]${NC} $*" } +# ============================================================================= +# Test Classification Files +# ============================================================================= +# Tests are classified into three categories stored in text files: +# - non_standard_tests.txt: Ceph/RGW specific tests (permanently excluded) +# - unimplemented_tests.txt: Standard S3 features not yet implemented +# - implemented_tests.txt: Tests that should pass on RustFS +# +# By default, only tests listed in implemented_tests.txt are run. +# Use TESTEXPR env var to override and run custom test selection. +# ============================================================================= + +# Test list files location +TEST_LISTS_DIR="${SCRIPT_DIR}" +IMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/implemented_tests.txt" +NON_STANDARD_TESTS_FILE="${TEST_LISTS_DIR}/non_standard_tests.txt" +UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt" + +# ============================================================================= +# build_testexpr_from_file: Read test names from file and build pytest -k expr +# ============================================================================= +# Reads test names from a file (one per line, ignoring comments and empty lines) +# and builds a pytest -k expression to include only those tests. +# ============================================================================= +build_testexpr_from_file() { + local file="$1" + local expr="" + + if [[ ! -f "${file}" ]]; then + log_error "Test list file not found: ${file}" + return 1 + fi + + while IFS= read -r line || [[ -n "$line" ]]; do + # Skip empty lines and comments + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + # Trim whitespace + line=$(echo "$line" | xargs) + [[ -z "$line" ]] && continue + + if [[ -n "${expr}" ]]; then + expr+=" or " + fi + expr+="${line}" + done < "${file}" + + echo "${expr}" +} + +# ============================================================================= +# MARKEXPR: pytest marker expression (safety net for marker-based filtering) +# ============================================================================= +# Even though we use file-based test selection, we keep marker exclusions +# as a safety net to ensure no non-standard tests slip through. +# ============================================================================= +if [[ -z "${MARKEXPR:-}" ]]; then + # Minimal marker exclusions as safety net (file-based filtering is primary) + MARKEXPR="not fails_on_aws and not fails_on_rgw and not fails_on_dbstore" +fi + +# ============================================================================= +# TESTEXPR: pytest -k expression to select specific tests +# ============================================================================= +# By default, builds an inclusion expression from implemented_tests.txt. +# Use TESTEXPR env var to override with custom selection. +# +# The file-based approach provides: +# 1. Clear visibility of which tests are run +# 2. Easy maintenance - edit txt files to add/remove tests +# 3. Separation of concerns - test classification vs test execution +# ============================================================================= +if [[ -z "${TESTEXPR:-}" ]]; then + if [[ -f "${IMPLEMENTED_TESTS_FILE}" ]]; then + log_info "Loading test list from: ${IMPLEMENTED_TESTS_FILE}" + TESTEXPR=$(build_testexpr_from_file "${IMPLEMENTED_TESTS_FILE}") + if [[ -z "${TESTEXPR}" ]]; then + log_error "No tests found in ${IMPLEMENTED_TESTS_FILE}" + exit 1 + fi + # Count tests for logging + TEST_COUNT=$(grep -v '^#' "${IMPLEMENTED_TESTS_FILE}" | grep -v '^[[:space:]]*$' | wc -l | xargs) + log_info "Loaded ${TEST_COUNT} tests from implemented_tests.txt" + else + log_warn "Test list file not found: ${IMPLEMENTED_TESTS_FILE}" + log_warn "Falling back to exclusion-based filtering" + # Fallback to exclusion-based filtering if file doesn't exist + EXCLUDED_TESTS=( + "test_post_object" + "test_bucket_list_objects_anonymous" + "test_bucket_listv2_objects_anonymous" + "test_bucket_concurrent_set_canned_acl" + "test_bucket_acl" + "test_object_acl" + "test_access_bucket" + "test_100_continue" + "test_cors" + "test_object_raw" + "test_versioning" + "test_versioned" + ) + TESTEXPR="" + for pattern in "${EXCLUDED_TESTS[@]}"; do + if [[ -n "${TESTEXPR}" ]]; then + TESTEXPR+=" and " + fi + TESTEXPR+="not ${pattern}" + done + fi +fi + +# Configuration file paths +S3TESTS_CONF_TEMPLATE="${S3TESTS_CONF_TEMPLATE:-.github/s3tests/s3tests.conf}" +S3TESTS_CONF="${S3TESTS_CONF:-s3tests.conf}" + +# Service deployment mode: "build", "binary", "docker", or "existing" +# - "build": Compile with cargo build --release and run (default) +# - "binary": Use pre-compiled binary (RUSTFS_BINARY path or default) +# - "docker": Build Docker image and run in container +# - "existing": Use already running service (skip start, use S3_HOST and S3_PORT) +DEPLOY_MODE="${DEPLOY_MODE:-build}" +RUSTFS_BINARY="${RUSTFS_BINARY:-}" +NO_CACHE="${NO_CACHE:-false}" + +# Additional directories (SCRIPT_DIR and PROJECT_ROOT defined earlier) +ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}" +CONTAINER_NAME="rustfs-${TEST_MODE}" +NETWORK_NAME="rustfs-net" +DATA_ROOT="${DATA_ROOT:-target}" +DATA_DIR="${PROJECT_ROOT}/${DATA_ROOT}/test-data/${CONTAINER_NAME}" +RUSTFS_PID="" + show_usage() { cat << EOF Usage: $0 [OPTIONS] @@ -205,15 +213,22 @@ Environment Variables: S3_ALT_SECRET_KEY - Alt user secret key (default: rustfsalt) MAXFAIL - Stop after N failures (default: 1) XDIST - Enable parallel execution with N workers (default: 0) - MARKEXPR - pytest marker expression (default: exclude unsupported features) - TESTEXPR - pytest -k expression to filter tests by name (default: exclude unimplemented) + MARKEXPR - pytest marker expression (default: safety net exclusions) + TESTEXPR - pytest -k expression (default: from implemented_tests.txt) S3TESTS_CONF_TEMPLATE - Path to s3tests config template (default: .github/s3tests/s3tests.conf) S3TESTS_CONF - Path to generated s3tests config (default: s3tests.conf) DATA_ROOT - Root directory for test data storage (default: target) - Final path: ${DATA_ROOT}/test-data/${CONTAINER_NAME} + Final path: \${DATA_ROOT}/test-data/\${CONTAINER_NAME} + +Test Classification Files (in scripts/s3-tests/): + implemented_tests.txt - Tests that should pass (run by default) + unimplemented_tests.txt - Standard S3 features not yet implemented + non_standard_tests.txt - Ceph/RGW specific tests (permanently excluded) Notes: - - In build mode, if the binary exists and was compiled less than 5 minutes ago, + - Tests are loaded from implemented_tests.txt by default + - Set TESTEXPR to override with custom test selection + - In build mode, if the binary exists and was compiled less than 30 minutes ago, compilation will be skipped unless --no-cache is specified. Examples: diff --git a/scripts/s3-tests/unimplemented_tests.txt b/scripts/s3-tests/unimplemented_tests.txt new file mode 100644 index 00000000..85e9b456 --- /dev/null +++ b/scripts/s3-tests/unimplemented_tests.txt @@ -0,0 +1,191 @@ +# Unimplemented S3 feature tests +# ============================== +# +# These tests cover STANDARD S3 features not yet implemented in RustFS. +# They are TEMPORARILY EXCLUDED and should be enabled as features are added. +# +# Unimplemented features: +# - Versioning: Object versioning support +# - Lifecycle: Object lifecycle management +# - S3 Website: Static website hosting +# - Bucket Logging: Access logging +# - SSE-S3: Server-side encryption with S3-managed keys +# - Object Lock: WORM protection +# - IAM: Identity and Access Management roles/users +# - SNS: Event notifications +# - STS: Security Token Service +# - Checksum: Full checksum validation +# - Conditional writes: If-Match/If-None-Match for writes +# - Object ownership: BucketOwnerEnforced/Preferred +# +# Total: all unimplemented S3 feature tests listed below (keep this comment in sync with the list) + +test_bucket_create_delete_bucket_ownership +test_bucket_logging_owner +test_bucket_policy_deny_self_denied_policy +test_bucket_policy_deny_self_denied_policy_confirm_header +test_bucket_policy_put_obj_kms_s3 +test_bucket_policy_put_obj_s3_kms +test_copy_enc +test_copy_part_enc +test_delete_bucket_encryption_kms +test_delete_bucket_encryption_s3 +test_encryption_key_no_sse_c +test_encryption_sse_c_invalid_md5 +test_encryption_sse_c_method_head +test_encryption_sse_c_multipart_bad_download +test_encryption_sse_c_no_key +test_encryption_sse_c_no_md5 +test_encryption_sse_c_other_key +test_encryption_sse_c_present +test_get_bucket_encryption_kms +test_get_bucket_encryption_s3 +test_get_versioned_object_attributes +test_lifecycle_delete +test_lifecycle_expiration_days0 +test_lifecycle_expiration_header_put +test_lifecycle_get +test_lifecycle_get_no_id +test_lifecycle_id_too_long +test_lifecycle_invalid_status +test_lifecycle_plain_null_version_current_transition +test_lifecycle_same_id +test_lifecycle_set +test_lifecycle_set_date +test_lifecycle_set_deletemarker +test_lifecycle_set_empty_filter +test_lifecycle_set_filter +test_lifecycle_set_invalid_date +test_lifecycle_set_multipart +test_lifecycle_set_noncurrent +test_lifecycle_set_noncurrent_transition +test_lifecycle_transition_encrypted +test_lifecycle_transition_set_invalid_date +test_object_checksum_crc64nvme +test_object_checksum_sha256 +test_object_lock_get_legal_hold_invalid_bucket +test_object_lock_get_obj_lock_invalid_bucket +test_object_lock_get_obj_retention_invalid_bucket +test_object_lock_put_legal_hold_invalid_bucket +test_object_lock_put_obj_lock_enable_after_create +test_object_lock_put_obj_lock_invalid_bucket +test_object_lock_put_obj_retention_invalid_bucket +test_post_object_upload_checksum +test_put_bucket_encryption_kms +test_put_bucket_encryption_s3 +test_put_bucket_logging +test_put_bucket_logging_errors +test_put_bucket_logging_permissions +test_put_bucket_logging_policy_wildcard +test_put_obj_enc_conflict_bad_enc_kms +test_put_obj_enc_conflict_c_kms +test_put_obj_enc_conflict_c_s3 +test_put_obj_enc_conflict_s3_kms +test_rm_bucket_logging +test_sse_kms_no_key +test_sse_kms_not_declared +test_sse_kms_read_declare +test_versioned_concurrent_object_create_and_remove +test_versioned_concurrent_object_create_concurrent_remove +test_versioned_object_acl +test_versioning_bucket_atomic_upload_return_version_id +test_versioning_bucket_create_suspend +test_versioning_bucket_multipart_upload_return_version_id +test_versioning_concurrent_multi_object_delete +test_versioning_multi_object_delete +test_versioning_multi_object_delete_with_marker +test_versioning_obj_create_read_remove +test_versioning_obj_create_read_remove_head +test_versioning_obj_create_versions_remove_all +test_versioning_obj_create_versions_remove_special_names +test_versioning_obj_list_marker +test_versioning_obj_plain_null_version_overwrite +test_versioning_obj_plain_null_version_overwrite_suspended +test_versioning_obj_plain_null_version_removal +test_versioning_obj_suspend_versions + +# Teardown issues (list_object_versions on non-versioned buckets) +test_bucket_list_delimiter_alt +test_bucket_list_delimiter_basic +test_bucket_list_delimiter_dot +test_bucket_list_delimiter_empty +test_bucket_list_delimiter_none +test_bucket_list_delimiter_not_exist +test_bucket_list_delimiter_percentage +test_bucket_list_delimiter_prefix_ends_with_delimiter +test_bucket_list_delimiter_unreadable +test_bucket_list_delimiter_whitespace +test_bucket_list_encoding_basic +test_bucket_listv2_delimiter_alt +test_bucket_listv2_delimiter_basic +test_bucket_listv2_delimiter_dot +test_bucket_listv2_delimiter_empty +test_bucket_listv2_delimiter_none +test_bucket_listv2_delimiter_not_exist +test_bucket_listv2_delimiter_percentage +test_bucket_listv2_delimiter_prefix_ends_with_delimiter +test_bucket_listv2_delimiter_unreadable +test_bucket_listv2_delimiter_whitespace +test_bucket_listv2_encoding_basic + +# Checksum and atomic write tests (require x-amz-checksum-* support) +test_atomic_dual_write_1mb +test_atomic_dual_write_4mb +test_atomic_dual_write_8mb +test_atomic_multipart_upload_write +test_atomic_read_1mb +test_atomic_read_4mb +test_atomic_read_8mb +test_atomic_write_1mb +test_atomic_write_4mb +test_atomic_write_8mb +test_set_bucket_tagging + +# Tests with implementation issues (need investigation) +test_bucket_policy_acl +test_bucket_policy_different_tenant +test_bucketv2_policy_acl +test_multipart_resend_first_finishes_last + +# Multipart abort and policy issues +test_abort_multipart_upload +test_bucket_policy_multipart + +# Tests with prefix conflicts or ACL/tenant dependencies +test_bucket_policy +test_bucket_policy_allow_notprincipal +test_bucket_policy_another_bucket +test_bucket_policy_put_obj_acl +test_bucket_policy_put_obj_grant +test_bucket_policy_tenanted_bucket +test_bucketv2_policy +test_object_presigned_put_object_with_acl +test_object_presigned_put_object_with_acl_tenant +test_object_put_acl_mtime + +# ACL-dependent tests (PutBucketAcl not implemented) +test_block_public_object_canned_acls +test_block_public_put_bucket_acls +test_get_authpublic_acl_bucket_policy_status +test_get_nonpublicpolicy_acl_bucket_policy_status +test_get_public_acl_bucket_policy_status +test_get_publicpolicy_acl_bucket_policy_status +test_ignore_public_acls + +# PublicAccessBlock and tag validation tests +test_block_public_policy +test_block_public_policy_with_principal +test_get_obj_head_tagging +test_get_public_block_deny_bucket_policy +test_get_undefined_public_block +test_put_excess_key_tags +test_put_excess_tags +test_put_excess_val_tags +test_put_get_delete_public_block +test_put_public_block +test_set_get_del_bucket_policy + +# Object attributes and torrent tests +test_create_bucket_no_ownership_controls +test_get_checksum_object_attributes +test_get_object_torrent From 40ad2a6ea9e6e3fd7d8af92bdcdd266134c481c3 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 5 Jan 2026 23:18:08 +0800 Subject: [PATCH 03/12] Remove unused crates (#1394) --- Cargo.lock | 4 ---- crates/config/src/constants/app.rs | 6 ------ crates/credentials/src/constants.rs | 8 ++++---- crates/credentials/src/credentials.rs | 16 ++++++++-------- crates/ecstore/src/rpc/http_auth.rs | 23 ++++++++++++----------- crates/lock/Cargo.toml | 3 --- crates/protos/Cargo.toml | 3 +-- 7 files changed, 25 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90501109..e356e602 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8130,11 +8130,9 @@ name = "rustfs-lock" version = "0.0.5" dependencies = [ "async-trait", - "bytes", "crossbeam-queue", "futures", "parking_lot", - "rustfs-protos", "serde", "serde_json", "smallvec", @@ -8143,7 +8141,6 @@ dependencies = [ "tokio", "tonic", "tracing", - "url", "uuid", ] @@ -8269,7 +8266,6 @@ dependencies = [ "flatbuffers", "prost 0.14.1", "rustfs-common", - "rustfs-credentials", "tonic", "tonic-prost", "tonic-prost-build", diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 0b2035e4..a38affec 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -170,12 +170,6 @@ pub const KI_B: usize = 1024; /// Default value: 1048576 pub const MI_B: usize = 1024 * 1024; -/// Environment variable for gRPC authentication token -/// Used to set the authentication token for gRPC communication -/// Example: RUSTFS_GRPC_AUTH_TOKEN=your_token_here -/// Default value: No default value. RUSTFS_SECRET_KEY value is recommended. -pub const ENV_GRPC_AUTH_TOKEN: &str = "RUSTFS_GRPC_AUTH_TOKEN"; - #[cfg(test)] mod tests { use super::*; diff --git a/crates/credentials/src/constants.rs b/crates/credentials/src/constants.rs index 442e2833..b73968bf 100644 --- a/crates/credentials/src/constants.rs +++ b/crates/credentials/src/constants.rs @@ -27,11 +27,11 @@ pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; /// Example: --secret-key rustfsadmin pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; -/// Environment variable for gRPC authentication token -/// Used to set the authentication token for gRPC communication -/// Example: RUSTFS_GRPC_AUTH_TOKEN=your_token_here +/// Environment variable for RPC authentication token +/// Used to set the authentication token for RPC communication +/// Example: RUSTFS_RPC_SECRET=your_token_here /// Default value: No default value. RUSTFS_SECRET_KEY value is recommended. -pub const ENV_GRPC_AUTH_TOKEN: &str = "RUSTFS_GRPC_AUTH_TOKEN"; +pub const ENV_RPC_SECRET: &str = "RUSTFS_RPC_SECRET"; /// IAM Policy Types /// Used to differentiate between embedded and inherited policies diff --git a/crates/credentials/src/credentials.rs b/crates/credentials/src/credentials.rs index 34aa0fe9..16990599 100644 --- a/crates/credentials/src/credentials.rs +++ b/crates/credentials/src/credentials.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{DEFAULT_SECRET_KEY, ENV_GRPC_AUTH_TOKEN, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE}; +use crate::{DEFAULT_SECRET_KEY, ENV_RPC_SECRET, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE}; use rand::{Rng, RngCore}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -25,8 +25,8 @@ use time::OffsetDateTime; /// Global active credentials static GLOBAL_ACTIVE_CRED: OnceLock = OnceLock::new(); -/// Global gRPC authentication token -static GLOBAL_GRPC_AUTH_TOKEN: OnceLock = OnceLock::new(); +/// Global RPC authentication token +pub static GLOBAL_RUSTFS_RPC_SECRET: OnceLock = OnceLock::new(); /// Initialize the global action credentials /// @@ -181,15 +181,15 @@ pub fn gen_secret_key(length: usize) -> std::io::Result { Ok(key_str) } -/// Get the gRPC authentication token from environment variable +/// Get the RPC authentication token from environment variable /// /// # Returns -/// * `String` - The gRPC authentication token +/// * `String` - The RPC authentication token /// -pub fn get_grpc_token() -> String { - GLOBAL_GRPC_AUTH_TOKEN +pub fn get_rpc_token() -> String { + GLOBAL_RUSTFS_RPC_SECRET .get_or_init(|| { - env::var(ENV_GRPC_AUTH_TOKEN) + env::var(ENV_RPC_SECRET) .unwrap_or_else(|_| get_global_secret_key_opt().unwrap_or_else(|| DEFAULT_SECRET_KEY.to_string())) }) .clone() diff --git a/crates/ecstore/src/rpc/http_auth.rs b/crates/ecstore/src/rpc/http_auth.rs index 525ecda2..e974e79b 100644 --- a/crates/ecstore/src/rpc/http_auth.rs +++ b/crates/ecstore/src/rpc/http_auth.rs @@ -15,11 +15,8 @@ use base64::Engine as _; use base64::engine::general_purpose; use hmac::{Hmac, KeyInit, Mac}; -use http::HeaderMap; -use http::HeaderValue; -use http::Method; -use http::Uri; -use rustfs_credentials::get_global_action_cred; +use http::{HeaderMap, HeaderValue, Method, Uri}; +use rustfs_credentials::{DEFAULT_SECRET_KEY, ENV_RPC_SECRET, get_global_secret_key_opt}; use sha2::Sha256; use time::OffsetDateTime; use tracing::error; @@ -33,12 +30,16 @@ pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService"; /// Get the shared secret for HMAC signing fn get_shared_secret() -> String { - if let Some(cred) = get_global_action_cred() { - cred.secret_key - } else { - // Fallback to environment variable if global credentials are not available - std::env::var("RUSTFS_RPC_SECRET").unwrap_or_else(|_| "rustfs-default-secret".to_string()) - } + rustfs_credentials::GLOBAL_RUSTFS_RPC_SECRET + .get_or_init(|| { + rustfs_utils::get_env_str( + ENV_RPC_SECRET, + get_global_secret_key_opt() + .unwrap_or_else(|| DEFAULT_SECRET_KEY.to_string()) + .as_str(), + ) + }) + .clone() } /// Generate HMAC-SHA256 signature for the given data diff --git a/crates/lock/Cargo.toml b/crates/lock/Cargo.toml index f574b685..ed0dd829 100644 --- a/crates/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -30,15 +30,12 @@ workspace = true [dependencies] async-trait.workspace = true -bytes.workspace = true futures.workspace = true -rustfs-protos.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true tonic.workspace = true tracing.workspace = true -url.workspace = true uuid.workspace = true thiserror.workspace = true parking_lot.workspace = true diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index 29ada67a..f0c7bd2a 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -34,10 +34,9 @@ path = "src/main.rs" [dependencies] rustfs-common.workspace = true -rustfs-credentials = { workspace = true } flatbuffers = { workspace = true } prost = { workspace = true } tonic = { workspace = true, features = ["transport"] } tonic-prost = { workspace = true } tonic-prost-build = { workspace = true } -tracing = { workspace = true } \ No newline at end of file +tracing = { workspace = true } From 5f19eef945dc30d860d5c17924127eef29953398 Mon Sep 17 00:00:00 2001 From: Jan S Date: Mon, 5 Jan 2026 17:41:39 +0100 Subject: [PATCH 04/12] fix: OpenBSD does not support TCPKeepalive intervals (#1382) Signed-off-by: houseme Co-authored-by: houseme Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- crates/utils/src/sys/user_agent.rs | 1 + rustfs/src/server/http.rs | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/utils/src/sys/user_agent.rs b/crates/utils/src/sys/user_agent.rs index 20eee81a..28486e89 100644 --- a/crates/utils/src/sys/user_agent.rs +++ b/crates/utils/src/sys/user_agent.rs @@ -15,6 +15,7 @@ use rustfs_config::VERSION; use std::env; use std::fmt; +#[cfg(not(any(target_os = "openbsd", target_os = "freebsd")))] use sysinfo::System; /// Business Type Enumeration diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index c698ce4b..3f3473b7 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -30,7 +30,10 @@ use hyper_util::{ }; use metrics::{counter, histogram}; use rustfs_common::GlobalReadiness; +#[cfg(not(target_os = "openbsd"))] use rustfs_config::{MI_B, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +#[cfg(target_os = "openbsd")] +use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature}; use rustfs_protos::proto_gen::node_service::node_service_server::NodeServiceServer; use rustfs_utils::net::parse_and_resolve_address; @@ -375,12 +378,20 @@ pub async fn start_http_server( // Enable TCP Keepalive to detect dead clients (e.g. power loss) // Idle: 10s, Interval: 5s, Retries: 3 - let ka = TcpKeepalive::new() - .with_time(Duration::from_secs(10)) - .with_interval(Duration::from_secs(5)); + let ka = { + #[cfg(not(target_os = "openbsd"))] + let ka = TcpKeepalive::new() + .with_time(Duration::from_secs(10)) + .with_interval(Duration::from_secs(5)) + .with_retries(3); - #[cfg(not(any(target_os = "openbsd", target_os = "netbsd")))] - let ka = ka.with_retries(3); + // On OpenBSD socket2 only supports configuring the initial + // TCP keepalive timeout; intervals and retries cannot be set. + #[cfg(target_os = "openbsd")] + let ka = TcpKeepalive::new().with_time(Duration::from_secs(10)); + + ka + }; if let Err(err) = socket_ref.set_tcp_keepalive(&ka) { warn!(?err, "Failed to set TCP_KEEPALIVE"); @@ -389,9 +400,11 @@ pub async fn start_http_server( if let Err(err) = socket_ref.set_tcp_nodelay(true) { warn!(?err, "Failed to set TCP_NODELAY"); } + #[cfg(not(any(target_os = "openbsd")))] if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { warn!(?err, "Failed to set set_recv_buffer_size"); } + #[cfg(not(any(target_os = "openbsd")))] if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { warn!(?err, "Failed to set set_send_buffer_size"); } From 18fb920fa4c3d434e477d6d16cefc2f11468a5b1 Mon Sep 17 00:00:00 2001 From: Jan S Date: Tue, 6 Jan 2026 03:26:09 +0100 Subject: [PATCH 05/12] Remove the sysctl crate and use libc's sysctl call interface (#1396) Co-authored-by: houseme --- Cargo.lock | 28 +----------------------- Cargo.toml | 1 - rustfs/Cargo.toml | 4 +--- rustfs/src/server/http.rs | 46 +++++++++++++++++++++++++++++---------- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e356e602..875747ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3419,18 +3419,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.113", -] - [[package]] name = "enum_dispatch" version = "0.3.13" @@ -7767,6 +7755,7 @@ dependencies = [ "hyper", "hyper-util", "jemalloc_pprof", + "libc", "libsystemd", "libunftp", "matchit 0.9.1", @@ -7815,7 +7804,6 @@ dependencies = [ "socket2", "ssh-key", "subtle", - "sysctl", "sysinfo", "thiserror 2.0.17", "tikv-jemalloc-ctl", @@ -9680,20 +9668,6 @@ dependencies = [ "syn 2.0.113", ] -[[package]] -name = "sysctl" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cca424247104946a59dacd27eaad296223b7feec3d168a6dd04585183091eb0b" -dependencies = [ - "bitflags 2.10.0", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 2.0.17", - "walkdir", -] - [[package]] name = "sysinfo" version = "0.37.2" diff --git a/Cargo.toml b/Cargo.toml index 90ec9065..083860d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -234,7 +234,6 @@ snafu = "0.8.9" snap = "1.1.1" starshard = { version = "0.6.0", features = ["rayon", "async", "serde"] } strum = { version = "0.27.2", features = ["derive"] } -sysctl = "0.7.1" sysinfo = "0.37.2" temp-env = "0.3.6" tempfile = "3.24.0" diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index c50548cd..df2f23d9 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -125,6 +125,7 @@ url = { workspace = true } urlencoding = { workspace = true } uuid = { workspace = true } zip = { workspace = true } +libc = { workspace = true } # Observability and Metrics metrics = { workspace = true } @@ -135,9 +136,6 @@ russh = { workspace = true } russh-sftp = { workspace = true } ssh-key = { workspace = true } -[target.'cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))'.dependencies] -sysctl = { workspace = true } - [target.'cfg(target_os = "linux")'.dependencies] libsystemd.workspace = true diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 3f3473b7..90589dd4 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -743,34 +743,58 @@ fn check_auth(req: Request<()>) -> std::result::Result, Status> { Ok(req) } +// For macOS and BSD variants use the syscall way of getting the connection queue length. +#[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))] +#[allow(unsafe_code)] +fn get_conn_queue_len() -> i32 { + const DEFAULT_BACKLOG: i32 = 1024; + + #[cfg(target_os = "openbsd")] + let mut name = [libc::CTL_KERN, libc::KERN_SOMAXCONN]; + #[cfg(any(target_os = "netbsd", target_os = "macos", target_os = "freebsd"))] + let mut name = [libc::CTL_KERN, libc::KERN_IPC, libc::KIPC_SOMAXCONN]; + let mut buf = [0; 1]; + let mut buf_len = std::mem::size_of_val(&buf); + + if unsafe { + libc::sysctl( + name.as_mut_ptr(), + name.len() as u32, + buf.as_mut_ptr() as *mut libc::c_void, + &mut buf_len, + std::ptr::null_mut(), + 0, + ) + } != 0 + { + return DEFAULT_BACKLOG; + } + + buf[0] +} + /// Determines the listen backlog size. /// /// It tries to read the system's maximum connection queue length (`somaxconn`). /// If reading fails, it falls back to a default value (e.g., 1024). /// This makes the backlog size adaptive to the system configuration. fn get_listen_backlog() -> i32 { - const DEFAULT_BACKLOG: i32 = 1024; - #[cfg(target_os = "linux")] { + const DEFAULT_BACKLOG: i32 = 1024; + // For Linux, read from /proc/sys/net/core/somaxconn match std::fs::read_to_string("/proc/sys/net/core/somaxconn") { Ok(s) => s.trim().parse().unwrap_or(DEFAULT_BACKLOG), Err(_) => DEFAULT_BACKLOG, } } + #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))] { - // For macOS and BSD variants, use sysctl - use sysctl::Sysctl; - match sysctl::Ctl::new("kern.ipc.somaxconn") { - Ok(ctl) => match ctl.value() { - Ok(sysctl::CtlValue::Int(val)) => val, - _ => DEFAULT_BACKLOG, - }, - Err(_) => DEFAULT_BACKLOG, - } + get_conn_queue_len() } + #[cfg(not(any( target_os = "linux", target_os = "macos", From b95bee64b2fb55e88c0c0a2205d6be9e143c202f Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Tue, 6 Jan 2026 14:53:26 +0800 Subject: [PATCH 06/12] fix: Correct import permissions (#1402) --- rustfs/src/admin/handlers/user.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index a4981f30..80233b4d 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -651,7 +651,7 @@ impl Operation for ImportIam { &cred, owner, false, - vec![Action::AdminAction(AdminAction::ExportIAMAction)], + vec![Action::AdminAction(AdminAction::ImportIAMAction)], req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), ) .await?; From e4ad86ada6b92cc5036664484be903b423ea425b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Tue, 6 Jan 2026 21:13:39 +0800 Subject: [PATCH 07/12] test(s3): add 9 delimiter list tests to implemented tests (#1410) --- scripts/s3-tests/implemented_tests.txt | 11 ++++++++++- scripts/s3-tests/unimplemented_tests.txt | 10 +--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/s3-tests/implemented_tests.txt b/scripts/s3-tests/implemented_tests.txt index e88688de..8034aec0 100644 --- a/scripts/s3-tests/implemented_tests.txt +++ b/scripts/s3-tests/implemented_tests.txt @@ -17,7 +17,7 @@ # - Metadata: User-defined metadata # - Conditional GET: If-Match, If-None-Match, If-Modified-Since # -# Total: 109 tests +# Total: 118 tests test_basic_key_count test_bucket_create_naming_bad_short_one @@ -63,6 +63,15 @@ test_bucket_list_prefix_none test_bucket_list_prefix_not_exist test_bucket_list_prefix_unreadable test_bucket_list_special_prefix +test_bucket_list_delimiter_alt +test_bucket_list_delimiter_dot +test_bucket_list_delimiter_empty +test_bucket_list_delimiter_none +test_bucket_list_delimiter_not_exist +test_bucket_list_delimiter_percentage +test_bucket_list_delimiter_prefix_ends_with_delimiter +test_bucket_list_delimiter_unreadable +test_bucket_list_delimiter_whitespace test_bucket_listv2_continuationtoken test_bucket_listv2_continuationtoken_empty test_bucket_listv2_fetchowner_defaultempty diff --git a/scripts/s3-tests/unimplemented_tests.txt b/scripts/s3-tests/unimplemented_tests.txt index 85e9b456..27a35ee1 100644 --- a/scripts/s3-tests/unimplemented_tests.txt +++ b/scripts/s3-tests/unimplemented_tests.txt @@ -105,16 +105,8 @@ test_versioning_obj_plain_null_version_removal test_versioning_obj_suspend_versions # Teardown issues (list_object_versions on non-versioned buckets) -test_bucket_list_delimiter_alt +# These tests pass but have cleanup issues with list_object_versions test_bucket_list_delimiter_basic -test_bucket_list_delimiter_dot -test_bucket_list_delimiter_empty -test_bucket_list_delimiter_none -test_bucket_list_delimiter_not_exist -test_bucket_list_delimiter_percentage -test_bucket_list_delimiter_prefix_ends_with_delimiter -test_bucket_list_delimiter_unreadable -test_bucket_list_delimiter_whitespace test_bucket_list_encoding_basic test_bucket_listv2_delimiter_alt test_bucket_listv2_delimiter_basic From 356dc7e0c2b4438ef3b447160003ea4571fb99ad Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Tue, 6 Jan 2026 21:47:18 +0800 Subject: [PATCH 08/12] feat: Add permission verification for account creation (#1401) Co-authored-by: loverustfs --- crates/policy/src/policy/policy.rs | 105 ++++++++++++++++++- rustfs/src/admin/handlers/service_account.rs | 4 +- rustfs/src/admin/handlers/user.rs | 12 ++- 3 files changed, 113 insertions(+), 8 deletions(-) diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index 6cde4e9d..7eb9e2a3 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -63,16 +63,23 @@ pub struct Policy { impl Policy { pub async fn is_allowed(&self, args: &Args<'_>) -> bool { + // First, check all Deny statements - if any Deny matches, deny the request for statement in self.statements.iter().filter(|s| matches!(s.effect, Effect::Deny)) { if !statement.is_allowed(args).await { return false; } } - if args.deny_only || args.is_owner { + // Owner has all permissions + if args.is_owner { return true; } + if args.deny_only { + return false; + } + + // Check Allow statements for statement in self.statements.iter().filter(|s| matches!(s.effect, Effect::Allow)) { if statement.is_allowed(args).await { return true; @@ -594,6 +601,102 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_deny_only_security_fix() -> Result<()> { + let data = r#" +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::bucket1/*"] + } + ] +} +"#; + + let policy = Policy::parse_config(data.as_bytes())?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + + // Test with deny_only=true but no matching Allow statement + let args_deny_only = Args { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::PutObjectAction), + bucket: "bucket2", + conditions: &conditions, + is_owner: false, + object: "test.txt", + claims: &claims, + deny_only: true, // Should NOT automatically allow + }; + + // Should return false because deny_only=true, regardless of whether there's a matching Allow statement + assert!( + !policy.is_allowed(&args_deny_only).await, + "deny_only should return false when deny_only=true, regardless of Allow statements" + ); + + // Test with deny_only=true and matching Allow statement + let args_deny_only_allowed = Args { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::GetObjectAction), + bucket: "bucket1", + conditions: &conditions, + is_owner: false, + object: "test.txt", + claims: &claims, + deny_only: true, + }; + + // Should return false because deny_only=true prevents checking Allow statements (unless is_owner=true) + assert!( + !policy.is_allowed(&args_deny_only_allowed).await, + "deny_only should return false even with matching Allow statement" + ); + + // Test with deny_only=false (normal case) + let args_normal = Args { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::GetObjectAction), + bucket: "bucket1", + conditions: &conditions, + is_owner: false, + object: "test.txt", + claims: &claims, + deny_only: false, + }; + + // Should return true because there's an Allow statement + assert!( + policy.is_allowed(&args_normal).await, + "normal policy evaluation should allow with matching Allow statement" + ); + + let args_owner_deny_only = Args { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::PutObjectAction), + bucket: "bucket2", + conditions: &conditions, + is_owner: true, // Owner has all permissions + object: "test.txt", + claims: &claims, + deny_only: true, // Even with deny_only=true, owner should be allowed + }; + + assert!( + policy.is_allowed(&args_owner_deny_only).await, + "owner should retain all permissions even when deny_only=true" + ); + + Ok(()) + } + #[tokio::test] async fn test_aws_username_policy_variable() -> Result<()> { let data = r#" diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index 739cdbbb..503bc005 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -112,8 +112,6 @@ impl Operation for AddServiceAccount { return Err(s3_error!(InvalidRequest, "iam not init")); }; - let deny_only = constant_time_eq(&cred.access_key, &target_user) || constant_time_eq(&cred.parent_user, &target_user); - if !iam_store .is_allowed(&Args { account: &cred.access_key, @@ -130,7 +128,7 @@ impl Operation for AddServiceAccount { is_owner: owner, object: "", claims: cred.claims.as_ref().unwrap_or(&HashMap::new()), - deny_only, + deny_only: false, // Always require explicit Allow permission }) .await { diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index 80233b4d..e7ddf1be 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -118,12 +118,14 @@ impl Operation for AddUser { return Err(s3_error!(InvalidArgument, "access key is not utf8")); } - let deny_only = ak == cred.access_key; + // Security fix: Always require explicit Allow permission for CreateUser + // Do not use deny_only to bypass permission checks, even when creating for self + // This ensures consistent security semantics and prevents privilege escalation validate_admin_request( &req.headers, &cred, owner, - deny_only, + false, // Always require explicit Allow permission vec![Action::AdminAction(AdminAction::CreateUserAdminAction)], req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), ) @@ -375,12 +377,14 @@ impl Operation for GetUserInfo { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - let deny_only = ak == cred.access_key; + // Security fix: Always require explicit Allow permission for GetUser + // Users should have explicit GetUser permission to view account information + // This ensures consistent security semantics across all admin operations validate_admin_request( &req.headers, &cred, owner, - deny_only, + false, // Always require explicit Allow permission vec![Action::AdminAction(AdminAction::GetUserAdminAction)], req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), ) From 02f809312b5689d870dbf1ef87d24f5a7d2548d2 Mon Sep 17 00:00:00 2001 From: Jan S Date: Tue, 6 Jan 2026 16:41:12 +0100 Subject: [PATCH 09/12] Fix windows missing default backlog (#1405) Co-authored-by: houseme --- rustfs/src/server/http.rs | 60 ++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 90589dd4..33c0bf40 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -743,10 +743,26 @@ fn check_auth(req: Request<()>) -> std::result::Result, Status> { Ok(req) } +/// Determines the listen backlog size. +/// +/// It tries to read the system's maximum connection queue length (`somaxconn`). +/// If reading fails, it falls back to a default value (e.g., 1024). +/// This makes the backlog size adaptive to the system configuration. +#[cfg(target_os = "linux")] +fn get_listen_backlog() -> i32 { + const DEFAULT_BACKLOG: i32 = 1024; + + // For Linux, read from /proc/sys/net/core/somaxconn + match std::fs::read_to_string("/proc/sys/net/core/somaxconn") { + Ok(s) => s.trim().parse().unwrap_or(DEFAULT_BACKLOG), + Err(_) => DEFAULT_BACKLOG, + } +} + // For macOS and BSD variants use the syscall way of getting the connection queue length. #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))] #[allow(unsafe_code)] -fn get_conn_queue_len() -> i32 { +fn get_listen_backlog() -> i32 { const DEFAULT_BACKLOG: i32 = 1024; #[cfg(target_os = "openbsd")] @@ -773,37 +789,15 @@ fn get_conn_queue_len() -> i32 { buf[0] } -/// Determines the listen backlog size. -/// -/// It tries to read the system's maximum connection queue length (`somaxconn`). -/// If reading fails, it falls back to a default value (e.g., 1024). -/// This makes the backlog size adaptive to the system configuration. +// Fallback for Windows and other operating systems +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" +)))] fn get_listen_backlog() -> i32 { - #[cfg(target_os = "linux")] - { - const DEFAULT_BACKLOG: i32 = 1024; - - // For Linux, read from /proc/sys/net/core/somaxconn - match std::fs::read_to_string("/proc/sys/net/core/somaxconn") { - Ok(s) => s.trim().parse().unwrap_or(DEFAULT_BACKLOG), - Err(_) => DEFAULT_BACKLOG, - } - } - - #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))] - { - get_conn_queue_len() - } - - #[cfg(not(any( - target_os = "linux", - target_os = "macos", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd" - )))] - { - // Fallback for Windows and other operating systems - DEFAULT_BACKLOG - } + const DEFAULT_BACKLOG: i32 = 1024; + DEFAULT_BACKLOG } From 3ce99939a348a7651e89c348f01a953b4ae0c3b6 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 6 Jan 2026 23:59:08 +0800 Subject: [PATCH 10/12] fix: improve memory ordering for disk health tracker (#1412) --- crates/ecstore/src/disk/disk_store.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index baab323f..bc736bb1 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -95,22 +95,22 @@ impl DiskHealthTracker { /// Check if disk is faulty pub fn is_faulty(&self) -> bool { - self.status.load(Ordering::Relaxed) == DISK_HEALTH_FAULTY + self.status.load(Ordering::Acquire) == DISK_HEALTH_FAULTY } /// Set disk as faulty pub fn set_faulty(&self) { - self.status.store(DISK_HEALTH_FAULTY, Ordering::Relaxed); + self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); } /// Set disk as OK pub fn set_ok(&self) { - self.status.store(DISK_HEALTH_OK, Ordering::Relaxed); + self.status.store(DISK_HEALTH_OK, Ordering::Release); } pub fn swap_ok_to_faulty(&self) -> bool { self.status - .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::Relaxed, Ordering::Relaxed) + .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed) .is_ok() } @@ -131,7 +131,7 @@ impl DiskHealthTracker { /// Get last success timestamp pub fn last_success(&self) -> i64 { - self.last_success.load(Ordering::Relaxed) + self.last_success.load(Ordering::Acquire) } } From 359c9d2d26c67493bc037d2a580cee56edc2171e Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 7 Jan 2026 10:44:35 +0800 Subject: [PATCH 11/12] Enhance Object Version Management and Replication Status Handling (#1413) --- crates/ahm/src/scanner/local_scan/mod.rs | 2 +- crates/ecstore/src/disk/local.rs | 96 ++++++++++++++++++++++-- crates/ecstore/src/set_disk.rs | 18 +---- crates/ecstore/src/store_api.rs | 54 +++++++++++-- crates/filemeta/src/fileinfo.rs | 6 +- crates/filemeta/src/filemeta.rs | 91 ++++++++++++++++++++-- crates/filemeta/src/metacache.rs | 24 ++++-- crates/utils/src/http/headers.rs | 1 + 8 files changed, 246 insertions(+), 46 deletions(-) diff --git a/crates/ahm/src/scanner/local_scan/mod.rs b/crates/ahm/src/scanner/local_scan/mod.rs index 24a5a260..16b3cc55 100644 --- a/crates/ahm/src/scanner/local_scan/mod.rs +++ b/crates/ahm/src/scanner/local_scan/mod.rs @@ -306,7 +306,7 @@ fn compute_object_usage(bucket: &str, object: &str, file_meta: &FileMeta) -> Res versions_count = versions_count.saturating_add(1); if latest_file_info.is_none() - && let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false) + && let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false, false) { latest_file_info = Some(info); } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 7f2c5c48..09991fc5 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -21,6 +21,7 @@ use super::{ }; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; +use crate::config::storageclass::DEFAULT_INLINE_BLOCK; use crate::data_usage::local_snapshot::ensure_data_usage_layout; use crate::disk::error::FileAccessDeniedWithContext; use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}; @@ -489,9 +490,17 @@ impl LocalDisk { let file_dir = self.get_bucket_path(volume)?; let (data, _) = self.read_raw(volume, file_dir, file_path, opts.read_data).await?; - get_file_info(&data, volume, path, version_id, FileInfoOpts { data: opts.read_data }) - .await - .map_err(|_e| DiskError::Unexpected) + get_file_info( + &data, + volume, + path, + version_id, + FileInfoOpts { + data: opts.read_data, + include_free_versions: false, + }, + ) + .map_err(|_e| DiskError::Unexpected) } // Batch metadata reading for multiple objects @@ -2240,20 +2249,93 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "debug", skip(self))] async fn read_version( &self, - _org_volume: &str, + org_volume: &str, volume: &str, path: &str, version_id: &str, opts: &ReadOptions, ) -> Result { + if !org_volume.is_empty() { + let org_volume_path = self.get_bucket_path(org_volume)?; + if !skip_access_checks(org_volume) { + access(&org_volume_path) + .await + .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; + } + } + let file_path = self.get_object_path(volume, path)?; - let file_dir = self.get_bucket_path(volume)?; + let volume_dir = self.get_bucket_path(volume)?; + + check_path_length(file_path.to_string_lossy().as_ref())?; let read_data = opts.read_data; - let (data, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; + let (data, _) = self + .read_raw(volume, volume_dir.clone(), file_path, read_data) + .await + .map_err(|e| { + if e == DiskError::FileNotFound && !version_id.is_empty() { + DiskError::FileVersionNotFound + } else { + e + } + })?; - let fi = get_file_info(&data, volume, path, version_id, FileInfoOpts { data: read_data }).await?; + let mut fi = get_file_info( + &data, + volume, + path, + version_id, + FileInfoOpts { + data: read_data, + include_free_versions: opts.incl_free_versions, + }, + )?; + + if opts.read_data { + if fi.data.as_ref().is_some_and(|d| !d.is_empty()) || fi.size == 0 { + if fi.inline_data() { + return Ok(fi); + } + + if fi.size == 0 || fi.version_id.is_none_or(|v| v.is_nil()) { + fi.set_inline_data(); + return Ok(fi); + }; + if let Some(part) = fi.parts.first() { + let part_path = format!("part.{}", part.number); + let part_path = path_join_buf(&[ + path, + fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(), + part_path.as_str(), + ]); + let part_path = self.get_object_path(volume, part_path.as_str())?; + if lstat(&part_path).await.is_err() { + fi.set_inline_data(); + return Ok(fi); + } + } + + fi.data = None; + } + + let inline = fi.transition_status.is_empty() && fi.data_dir.is_some() && fi.parts.len() == 1; + if inline && fi.shard_file_size(fi.parts[0].actual_size) < DEFAULT_INLINE_BLOCK as i64 { + let part_path = path_join_buf(&[ + path, + fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(), + format!("part.{}", fi.parts[0].number).as_str(), + ]); + let part_path = self.get_object_path(volume, part_path.as_str())?; + + let data = self.read_all_data(volume, volume_dir, part_path.clone()).await.map_err(|e| { + warn!("read_version read_all_data {:?} failed: {e}", part_path); + e + })?; + fi.data = Some(Bytes::from(data)); + } + } Ok(fi) } diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 782c3690..ccf2bcd9 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -1488,17 +1488,7 @@ impl SetDisks { if let Some(disk) = disk && disk.is_online().await { - if version_id.is_empty() { - match disk.read_xl(&bucket, &object, read_data).await { - Ok(info) => { - let fi = file_info_from_raw(info, &bucket, &object, read_data).await?; - Ok(fi) - } - Err(err) => Err(err), - } - } else { - disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await - } + disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await } else { Err(DiskError::DiskNotFound) } @@ -1626,7 +1616,7 @@ impl SetDisks { bucket: &str, object: &str, read_data: bool, - _incl_free_vers: bool, + incl_free_vers: bool, ) -> (Vec, Vec>) { let mut metadata_array = vec![None; fileinfos.len()]; let mut meta_file_infos = vec![FileInfo::default(); fileinfos.len()]; @@ -1676,7 +1666,7 @@ impl SetDisks { ..Default::default() }; - let finfo = match meta.into_fileinfo(bucket, object, "", true, true) { + let finfo = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) { Ok(res) => res, Err(err) => { for item in errs.iter_mut() { @@ -1703,7 +1693,7 @@ impl SetDisks { for (idx, meta_op) in metadata_array.iter().enumerate() { if let Some(meta) = meta_op { - match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, true) { + match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) { Ok(res) => meta_file_infos[idx] = res, Err(err) => errs[idx] = Some(err.into()), } diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index 58d62f87..2071e78a 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -28,14 +28,15 @@ use http::{HeaderMap, HeaderValue}; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{ FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState, - ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map, + ReplicationStatusType, RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, + version_purge_statuses_map, }; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_rio::Checksum; use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader}; use rustfs_utils::CompressionAlgorithm; -use rustfs_utils::http::AMZ_STORAGE_CLASS; use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER}; +use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS}; use rustfs_utils::path::decode_dir_object; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -756,7 +757,24 @@ impl ObjectInfo { .ok() }); - // TODO:ReplicationState + let replication_status_internal = fi + .replication_state_internal + .as_ref() + .and_then(|v| v.replication_status_internal.clone()); + let version_purge_status_internal = fi + .replication_state_internal + .as_ref() + .and_then(|v| v.version_purge_status_internal.clone()); + + let mut replication_status = fi.replication_status(); + if replication_status.is_empty() + && let Some(status) = fi.metadata.get(AMZ_BUCKET_REPLICATION_STATUS).cloned() + && status == ReplicationStatusType::Replica.as_str() + { + replication_status = ReplicationStatusType::Replica; + } + + let version_purge_status = fi.version_purge_status(); let transitioned_object = TransitionedObject { name: fi.transitioned_objname.clone(), @@ -777,10 +795,24 @@ impl ObjectInfo { }; // Extract storage class from metadata, default to STANDARD if not found - let storage_class = metadata - .get(AMZ_STORAGE_CLASS) - .cloned() - .or_else(|| Some(storageclass::STANDARD.to_string())); + let storage_class = if !fi.transition_tier.is_empty() { + Some(fi.transition_tier.clone()) + } else { + fi.metadata + .get(AMZ_STORAGE_CLASS) + .cloned() + .or_else(|| Some(storageclass::STANDARD.to_string())) + }; + + let mut restore_ongoing = false; + let mut restore_expires = None; + if let Some(restore_status) = fi.metadata.get(AMZ_RESTORE).cloned() { + // + if let Ok(restore_status) = parse_restore_obj_status(&restore_status) { + restore_ongoing = restore_status.on_going(); + restore_expires = restore_status.expiry(); + } + } // Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo let parts = fi @@ -798,6 +830,8 @@ impl ObjectInfo { }) .collect(); + // TODO: part checksums + ObjectInfo { bucket: bucket.to_string(), name, @@ -822,6 +856,12 @@ impl ObjectInfo { transitioned_object, checksum: fi.checksum.clone(), storage_class, + restore_ongoing, + restore_expires, + replication_status_internal, + replication_status, + version_purge_status_internal, + version_purge_status, ..Default::default() } } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index d56420ba..2b5755a2 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -505,6 +505,10 @@ impl FileInfo { ReplicationStatusType::Empty } } + + pub fn shard_file_size(&self, total_length: i64) -> i64 { + self.erasure.shard_file_size(total_length) + } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] @@ -590,7 +594,7 @@ impl RestoreStatusOps for RestoreStatus { } } -fn parse_restore_obj_status(restore_hdr: &str) -> Result { +pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect(); let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect(); if progress_tokens.len() != 2 { diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 38d0b677..1939d16c 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -14,7 +14,8 @@ use crate::{ ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState, - ReplicationStatusType, Result, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map, + ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, replication_statuses_map, + version_purge_statuses_map, }; use byteorder::ByteOrder; use bytes::Bytes; @@ -909,6 +910,7 @@ impl FileMeta { path: &str, version_id: &str, read_data: bool, + include_free_versions: bool, all_parts: bool, ) -> Result { let vid = { @@ -921,11 +923,35 @@ impl FileMeta { let mut is_latest = true; let mut succ_mod_time = None; + let mut non_free_versions = self.versions.len(); + + let mut found = false; + let mut found_free_version = None; + let mut found_fi = None; for ver in self.versions.iter() { let header = &ver.header; // TODO: freeVersion + if header.free_version() { + non_free_versions -= 1; + if include_free_versions && found_free_version.is_none() { + let mut found_free_fi = FileMetaVersion::default(); + if found_free_fi.unmarshal_msg(&ver.meta).is_ok() && found_free_fi.version_type != VersionType::Invalid { + let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts); + free_fi.is_latest = true; + found_free_version = Some(free_fi); + } + } + + if header.version_id != Some(vid) { + continue; + } + } + + if found { + continue; + } if !version_id.is_empty() && header.version_id != Some(vid) { is_latest = false; @@ -933,6 +959,8 @@ impl FileMeta { continue; } + found = true; + let mut fi = ver.into_fileinfo(volume, path, all_parts)?; fi.is_latest = is_latest; @@ -947,7 +975,25 @@ impl FileMeta { .map(bytes::Bytes::from); } - fi.num_versions = self.versions.len(); + found_fi = Some(fi); + } + + if !found { + if version_id.is_empty() { + if include_free_versions + && non_free_versions == 0 + && let Some(free_version) = found_free_version + { + return Ok(free_version); + } + return Err(Error::FileNotFound); + } else { + return Err(Error::FileVersionNotFound); + } + } + + if let Some(mut fi) = found_fi { + fi.num_versions = non_free_versions; return Ok(fi); } @@ -1767,14 +1813,27 @@ impl MetaObject { metadata.insert(k.to_owned(), v.to_owned()); } + let tier_fvidkey = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}").to_lowercase(); + let tier_fvmarker_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}").to_lowercase(); + for (k, v) in &self.meta_sys { - if k == AMZ_STORAGE_CLASS && v == b"STANDARD" { + let lower_k = k.to_lowercase(); + + if lower_k == tier_fvidkey || lower_k == tier_fvmarker_key { + continue; + } + + if lower_k == VERSION_PURGE_STATUS_KEY.to_lowercase() { + continue; + } + + if lower_k == AMZ_STORAGE_CLASS.to_lowercase() && v == b"STANDARD" { continue; } if k.starts_with(RESERVED_METADATA_PREFIX) || k.starts_with(RESERVED_METADATA_PREFIX_LOWER) - || k == VERSION_PURGE_STATUS_KEY + || lower_k == VERSION_PURGE_STATUS_KEY.to_lowercase() { metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default()); } @@ -2511,15 +2570,31 @@ pub fn merge_file_meta_versions( merged } -pub async fn file_info_from_raw(ri: RawFileInfo, bucket: &str, object: &str, read_data: bool) -> Result { - get_file_info(&ri.buf, bucket, object, "", FileInfoOpts { data: read_data }).await +pub fn file_info_from_raw( + ri: RawFileInfo, + bucket: &str, + object: &str, + read_data: bool, + include_free_versions: bool, +) -> Result { + get_file_info( + &ri.buf, + bucket, + object, + "", + FileInfoOpts { + data: read_data, + include_free_versions, + }, + ) } pub struct FileInfoOpts { pub data: bool, + pub include_free_versions: bool, } -pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { +pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { let vid = { if version_id.is_empty() { None @@ -2541,7 +2616,7 @@ pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &st }); } - let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?; + let fi = meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?; Ok(fi) } diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 4cc2c9b1..bfb20f80 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Error, FileInfo, FileInfoVersions, FileMeta, FileMetaShallowVersion, Result, VersionType, merge_file_meta_versions}; +use crate::{ + Error, FileInfo, FileInfoOpts, FileInfoVersions, FileMeta, FileMetaShallowVersion, Result, VersionType, get_file_info, + merge_file_meta_versions, +}; use rmp::Marker; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; @@ -141,8 +144,7 @@ impl MetaCacheEntry { }); } - if self.cached.is_some() { - let fm = self.cached.as_ref().unwrap(); + if let Some(fm) = &self.cached { if fm.versions.is_empty() { return Ok(FileInfo { volume: bucket.to_owned(), @@ -154,14 +156,20 @@ impl MetaCacheEntry { }); } - let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; + let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false, true)?; return Ok(fi); } - let mut fm = FileMeta::new(); - fm.unmarshal_msg(&self.metadata)?; - let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; - Ok(fi) + get_file_info( + &self.metadata, + bucket, + self.name.as_str(), + "", + FileInfoOpts { + data: false, + include_free_versions: false, + }, + ) } pub fn file_info_versions(&self, bucket: &str) -> Result { diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index 8e05dfcd..cea4bff7 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -51,6 +51,7 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; // S3 transition restore +pub const AMZ_RESTORE: &str = "x-amz-restore"; pub const AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; pub const AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; From 00f3275603819e80a61d851634281e29ab4e812f Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 7 Jan 2026 13:42:03 +0800 Subject: [PATCH 12/12] rm online check (#1416) --- crates/ecstore/src/set_disk.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index ccf2bcd9..92089d02 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -1485,9 +1485,7 @@ impl SetDisks { let object = object.clone(); let version_id = version_id.clone(); tokio::spawn(async move { - if let Some(disk) = disk - && disk.is_online().await - { + if let Some(disk) = disk { disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await } else { Err(DiskError::DiskNotFound)