From 08e1f4670b293767d3bcd0a86103c89d18bc6ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Tue, 3 Mar 2026 01:08:50 +0800 Subject: [PATCH] fix: restore default CORS fallback and STS object ACL ownership (#2053) Co-authored-by: houseme --- crates/obs/src/telemetry/filter.rs | 59 ++- rustfs/src/app/bucket_usecase.rs | 127 +----- rustfs/src/app/object_usecase.rs | 137 +------ rustfs/src/server/layer.rs | 168 +++++++- rustfs/src/storage/access.rs | 236 +---------- rustfs/src/storage/ecfs.rs | 498 +---------------------- rustfs/src/storage/ecfs_extend.rs | 44 +- rustfs/src/storage/ecfs_test.rs | 84 +++- scripts/s3-tests/excluded_tests.txt | 425 +++++++++++++++++++ scripts/s3-tests/implemented_tests.txt | 83 +--- scripts/s3-tests/non_standard_tests.txt | 7 +- scripts/s3-tests/run.sh | 29 +- scripts/s3-tests/unimplemented_tests.txt | 6 +- 13 files changed, 798 insertions(+), 1105 deletions(-) create mode 100644 scripts/s3-tests/excluded_tests.txt diff --git a/crates/obs/src/telemetry/filter.rs b/crates/obs/src/telemetry/filter.rs index 972596c5..cc75cb0b 100644 --- a/crates/obs/src/telemetry/filter.rs +++ b/crates/obs/src/telemetry/filter.rs @@ -22,10 +22,11 @@ use tracing_subscriber::EnvFilter; /// Build an `EnvFilter` from the given log level string. /// -/// If the `RUST_LOG` environment variable is set, it takes precedence over the -/// provided `logger_level`. For non-verbose levels (`info`, `warn`, `error`), -/// noisy internal crates (`hyper`, `tonic`, `h2`, `reqwest`, `tower`) are -/// automatically silenced to reduce log noise. +/// If `default_level` is provided, it is used directly. Otherwise, the +/// `RUST_LOG` environment variable takes precedence over `logger_level`. +/// For non-verbose levels (`info`, `warn`, `error`), noisy internal crates +/// (`hyper`, `tonic`, `h2`, `reqwest`, `tower`) are automatically silenced +/// based on the effective log configuration. /// /// # Arguments /// * `logger_level` - The desired log level string (e.g., `"info"`, `"debug"`). @@ -35,13 +36,48 @@ use tracing_subscriber::EnvFilter; /// /// # Returns /// A configured `EnvFilter` ready to be attached to a `tracing_subscriber` registry. +fn is_verbose_level(level: &str) -> bool { + matches!(level.to_ascii_lowercase().as_str(), "trace" | "debug") +} + +fn rust_log_requests_verbose(rust_log: &str) -> bool { + rust_log.split(',').any(|directive| { + let directive = directive.trim(); + if directive.is_empty() { + return false; + } + + let level = directive.rsplit('=').next().unwrap_or("").trim(); + is_verbose_level(level) + }) +} + +fn should_suppress_noisy_crates(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool { + if let Some(level) = default_level { + return !is_verbose_level(level); + } + + if let Some(rust_log) = rust_log { + return !rust_log_requests_verbose(rust_log); + } + + !is_verbose_level(logger_level) +} + pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter { let level = default_level.unwrap_or(logger_level); - let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level)); + let rust_log = if default_level.is_none() { + std::env::var("RUST_LOG").ok() + } else { + None + }; + let mut filter = default_level + .map(EnvFilter::new) + .unwrap_or_else(|| EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level))); // Suppress chatty infrastructure crates unless the operator explicitly // requests trace/debug output. - if !matches!(logger_level, "trace" | "debug") { + if should_suppress_noisy_crates(logger_level, default_level, rust_log.as_deref()) { let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"]; for directive in directives { filter = filter.add_directive(format!("{directive}=off").parse().unwrap()); @@ -70,7 +106,7 @@ mod tests { #[test] fn test_build_env_filter_suppresses_noisy_crates() { // For info level, hyper/tonic/etc. should be suppressed with OFF. - let filter = build_env_filter("info", None); + let filter = build_env_filter("debug", Some("info")); let dbg = format!("{filter:?}"); // The Debug output uses `LevelFilter::OFF` for suppressed crates. assert!( @@ -82,7 +118,7 @@ mod tests { #[test] fn test_build_env_filter_debug_no_suppression() { // For debug level, our code does NOT inject any OFF directives. - let filter = build_env_filter("debug", None); + let filter = build_env_filter("info", Some("debug")); let dbg = format!("{filter:?}"); // Verify the filter builds without panicking and contains the debug level. assert!(!dbg.is_empty()); @@ -91,4 +127,11 @@ mod tests { "Expected 'LevelFilter::DEBUG' in filter debug output: {dbg}" ); } + + #[test] + fn test_should_suppress_noisy_crates_respects_rust_log_debug() { + assert!(!should_suppress_noisy_crates("info", None, Some("debug"))); + assert!(!should_suppress_noisy_crates("info", None, Some("s3=info,hyper=debug"))); + assert!(should_suppress_noisy_crates("info", None, Some("info"))); + } } diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index adb4d057..baff520a 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -19,12 +19,9 @@ use crate::auth::get_condition_values; use crate::error::ApiError; use crate::server::RemoteAddr; use crate::storage::access::{ReqInfo, authorize_request, req_info_ref}; -use crate::storage::ecfs::{ - RUSTFS_OWNER, default_owner, is_public_grant, parse_acl_json_or_canned_bucket, serialize_acl, stored_acl_from_canned_bucket, - stored_acl_from_grant_headers, stored_acl_from_policy, stored_grant_to_dto, stored_owner_to_dto, -}; +use crate::storage::ecfs::{RUSTFS_OWNER, default_owner}; use crate::storage::helper::OperationHelper; -use crate::storage::s3_api::{encryption, replication, tagging}; +use crate::storage::s3_api::{acl, encryption, replication, tagging}; use crate::storage::*; use futures::StreamExt; use http::StatusCode; @@ -33,7 +30,7 @@ use rustfs_config::RUSTFS_REGION; use rustfs_ecstore::bucket::{ lifecycle::bucket_lifecycle_ops::validate_transition_tier, metadata::{ - BUCKET_ACL_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, + BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, }, @@ -158,12 +155,6 @@ impl DefaultBucketUsecase { }; let CreateBucketInput { bucket, - acl, - grant_full_control, - grant_read, - grant_read_acp, - grant_write, - grant_write_acp, object_lock_enabled_for_bucket, .. } = req.input; @@ -190,13 +181,7 @@ impl DefaultBucketUsecase { Err(StorageError::BucketExists(_)) => { // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // non-owner gets 409 BucketAlreadyExists. - let bucket_owner_id = match metadata_sys::get_bucket_acl_config(&bucket).await { - Ok((acl, _)) => parse_acl_json_or_canned_bucket(&acl, &default_owner()).owner.id, - Err(StorageError::ConfigNotFound) => default_owner().id, - Err(e) => return Err(ApiError::from(e).into()), - }; - - let is_owner = requester_id.as_deref().is_some_and(|req_id| req_id == bucket_owner_id); + let is_owner = requester_id.as_deref().is_some_and(|req_id| req_id == default_owner().id); if is_owner { let output = CreateBucketOutput::default(); @@ -214,28 +199,6 @@ impl DefaultBucketUsecase { Err(e) => return Err(ApiError::from(e).into()), } - let owner = default_owner(); - let mut stored_acl = stored_acl_from_grant_headers( - &owner, - grant_read.map(|v| v.to_string()), - grant_write.map(|v| v.to_string()), - grant_read_acp.map(|v| v.to_string()), - grant_write_acp.map(|v| v.to_string()), - grant_full_control.map(|v| v.to_string()), - )?; - - if stored_acl.is_none() - && let Some(canned) = acl - { - stored_acl = Some(stored_acl_from_canned_bucket(canned.as_str(), &owner)); - } - - let stored_acl = stored_acl.unwrap_or_else(|| stored_acl_from_canned_bucket(BucketCannedACL::PRIVATE, &owner)); - let data = serialize_acl(&stored_acl)?; - metadata_sys::update(&bucket, BUCKET_ACL_CONFIG, data) - .await - .map_err(ApiError::from)?; - let output = CreateBucketOutput::default(); let result = Ok(S3Response::new(output)); @@ -250,13 +213,7 @@ impl DefaultBucketUsecase { let PutBucketAclInput { bucket, - acl, access_control_policy, - grant_full_control, - grant_read, - grant_read_acp, - grant_write, - grant_write_acp, .. } = req.input; @@ -269,43 +226,13 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - let owner = default_owner(); - let mut stored_acl = access_control_policy - .as_ref() - .map(|policy| stored_acl_from_policy(policy, &owner)) - .transpose()?; - - if stored_acl.is_none() { - stored_acl = stored_acl_from_grant_headers( - &owner, - grant_read.map(|v| v.to_string()), - grant_write.map(|v| v.to_string()), - grant_read_acp.map(|v| v.to_string()), - grant_write_acp.map(|v| v.to_string()), - grant_full_control.map(|v| v.to_string()), - )?; + if access_control_policy.is_some() { + return Err(s3_error!( + NotImplemented, + "ACL XML grants are not supported; use canned ACL headers or omit ACL" + )); } - if stored_acl.is_none() - && let Some(canned) = acl - { - stored_acl = Some(stored_acl_from_canned_bucket(canned.as_str(), &owner)); - } - - let stored_acl = stored_acl.unwrap_or_else(|| stored_acl_from_canned_bucket(BucketCannedACL::PRIVATE, &owner)); - - if let Ok((config, _)) = metadata_sys::get_public_access_block_config(&bucket).await - && config.block_public_acls.unwrap_or(false) - && stored_acl.grants.iter().any(is_public_grant) - { - return Err(s3_error!(AccessDenied, "Access Denied")); - } - - let data = serialize_acl(&stored_acl)?; - metadata_sys::update(&bucket, BUCKET_ACL_CONFIG, data) - .await - .map_err(ApiError::from)?; - Ok(S3Response::new(PutBucketAclOutput::default())) } @@ -392,25 +319,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - let owner = default_owner(); - let stored_acl = match metadata_sys::get_bucket_acl_config(&bucket).await { - Ok((acl, _)) => parse_acl_json_or_canned_bucket(&acl, &owner), - Err(err) => { - if err != StorageError::ConfigNotFound { - return Err(ApiError::from(err).into()); - } - stored_acl_from_canned_bucket(BucketCannedACL::PRIVATE, &owner) - } - }; - - let mut sorted_grants = stored_acl.grants.clone(); - sorted_grants.sort_by_key(|grant| grant.grantee.grantee_type != "Group"); - let grants = sorted_grants.iter().map(stored_grant_to_dto).collect(); - - Ok(S3Response::new(GetBucketAclOutput { - grants: Some(grants), - owner: Some(stored_owner_to_dto(&stored_acl.owner)), - })) + Ok(S3Response::new(acl::build_get_bucket_acl_output())) } #[instrument(level = "debug", skip(self, req))] @@ -922,21 +831,7 @@ impl DefaultBucketUsecase { Err(_) => false, }; - let owner = default_owner(); - let acl_public = match metadata_sys::get_bucket_acl_config(&bucket).await { - Ok((acl, _)) => { - let stored_acl = parse_acl_json_or_canned_bucket(&acl, &owner); - stored_acl - .grants - .iter() - .any(|grant| is_public_grant(grant) && !ignore_public_acls) - } - Err(_) => false, - }; - - if acl_public { - is_public = true; - } + let _ = ignore_public_acls; let policy_public = match metadata_sys::get_bucket_policy(&bucket).await { Ok((cfg, _)) => cfg.statements.iter().any(|statement| { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 68fd8a6d..ec00a58f 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -17,7 +17,7 @@ use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::config::workload_profiles::RustFSBufferConfig; use crate::error::ApiError; -use crate::storage::access::{ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut}; +use crate::storage::access::{authorize_request, has_bypass_governance_header, req_info_mut}; use crate::storage::concurrency::{ CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, }; @@ -29,7 +29,7 @@ use crate::storage::options::{ filter_object_metadata, get_content_sha256_with_query, get_opts, put_opts, }; use crate::storage::s3_api::multipart::parse_list_parts_params; -use crate::storage::s3_api::{restore, select}; +use crate::storage::s3_api::{acl, restore, select}; use crate::storage::*; use bytes::Bytes; use datafusion::arrow::{ @@ -287,11 +287,6 @@ impl DefaultObjectUsecase { body, bucket, key, - acl, - grant_full_control, - grant_read, - grant_read_acp, - grant_write_acp, content_length, content_type, tagging, @@ -318,14 +313,6 @@ impl DefaultObjectUsecase { // Validate object key validate_object_key(&key, "PUT")?; - if let Some(acl) = &acl - && let Ok((config, _)) = metadata_sys::get_public_access_block_config(&bucket).await - && config.block_public_acls.unwrap_or(false) - && is_public_canned_acl(acl.as_str()) - { - return Err(s3_error!(AccessDenied, "Access Denied")); - } - if let Some(size) = content_length { self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) .await?; @@ -407,33 +394,6 @@ impl DefaultObjectUsecase { )?; let mut metadata = metadata.unwrap_or_default(); - let owner = req - .extensions - .get::() - .and_then(|info| info.cred.as_ref()) - .map(|cred| owner_from_access_key(&cred.access_key)) - .unwrap_or_else(default_owner); - let bucket_owner = default_owner(); - let mut stored_acl = stored_acl_from_grant_headers( - &owner, - grant_read.map(|v| v.to_string()), - None, - grant_read_acp.map(|v| v.to_string()), - grant_write_acp.map(|v| v.to_string()), - grant_full_control.map(|v| v.to_string()), - )?; - - if stored_acl.is_none() - && let Some(canned) = acl.as_ref() - { - stored_acl = Some(stored_acl_from_canned_object(canned.as_str(), &bucket_owner, &owner)); - } - - let stored_acl = - stored_acl.unwrap_or_else(|| stored_acl_from_canned_object(ObjectCannedACL::PRIVATE, &bucket_owner, &owner)); - let acl_data = serialize_acl(&stored_acl)?; - metadata.insert(INTERNAL_ACL_METADATA_KEY.to_string(), String::from_utf8_lossy(&acl_data).to_string()); - if let Some(content_type) = content_type { metadata.insert("content-type".to_string(), content_type.to_string()); } @@ -622,13 +582,7 @@ impl DefaultObjectUsecase { let PutObjectAclInput { bucket, key, - acl, access_control_policy, - grant_full_control, - grant_read, - grant_read_acp, - grant_write, - grant_write_acp, version_id, .. } = req.input; @@ -640,64 +594,15 @@ impl DefaultObjectUsecase { let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) .await .map_err(ApiError::from)?; - let info = store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?; + store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?; - let bucket_owner = default_owner(); - let existing_owner = info - .user_defined - .get(INTERNAL_ACL_METADATA_KEY) - .and_then(|acl| serde_json::from_str::(acl).ok()) - .map(|acl| acl.owner) - .unwrap_or_else(|| bucket_owner.clone()); - - let mut stored_acl = access_control_policy - .as_ref() - .map(|policy| stored_acl_from_policy(policy, &existing_owner)) - .transpose()?; - - if stored_acl.is_none() { - stored_acl = stored_acl_from_grant_headers( - &existing_owner, - grant_read.map(|v| v.to_string()), - grant_write.map(|v| v.to_string()), - grant_read_acp.map(|v| v.to_string()), - grant_write_acp.map(|v| v.to_string()), - grant_full_control.map(|v| v.to_string()), - )?; + if access_control_policy.is_some() { + return Err(s3_error!( + NotImplemented, + "ACL XML grants are not supported; use canned ACL headers or omit ACL" + )); } - if stored_acl.is_none() - && let Some(canned) = acl - { - stored_acl = Some(stored_acl_from_canned_object(canned.as_str(), &bucket_owner, &existing_owner)); - } - - let stored_acl = - stored_acl.unwrap_or_else(|| stored_acl_from_canned_object(ObjectCannedACL::PRIVATE, &bucket_owner, &existing_owner)); - - if let Ok((config, _)) = metadata_sys::get_public_access_block_config(&bucket).await - && config.block_public_acls.unwrap_or(false) - && stored_acl.grants.iter().any(is_public_grant) - { - return Err(s3_error!(AccessDenied, "Access Denied")); - } - - let acl_data = serialize_acl(&stored_acl)?; - let mut eval_metadata = HashMap::new(); - eval_metadata.insert(INTERNAL_ACL_METADATA_KEY.to_string(), String::from_utf8_lossy(&acl_data).to_string()); - - let popts = ObjectOptions { - mod_time: info.mod_time, - version_id: opts.version_id, - eval_metadata: Some(eval_metadata), - ..Default::default() - }; - - store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| { - error!("put_object_metadata failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - Ok(S3Response::new(PutObjectAclOutput::default())) } @@ -1606,31 +1511,9 @@ impl DefaultObjectUsecase { let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) .await .map_err(ApiError::from)?; - let info = store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?; + store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?; - let bucket_owner = default_owner(); - let object_owner = info - .user_defined - .get(INTERNAL_ACL_METADATA_KEY) - .and_then(|acl| serde_json::from_str::(acl).ok()) - .map(|acl| acl.owner) - .unwrap_or_else(default_owner); - - let stored_acl = info - .user_defined - .get(INTERNAL_ACL_METADATA_KEY) - .map(|acl| parse_acl_json_or_canned_object(acl, &bucket_owner, &object_owner)) - .unwrap_or_else(|| stored_acl_from_canned_object(ObjectCannedACL::PRIVATE, &bucket_owner, &object_owner)); - - let mut sorted_grants = stored_acl.grants.clone(); - sorted_grants.sort_by_key(|grant| grant.grantee.grantee_type != "Group"); - let grants = sorted_grants.iter().map(stored_grant_to_dto).collect(); - - Ok(S3Response::new(GetObjectAclOutput { - grants: Some(grants), - owner: Some(stored_owner_to_dto(&stored_acl.owner)), - ..Default::default() - })) + Ok(S3Response::new(acl::build_get_object_acl_output())) } pub async fn execute_get_object_attributes( diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index 338fb91c..f3e7eddc 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -343,6 +343,29 @@ pub struct ConditionalCorsService { cors_origins: Arc>, } +async fn resolve_s3_options_cors_headers(bucket: &str, request_headers: &HeaderMap) -> Option { + apply_cors_headers(bucket, &http::Method::OPTIONS, request_headers).await +} + +fn clear_cors_response_headers(headers: &mut HeaderMap) { + headers.remove(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN); + headers.remove(cors::response::ACCESS_CONTROL_ALLOW_METHODS); + headers.remove(cors::response::ACCESS_CONTROL_ALLOW_HEADERS); + headers.remove(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS); + headers.remove(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS); + headers.remove(cors::response::ACCESS_CONTROL_MAX_AGE); +} + +fn apply_bucket_cors_result(response_headers: &mut HeaderMap, bucket_cors_headers: &HeaderMap) { + // Bucket-level CORS is authoritative for S3 object/bucket paths. + // Clear any previously-populated CORS response headers (e.g. generic/system defaults), + // then apply the evaluated bucket result (which may be intentionally empty). + clear_cors_response_headers(response_headers); + for (key, value) in bucket_cors_headers.iter() { + response_headers.insert(key, value.clone()); + } +} + impl Service> for ConditionalCorsService where S: Service, Response = Response> + Clone + Send + 'static, @@ -364,12 +387,32 @@ where let request_headers = req.headers().clone(); let cors_origins = self.cors_origins.clone(); let is_s3 = ConditionalCorsLayer::is_s3_path(&path); + let is_root = path == "/"; if method == Method::OPTIONS { + let has_acrm = request_headers.contains_key(cors::request::ACCESS_CONTROL_REQUEST_METHOD); + + if is_root { + return Box::pin(async move { + if !has_acrm || !request_headers.contains_key(cors::standard::ORIGIN) { + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(ResBody::default()) + .unwrap()); + } + + let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).unwrap(); + let cors_layer = ConditionalCorsLayer { + cors_origins: (*cors_origins).clone(), + }; + cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); + Ok(response) + }); + } + if is_s3 { let path_trimmed = path.trim_start_matches('/'); let bucket = path_trimmed.split('/').next().unwrap_or("").to_string(); - let has_acrm = request_headers.contains_key(cors::request::ACCESS_CONTROL_REQUEST_METHOD); return Box::pin(async move { if !has_acrm || !request_headers.contains_key(cors::standard::ORIGIN) { @@ -379,20 +422,27 @@ where .unwrap()); } - if !bucket.is_empty() - && let Some(cors_headers) = apply_cors_headers(&bucket, &method, &request_headers).await - { - let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).unwrap(); - for (key, value) in cors_headers.iter() { - response.headers_mut().insert(key, value.clone()); + let cors_layer = ConditionalCorsLayer { + cors_origins: (*cors_origins).clone(), + }; + + if let Some(cors_headers) = resolve_s3_options_cors_headers(&bucket, &request_headers).await { + let cors_allowed = cors_headers.contains_key(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN); + let status = if cors_allowed { StatusCode::OK } else { StatusCode::FORBIDDEN }; + + let mut response = Response::builder().status(status).body(ResBody::default()).unwrap(); + if cors_allowed { + for (key, value) in cors_headers.iter() { + response.headers_mut().insert(key, value.clone()); + } } return Ok(response); } - Ok(Response::builder() - .status(StatusCode::FORBIDDEN) - .body(ResBody::default()) - .unwrap()) + // No bucket-level CORS config: fall back to global/default CORS behavior. + let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).unwrap(); + cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); + Ok(response) }); } @@ -415,19 +465,30 @@ where if request_headers.contains_key(cors::standard::ORIGIN) && !response.headers().contains_key(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN) { + let cors_layer = ConditionalCorsLayer { + cors_origins: (*cors_origins).clone(), + }; + if is_s3 { let bucket = path.trim_start_matches('/').split('/').next().unwrap_or(""); - if !bucket.is_empty() - && let Some(cors_headers) = apply_cors_headers(bucket, &method, &request_headers).await - { - for (key, value) in cors_headers.iter() { - response.headers_mut().insert(key, value.clone()); + if path == "/" { + cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); + } else if !bucket.is_empty() { + match apply_cors_headers(bucket, &method, &request_headers).await { + Some(bucket_cors_headers) => { + // Bucket-level CORS is authoritative when configured, even if it + // intentionally resolves to an empty header set (no rule match). + apply_bucket_cors_result(response.headers_mut(), &bucket_cors_headers); + } + None => { + // No bucket-level CORS config: fall back to global/default policy. + cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); + } } + } else { + cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); } } else { - let cors_layer = ConditionalCorsLayer { - cors_origins: (*cors_origins).clone(), - }; cors_layer.apply_cors_headers(&request_headers, response.headers_mut()); } } @@ -532,4 +593,73 @@ mod tests { "https://allowed.com" ); } + + #[tokio::test] + async fn test_resolve_s3_options_cors_headers_no_headers_without_match() { + let mut req_headers = HeaderMap::new(); + req_headers.insert("origin", "https://example.com".parse().unwrap()); + req_headers.insert("access-control-request-method", "GET".parse().unwrap()); + + let headers = resolve_s3_options_cors_headers("bbb", &req_headers).await; + assert!(headers.is_none()); + } + + #[test] + fn test_apply_bucket_cors_result_clears_existing_cors_headers_with_empty_result() { + let mut response_headers = HeaderMap::new(); + response_headers.insert( + cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("https://foo.example"), + ); + response_headers.insert( + cors::response::ACCESS_CONTROL_ALLOW_METHODS, + HeaderValue::from_static("GET, POST, PUT, DELETE, OPTIONS, HEAD"), + ); + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_HEADERS, HeaderValue::from_static("*")); + response_headers.insert(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS, HeaderValue::from_static("etag")); + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS, HeaderValue::from_static("true")); + response_headers.insert(cors::response::ACCESS_CONTROL_MAX_AGE, HeaderValue::from_static("3600")); + + let bucket_cors_headers = HeaderMap::new(); + apply_bucket_cors_result(&mut response_headers, &bucket_cors_headers); + + assert!(response_headers.get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN).is_none()); + assert!(response_headers.get(cors::response::ACCESS_CONTROL_ALLOW_METHODS).is_none()); + assert!(response_headers.get(cors::response::ACCESS_CONTROL_ALLOW_HEADERS).is_none()); + assert!(response_headers.get(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS).is_none()); + assert!( + response_headers + .get(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS) + .is_none() + ); + assert!(response_headers.get(cors::response::ACCESS_CONTROL_MAX_AGE).is_none()); + } + + #[test] + fn test_apply_bucket_cors_result_replaces_existing_cors_headers() { + let mut response_headers = HeaderMap::new(); + response_headers.insert( + cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("https://foo.example"), + ); + response_headers.insert( + cors::response::ACCESS_CONTROL_ALLOW_METHODS, + HeaderValue::from_static("GET, POST, PUT, DELETE, OPTIONS, HEAD"), + ); + + let mut bucket_cors_headers = HeaderMap::new(); + bucket_cors_headers.insert( + cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("https://allowed.example"), + ); + bucket_cors_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_METHODS, HeaderValue::from_static("GET")); + + apply_bucket_cors_result(&mut response_headers, &bucket_cors_headers); + + assert_eq!( + response_headers.get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(), + "https://allowed.example" + ); + assert_eq!(response_headers.get(cors::response::ACCESS_CONTROL_ALLOW_METHODS).unwrap(), "GET"); + } } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index fdae2ec1..e66360cc 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -13,14 +13,7 @@ // limitations under the License. use super::ecfs::FS; -use super::ecfs::{ - ACL_GROUP_ALL_USERS, ACL_GROUP_AUTHENTICATED_USERS, INTERNAL_ACL_METADATA_KEY, StoredAcl, default_owner, - parse_acl_json_or_canned_bucket, parse_acl_json_or_canned_object, stored_acl_from_canned_bucket, - stored_acl_from_canned_object, -}; -use super::options::get_opts; use crate::auth::{check_key_valid, get_condition_values_with_query, get_session_token}; -use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; use metrics::counter; @@ -28,7 +21,7 @@ use rustfs_ecstore::bucket::metadata_sys; use rustfs_ecstore::bucket::policy_sys::PolicySys; use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found}; use rustfs_ecstore::new_object_layer_fn; -use rustfs_ecstore::store_api::{BucketOperations, ObjectOperations}; +use rustfs_ecstore::store_api::BucketOperations; use rustfs_iam::error::Error as IamError; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_policy::policy::{Args, BucketPolicyArgs}; @@ -48,165 +41,6 @@ pub(crate) struct ReqInfo { pub region: Option, } -#[derive(Clone, Copy)] -enum AclTarget { - Bucket, - Object, -} - -#[derive(Clone, Copy)] -enum AclPermission { - Read, - Write, - ReadAcp, - WriteAcp, -} - -fn acl_permission_for_action(action: &Action) -> Option<(AclTarget, AclPermission)> { - match action { - Action::S3Action(S3Action::ListBucketAction) => Some((AclTarget::Bucket, AclPermission::Read)), - Action::S3Action(S3Action::PutObjectAction) => Some((AclTarget::Bucket, AclPermission::Write)), - Action::S3Action(S3Action::GetBucketAclAction) => Some((AclTarget::Bucket, AclPermission::ReadAcp)), - Action::S3Action(S3Action::PutBucketAclAction) => Some((AclTarget::Bucket, AclPermission::WriteAcp)), - Action::S3Action(S3Action::GetObjectAction) => Some((AclTarget::Object, AclPermission::Read)), - Action::S3Action(S3Action::GetObjectAclAction) => Some((AclTarget::Object, AclPermission::ReadAcp)), - Action::S3Action(S3Action::PutObjectAclAction) => Some((AclTarget::Object, AclPermission::WriteAcp)), - _ => None, - } -} - -fn permission_matches(grant_perm: &str, required: AclPermission) -> bool { - if grant_perm == Permission::FULL_CONTROL { - return true; - } - - match required { - AclPermission::Read => grant_perm == Permission::READ, - AclPermission::Write => grant_perm == Permission::WRITE, - AclPermission::ReadAcp => grant_perm == Permission::READ_ACP, - AclPermission::WriteAcp => grant_perm == Permission::WRITE_ACP, - } -} - -fn acl_allows( - acl: &StoredAcl, - user_id: Option<&str>, - is_authenticated: bool, - required: AclPermission, - ignore_public_acls: bool, -) -> bool { - for grant in &acl.grants { - if !permission_matches(grant.permission.as_str(), required) { - continue; - } - - match grant.grantee.grantee_type.as_str() { - "CanonicalUser" => { - if user_id.is_some_and(|id| grant.grantee.id.as_deref() == Some(id)) { - return true; - } - } - "Group" => { - if ignore_public_acls { - continue; - } - if let Some(uri) = grant.grantee.uri.as_deref() { - if uri == ACL_GROUP_ALL_USERS { - return true; - } - if uri == ACL_GROUP_AUTHENTICATED_USERS && is_authenticated { - return true; - } - } - } - _ => {} - } - } - - false -} - -async fn load_bucket_acl(bucket: &str) -> S3Result { - let owner = default_owner(); - match metadata_sys::get_bucket_acl_config(bucket).await { - Ok((acl, _)) => Ok(parse_acl_json_or_canned_bucket(&acl, &owner)), - Err(err) => { - if err == StorageError::ConfigNotFound { - Ok(stored_acl_from_canned_bucket(BucketCannedACL::PRIVATE, &owner)) - } else { - Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())) - } - } - } -} - -async fn load_object_acl(bucket: &str, object: &str, version_id: Option<&str>, headers: &http::HeaderMap) -> S3Result { - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let opts = get_opts(bucket, object, version_id.map(|v| v.to_string()), None, headers) - .await - .map_err(ApiError::from)?; - let info = store.get_object_info(bucket, object, &opts).await.map_err(ApiError::from)?; - - let bucket_owner = default_owner(); - let object_owner = info - .user_defined - .get(INTERNAL_ACL_METADATA_KEY) - .and_then(|acl| serde_json::from_str::(acl).ok()) - .map(|acl| acl.owner) - .unwrap_or_else(default_owner); - - Ok(info - .user_defined - .get(INTERNAL_ACL_METADATA_KEY) - .map(|acl| parse_acl_json_or_canned_object(acl, &bucket_owner, &object_owner)) - .unwrap_or_else(|| stored_acl_from_canned_object(ObjectCannedACL::PRIVATE, &bucket_owner, &object_owner))) -} - -async fn check_acl_access(req: &S3Request, req_info: &ReqInfo, action: &Action, policy_allowed: bool) -> S3Result { - let Some((target, permission)) = acl_permission_for_action(action) else { - return Ok(true); - }; - - // For object-level operations, do not bypass on account-level is_owner. - // The bucket/account owner must have explicit ACL grant to access objects owned by others. - let bypass_owner = match target { - AclTarget::Bucket => req_info.is_owner, - AclTarget::Object => false, - }; - if bypass_owner || policy_allowed { - return Ok(true); - } - - let bucket = req_info.bucket.as_deref().unwrap_or(""); - if bucket.is_empty() { - return Ok(true); - } - - let ignore_public_acls = match metadata_sys::get_public_access_block_config(bucket).await { - Ok((config, _)) => config.ignore_public_acls.unwrap_or(false), - Err(_) => false, - }; - - let user_id = req_info.cred.as_ref().map(|cred| cred.access_key.as_str()); - let is_authenticated = user_id.is_some(); - - let acl = match target { - AclTarget::Bucket => load_bucket_acl(bucket).await?, - AclTarget::Object => { - let object = req_info.object.as_deref().unwrap_or(""); - if object.is_empty() { - return Ok(true); - } - load_object_acl(bucket, object, req_info.version_id.as_deref(), &req.headers).await? - } - }; - - Ok(acl_allows(&acl, user_id, is_authenticated, permission, ignore_public_acls)) -} - pub(crate) fn req_info_ref(req: &S3Request) -> S3Result<&ReqInfo> { req.extensions .get::() @@ -299,7 +133,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R && !PolicySys::is_allowed(&BucketPolicyArgs { bucket: bucket_name, action, - // Run this early check in deny-only mode so ACL/IAM fallbacks can still grant access. + // Run this early check in deny-only mode so IAM fallback can still grant access. is_owner: true, account: &cred.access_key, groups: &cred.groups, @@ -355,26 +189,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R .await; if iam_allowed { - let policy_allowed = if !bucket_name.is_empty() { - PolicySys::is_allowed(&BucketPolicyArgs { - bucket: bucket_name, - action, - is_owner: false, - account: &cred.access_key, - groups: &cred.groups, - conditions: &conditions, - object: req_info.object.as_deref().unwrap_or(""), - }) - .await - } else { - false - }; - - if check_acl_access(req, req_info, &action, policy_allowed).await? { - return Ok(()); - } - - return Err(s3_error!(AccessDenied, "Access Denied")); + return Ok(()); } let policy_allowed_fallback = PolicySys::is_allowed(&BucketPolicyArgs { @@ -389,13 +204,6 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R .await; if policy_allowed_fallback { - // For object-level operations, bucket owner (is_owner) must still pass ACL check. - // Policy may have allowed due to is_owner, but object owner is authoritative. - if let Some((AclTarget::Object, _)) = acl_permission_for_action(&action) - && !check_acl_access(req, req_info, &action, false).await? - { - return Err(s3_error!(AccessDenied, "Access Denied")); - } return Ok(()); } @@ -455,7 +263,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R && !PolicySys::is_allowed(&BucketPolicyArgs { bucket: bucket_name, action, - // Run this early check in deny-only mode so ACL checks are not bypassed. + // Run this early check in deny-only mode so later policy checks are not bypassed. is_owner: true, account: "", groups: &None, @@ -509,10 +317,6 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R { return Ok(()); } - - if acl_permission_for_action(&action).is_some() && check_acl_access(req, req_info, &action, false).await? { - return Ok(()); - } } } @@ -532,18 +336,10 @@ fn get_bucket_policy_authorize_action() -> Action { Action::S3Action(S3Action::GetBucketPolicyAction) } -fn get_object_acl_authorize_action() -> Action { - Action::S3Action(S3Action::GetObjectAclAction) -} - fn put_bucket_policy_authorize_action() -> Action { Action::S3Action(S3Action::PutBucketPolicyAction) } -fn put_object_acl_authorize_action() -> Action { - Action::S3Action(S3Action::PutObjectAclAction) -} - #[async_trait::async_trait] impl S3Access for FS { // /// Checks whether the current request has accesses to the resources. @@ -1104,12 +900,7 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); - let tag_conds = self - .fetch_tag_conditions(&req.input.bucket, &req.input.key, req.input.version_id.as_deref(), "get_object_acl") - .await?; - req.extensions.insert(tag_conds); - - authorize_request(req, get_object_acl_authorize_action()).await + authorize_request(req, Action::S3Action(S3Action::GetObjectAclAction)).await } /// Checks whether the GetObjectAttributes request has accesses to the resources. @@ -1526,12 +1317,7 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); - let tag_conds = self - .fetch_tag_conditions(&req.input.bucket, &req.input.key, req.input.version_id.as_deref(), "put_object_acl") - .await?; - req.extensions.insert(tag_conds); - - authorize_request(req, put_object_acl_authorize_action()).await + authorize_request(req, Action::S3Action(S3Action::PutObjectAclAction)).await } /// Checks whether the PutObjectLegalHold request has accesses to the resources. @@ -1659,21 +1445,11 @@ mod tests { assert_eq!(get_bucket_policy_authorize_action(), Action::S3Action(S3Action::GetBucketPolicyAction)); } - #[test] - fn get_object_acl_uses_get_object_acl_action() { - assert_eq!(get_object_acl_authorize_action(), Action::S3Action(S3Action::GetObjectAclAction)); - } - #[test] fn put_bucket_policy_uses_put_bucket_policy_action() { assert_eq!(put_bucket_policy_authorize_action(), Action::S3Action(S3Action::PutBucketPolicyAction)); } - #[test] - fn put_object_acl_uses_put_object_acl_action() { - assert_eq!(put_object_acl_authorize_action(), Action::S3Action(S3Action::PutObjectAclAction)); - } - /// Object tag conditions must use keys like ExistingObjectTag/ so that /// bucket policy conditions (e.g. s3:ExistingObjectTag/security) are evaluated correctly. #[test] diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index bbc22bd3..e8a33eac 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -22,7 +22,6 @@ use rustfs_ecstore::{ store_api::{BucketOperations, BucketOptions, ObjectOperations, ObjectOptions}, }; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; -use serde::{Deserialize, Serialize}; use std::{fmt::Debug, sync::LazyLock}; use tokio::io::{AsyncRead, AsyncSeek}; use tracing::{debug, error, instrument, warn}; @@ -30,511 +29,23 @@ use uuid::Uuid; const DEFAULT_OWNER_ID: &str = "rustfsadmin"; const DEFAULT_OWNER_DISPLAY_NAME: &str = "RustFS Tester"; -const DEFAULT_OWNER_EMAIL: &str = "tester@rustfs.local"; -const DEFAULT_ALT_ID: &str = "rustfsalt"; -const DEFAULT_ALT_DISPLAY_NAME: &str = "RustFS Alt Tester"; -const DEFAULT_ALT_EMAIL: &str = "alt@rustfs.local"; - -pub(crate) const INTERNAL_ACL_METADATA_KEY: &str = "x-rustfs-internal-acl"; pub(crate) static RUSTFS_OWNER: LazyLock = LazyLock::new(|| Owner { display_name: Some(DEFAULT_OWNER_DISPLAY_NAME.to_owned()), id: Some(DEFAULT_OWNER_ID.to_owned()), }); -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] pub(crate) struct StoredOwner { pub(crate) id: String, - pub(crate) display_name: String, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub(crate) struct StoredGrantee { - pub(crate) grantee_type: String, - pub(crate) id: Option, - pub(crate) display_name: Option, - pub(crate) uri: Option, - pub(crate) email_address: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct StoredGrant { - pub(crate) grantee: StoredGrantee, - pub(crate) permission: String, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub(crate) struct StoredAcl { - pub(crate) owner: StoredOwner, - pub(crate) grants: Vec, } pub(crate) fn default_owner() -> StoredOwner { StoredOwner { id: DEFAULT_OWNER_ID.to_string(), - display_name: DEFAULT_OWNER_DISPLAY_NAME.to_string(), } } -pub(crate) fn owner_from_access_key(access_key: &str) -> StoredOwner { - if access_key == DEFAULT_OWNER_ID { - return default_owner(); - } - if access_key == DEFAULT_ALT_ID { - return StoredOwner { - id: DEFAULT_ALT_ID.to_string(), - display_name: DEFAULT_ALT_DISPLAY_NAME.to_string(), - }; - } - - StoredOwner { - id: access_key.to_string(), - display_name: access_key.to_string(), - } -} - -fn user_id_from_email(email: &str) -> Option { - if email.eq_ignore_ascii_case(DEFAULT_OWNER_EMAIL) { - return Some(default_owner()); - } - if email.eq_ignore_ascii_case(DEFAULT_ALT_EMAIL) { - return Some(StoredOwner { - id: DEFAULT_ALT_ID.to_string(), - display_name: DEFAULT_ALT_DISPLAY_NAME.to_string(), - }); - } - - None -} - -fn display_name_for_user_id(user_id: &str) -> Option { - if user_id == DEFAULT_OWNER_ID { - return Some(DEFAULT_OWNER_DISPLAY_NAME.to_string()); - } - if user_id == DEFAULT_ALT_ID { - return Some(DEFAULT_ALT_DISPLAY_NAME.to_string()); - } - - None -} - -fn is_known_user_id(user_id: &str) -> bool { - user_id == DEFAULT_OWNER_ID || user_id == DEFAULT_ALT_ID -} - -pub(crate) fn stored_owner_to_dto(owner: &StoredOwner) -> Owner { - Owner { - id: Some(owner.id.clone()), - display_name: Some(owner.display_name.clone()), - } -} - -pub(crate) fn stored_grant_to_dto(grant: &StoredGrant) -> Grant { - let grantee = match grant.grantee.grantee_type.as_str() { - "Group" => Grantee { - type_: Type::from_static(Type::GROUP), - display_name: None, - email_address: None, - id: None, - uri: grant.grantee.uri.clone(), - }, - _ => Grantee { - type_: Type::from_static(Type::CANONICAL_USER), - display_name: grant.grantee.display_name.clone(), - email_address: None, - id: grant.grantee.id.clone(), - uri: None, - }, - }; - - Grant { - grantee: Some(grantee), - permission: Some(Permission::from(grant.permission.clone())), - } -} - -pub(crate) fn is_public_grant(grant: &StoredGrant) -> bool { - matches!(grant.grantee.grantee_type.as_str(), "Group") - && grant - .grantee - .uri - .as_deref() - .is_some_and(|uri| uri == ACL_GROUP_ALL_USERS || uri == ACL_GROUP_AUTHENTICATED_USERS) -} - -fn grants_for_canned_bucket_acl(acl: &str, owner: &StoredOwner) -> Vec { - let mut grants = Vec::new(); - - match acl { - BucketCannedACL::PUBLIC_READ => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - } - BucketCannedACL::PUBLIC_READ_WRITE => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::WRITE.to_string(), - }); - } - BucketCannedACL::AUTHENTICATED_READ => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_AUTHENTICATED_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - } - _ => {} - } - - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(owner.id.clone()), - display_name: Some(owner.display_name.clone()), - uri: None, - email_address: None, - }, - permission: Permission::FULL_CONTROL.to_string(), - }); - - grants -} - -fn grants_for_canned_object_acl(acl: &str, bucket_owner: &StoredOwner, object_owner: &StoredOwner) -> Vec { - let mut grants = Vec::new(); - - match acl { - ObjectCannedACL::PUBLIC_READ => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - } - ObjectCannedACL::PUBLIC_READ_WRITE => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_ALL_USERS.to_string()), - email_address: None, - }, - permission: Permission::WRITE.to_string(), - }); - } - ObjectCannedACL::AUTHENTICATED_READ => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(ACL_GROUP_AUTHENTICATED_USERS.to_string()), - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - } - ObjectCannedACL::BUCKET_OWNER_READ => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(bucket_owner.id.clone()), - display_name: Some(bucket_owner.display_name.clone()), - uri: None, - email_address: None, - }, - permission: Permission::READ.to_string(), - }); - } - ObjectCannedACL::BUCKET_OWNER_FULL_CONTROL => { - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(bucket_owner.id.clone()), - display_name: Some(bucket_owner.display_name.clone()), - uri: None, - email_address: None, - }, - permission: Permission::FULL_CONTROL.to_string(), - }); - } - _ => {} - } - - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(object_owner.id.clone()), - display_name: Some(object_owner.display_name.clone()), - uri: None, - email_address: None, - }, - permission: Permission::FULL_CONTROL.to_string(), - }); - - grants -} - -pub(crate) fn stored_acl_from_canned_bucket(acl: &str, owner: &StoredOwner) -> StoredAcl { - StoredAcl { - owner: owner.clone(), - grants: grants_for_canned_bucket_acl(acl, owner), - } -} - -pub(crate) fn stored_acl_from_canned_object(acl: &str, bucket_owner: &StoredOwner, object_owner: &StoredOwner) -> StoredAcl { - StoredAcl { - owner: object_owner.clone(), - grants: grants_for_canned_object_acl(acl, bucket_owner, object_owner), - } -} - -pub(crate) fn parse_acl_json_or_canned_bucket(data: &str, owner: &StoredOwner) -> StoredAcl { - match serde_json::from_str::(data) { - Ok(acl) => acl, - Err(_) => stored_acl_from_canned_bucket(data, owner), - } -} - -pub(crate) fn parse_acl_json_or_canned_object(data: &str, bucket_owner: &StoredOwner, object_owner: &StoredOwner) -> StoredAcl { - match serde_json::from_str::(data) { - Ok(acl) => acl, - Err(_) => stored_acl_from_canned_object(data, bucket_owner, object_owner), - } -} - -pub(crate) fn serialize_acl(acl: &StoredAcl) -> std::result::Result, S3Error> { - serde_json::to_vec(acl).map_err(|e| s3_error!(InternalError, "serialize acl failed {e}")) -} - -fn grantee_from_grant(grantee: &Grantee) -> Result { - match grantee.type_.as_str() { - Type::GROUP => { - let uri = grantee - .uri - .clone() - .filter(|uri| uri == ACL_GROUP_ALL_USERS || uri == ACL_GROUP_AUTHENTICATED_USERS) - .ok_or_else(|| s3_error!(InvalidArgument))?; - - Ok(StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(uri), - email_address: None, - }) - } - Type::CANONICAL_USER => { - let id = grantee.id.clone().ok_or_else(|| s3_error!(InvalidArgument))?; - if !is_known_user_id(&id) { - return Err(s3_error!(InvalidArgument)); - } - let display_name = display_name_for_user_id(&id).or_else(|| grantee.display_name.clone()); - - Ok(StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(id), - display_name, - uri: None, - email_address: None, - }) - } - Type::AMAZON_CUSTOMER_BY_EMAIL => { - let email = grantee.email_address.clone().ok_or_else(|| s3_error!(InvalidArgument))?; - let owner = user_id_from_email(&email).ok_or_else(|| s3_error!(UnresolvableGrantByEmailAddress))?; - - Ok(StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(owner.id), - display_name: Some(owner.display_name), - uri: None, - email_address: None, - }) - } - _ => Err(s3_error!(InvalidArgument)), - } -} - -pub(crate) fn stored_acl_from_policy(policy: &AccessControlPolicy, owner_fallback: &StoredOwner) -> Result { - let owner = policy - .owner - .as_ref() - .and_then(|owner| { - let id = owner.id.clone()?; - let display_name = owner.display_name.clone()?; - Some(StoredOwner { id, display_name }) - }) - .unwrap_or_else(|| owner_fallback.clone()); - - let grants = policy - .grants - .clone() - .unwrap_or_default() - .into_iter() - .map(|grant| { - let permission = grant - .permission - .clone() - .ok_or_else(|| s3_error!(InvalidArgument))? - .as_str() - .to_string(); - let grantee = grant - .grantee - .as_ref() - .ok_or_else(|| s3_error!(InvalidArgument)) - .and_then(grantee_from_grant)?; - Ok(StoredGrant { grantee, permission }) - }) - .collect::, S3Error>>()?; - - Ok(StoredAcl { owner, grants }) -} - -fn parse_grant_header(value: &str) -> Result, S3Error> { - let mut grantees = Vec::new(); - for entry in value.split(',') { - let entry = entry.trim(); - if entry.is_empty() { - continue; - } - let mut parts = entry.splitn(2, '='); - let key = parts.next().unwrap_or_default().trim(); - let val = parts.next().unwrap_or_default().trim(); - match key { - "id" => { - if !is_known_user_id(val) { - return Err(s3_error!(InvalidArgument)); - } - grantees.push(StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(val.to_string()), - display_name: display_name_for_user_id(val), - uri: None, - email_address: None, - }); - } - "uri" => { - if val != ACL_GROUP_ALL_USERS && val != ACL_GROUP_AUTHENTICATED_USERS { - return Err(s3_error!(InvalidArgument)); - } - grantees.push(StoredGrantee { - grantee_type: "Group".to_string(), - id: None, - display_name: None, - uri: Some(val.to_string()), - email_address: None, - }); - } - "emailAddress" => { - let owner = user_id_from_email(val).ok_or_else(|| s3_error!(UnresolvableGrantByEmailAddress))?; - grantees.push(StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(owner.id), - display_name: Some(owner.display_name), - uri: None, - email_address: None, - }); - } - _ => return Err(s3_error!(InvalidArgument)), - } - } - Ok(grantees) -} - -pub(crate) fn stored_acl_from_grant_headers( - owner: &StoredOwner, - grant_read: Option, - grant_write: Option, - grant_read_acp: Option, - grant_write_acp: Option, - grant_full_control: Option, -) -> Result, S3Error> { - let mut grants = Vec::new(); - - let mut push_grants = |value: Option, permission: &str| -> Result<(), S3Error> { - if let Some(value) = value { - for grantee in parse_grant_header(&value)? { - grants.push(StoredGrant { - grantee, - permission: permission.to_string(), - }); - } - } - Ok(()) - }; - - push_grants(grant_read, Permission::READ)?; - push_grants(grant_write, Permission::WRITE)?; - push_grants(grant_read_acp, Permission::READ_ACP)?; - push_grants(grant_write_acp, Permission::WRITE_ACP)?; - push_grants(grant_full_control, Permission::FULL_CONTROL)?; - - if grants.is_empty() { - return Ok(None); - } - - grants.push(StoredGrant { - grantee: StoredGrantee { - grantee_type: "CanonicalUser".to_string(), - id: Some(owner.id.clone()), - display_name: Some(owner.display_name.clone()), - uri: None, - email_address: None, - }, - permission: Permission::FULL_CONTROL.to_string(), - }); - - Ok(Some(StoredAcl { - owner: owner.clone(), - grants, - })) -} - #[derive(Debug, Clone)] pub struct FS { // pub store: ECStore, @@ -659,13 +170,6 @@ impl FS { } } -pub(crate) const ACL_GROUP_ALL_USERS: &str = "http://acs.amazonaws.com/groups/global/AllUsers"; -pub(crate) const ACL_GROUP_AUTHENTICATED_USERS: &str = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"; - -pub(crate) fn is_public_canned_acl(acl: &str) -> bool { - matches!(acl, "public-read" | "public-read-write" | "authenticated-read") -} - pub(crate) fn parse_object_version_id(version_id: Option) -> S3Result> { if let Some(vid) = version_id { let uuid = Uuid::parse_str(&vid).map_err(|e| { diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index e261cecc..91bbcf10 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -641,13 +641,23 @@ pub(crate) fn needs_cors_processing(headers: &HeaderMap) -> bool { /// 2. Retrieves the bucket's CORS configuration /// 3. Matches the origin against CORS rules /// 4. Validates AllowedHeaders if request headers are present -/// 5. Returns headers to add to the response if a match is found +/// 5. Returns one of: +/// - `None`: bucket has no CORS config (or request has no valid `Origin`) +/// - `Some(empty headers)`: bucket CORS exists but request is denied / no rule matched +/// - `Some(non-empty headers)`: bucket CORS exists and request matched /// /// Note: This function should only be called if `needs_cors_processing()` returns true /// to avoid unnecessary overhead for non-CORS requests. pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, headers: &HeaderMap) -> Option { use http::HeaderValue; + fn is_credentialed_request(headers: &HeaderMap) -> bool { + headers.contains_key(http::header::AUTHORIZATION) + || headers.contains_key(http::header::COOKIE) + || headers.contains_key("x-amz-security-token") + || headers.contains_key("x-amz-content-sha256") + } + // Get Origin header from request let origin = headers.get(cors::standard::ORIGIN)?.to_str().ok()?; @@ -659,14 +669,14 @@ pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, head // Early return if no CORS rules configured if cors_config.cors_rules.is_empty() { - return None; + return Some(HeaderMap::new()); } // Check if method is supported and get its string representation const SUPPORTED_METHODS: &[&str] = &["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"]; let method_str = method.as_str(); if !SUPPORTED_METHODS.contains(&method_str) { - return None; + return Some(HeaderMap::new()); } // Use Access-Control-Request-Method if present (for preflight and non-preflight requests), @@ -740,19 +750,37 @@ pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, head let mut response_headers = HeaderMap::new(); // Access-Control-Allow-Origin - // If origin is "*", use "*", otherwise echo back the origin + // Credentials mode + wildcard allow list requires echoing the request origin. + // Browsers reject `Access-Control-Allow-Origin: *` with `credentials: include`. let has_wildcard_origin = rule.allowed_origins.iter().any(|o| o == "*"); + let credentialed_request = is_credentialed_request(headers); + let mut origin_reflected = false; + if has_wildcard_origin { - response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, HeaderValue::from_static("*")); + if credentialed_request { + if let Ok(origin_value) = HeaderValue::from_str(origin) { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, origin_value); + origin_reflected = true; + } + } else { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, HeaderValue::from_static("*")); + } } else if let Ok(origin_value) = HeaderValue::from_str(origin) { response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, origin_value); + origin_reflected = true; } - // Vary: Origin (required for caching, except when using wildcard) - if !has_wildcard_origin { + // Vary: Origin whenever origin is reflected (non-"*" allow-origin). + // This prevents proxy/browser caches from reusing CORS headers across different origins. + if origin_reflected { response_headers.insert(cors::standard::VARY, HeaderValue::from_static("Origin")); } + // Credentials mode requires explicit allow-credentials. + if credentialed_request { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS, HeaderValue::from_static("true")); + } + // Access-Control-Allow-Methods (required for preflight) if is_preflight || !rule.allowed_methods.is_empty() { let methods_str = rule.allowed_methods.iter().map(|m| m.as_str()).collect::>().join(", "); @@ -788,7 +816,7 @@ pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, head return Some(response_headers); } - None // No matching rule found + Some(HeaderMap::new()) // No matching rule found } /// Check if an origin matches a pattern (supports wildcards like https://*.example.com) pub(crate) fn matches_origin_pattern(pattern: &str, origin: &str) -> bool { diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 4314e949..9b2c1877 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -15,6 +15,7 @@ #[cfg(test)] mod tests { use crate::config::workload_profiles::WorkloadProfile; + use crate::server::cors; use crate::storage::ecfs::FS; use crate::storage::ecfs::RUSTFS_OWNER; use crate::storage::{ @@ -25,13 +26,14 @@ mod tests { }; use http::{HeaderMap, HeaderValue, StatusCode}; use rustfs_config::MI_B; + use rustfs_ecstore::bucket::{metadata::BucketMetadata, metadata_sys}; use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use rustfs_ecstore::store_api::ObjectInfo; use rustfs_utils::http::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, RESERVED_METADATA_PREFIX_LOWER}; use rustfs_zip::CompressionFormat; use s3s::dto::{ - Delimiter, LambdaFunctionConfiguration, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, - ObjectLockRetentionMode, QueueConfiguration, TopicConfiguration, + CORSConfiguration, CORSRule, Delimiter, LambdaFunctionConfiguration, ObjectLockLegalHold, ObjectLockLegalHoldStatus, + ObjectLockRetention, ObjectLockRetentionMode, QueueConfiguration, TopicConfiguration, }; use s3s::{S3Error, S3ErrorCode, s3_error}; use time::OffsetDateTime; @@ -1126,6 +1128,84 @@ mod tests { // This is expected behavior - we just verify it doesn't panic } + #[tokio::test] + async fn test_apply_cors_headers_unmatched_origin_with_cors_config() { + if metadata_sys::get_global_bucket_metadata_sys().is_none() { + eprintln!("Skipping test: GLOBAL_BucketMetadataSys not initialized"); + return; + } + + let bucket = "test-bucket-no-match-cors"; + let mut bm = BucketMetadata::new(bucket); + bm.cors_config = Some(CORSConfiguration { + cors_rules: vec![CORSRule { + allowed_headers: Some(vec!["*".to_string()]), + allowed_methods: vec!["GET".to_string()], + allowed_origins: vec!["https://allowed.example.com".to_string()], + expose_headers: None, + id: Some("non-match-origin".to_string()), + max_age_seconds: None, + }], + }); + metadata_sys::set_bucket_metadata(bucket.to_string(), bm).await.unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(cors::standard::ORIGIN, "https://disallowed.example.com".parse().unwrap()); + + let result = apply_cors_headers(bucket, &http::Method::GET, &headers).await; + assert!( + result.is_some(), + "Expected Some empty headers when bucket has CORS config but origin does not match any rule" + ); + let result = result.unwrap(); + assert!(result.get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN).is_none()); + assert!(result.get(cors::response::ACCESS_CONTROL_ALLOW_METHODS).is_none()); + + metadata_sys::set_bucket_metadata(bucket.to_string(), BucketMetadata::new(bucket)) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_apply_cors_headers_credentialed_request_with_wildcard_origin() { + if metadata_sys::get_global_bucket_metadata_sys().is_none() { + eprintln!("Skipping test: GLOBAL_BucketMetadataSys not initialized"); + return; + } + + let bucket = "test-bucket-credentialed-cors"; + let mut bm = BucketMetadata::new(bucket); + bm.cors_config = Some(CORSConfiguration { + cors_rules: vec![CORSRule { + allowed_headers: Some(vec!["*".to_string()]), + allowed_methods: vec!["GET".to_string()], + allowed_origins: vec!["*".to_string()], + expose_headers: None, + id: Some("credentialed-unit".to_string()), + max_age_seconds: None, + }], + }); + metadata_sys::set_bucket_metadata(bucket.to_string(), bm).await.unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(cors::standard::ORIGIN, "https://console.localhost".parse().unwrap()); + headers.insert(cors::request::ACCESS_CONTROL_REQUEST_METHOD, "GET".parse().unwrap()); + headers.insert(cors::request::ACCESS_CONTROL_REQUEST_HEADERS, "x-amz-content-sha256".parse().unwrap()); + headers.insert(http::header::AUTHORIZATION, "AWS4-HMAC-SHA256 Credential=test/20260302/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256, Signature=abc".parse().unwrap()); + + let result = apply_cors_headers(bucket, &http::Method::OPTIONS, &headers).await.unwrap(); + assert_eq!( + result.get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(), + "https://console.localhost", + ); + assert_eq!(result.get(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true"); + assert_eq!(result.get(cors::standard::VARY).unwrap(), "Origin"); + + metadata_sys::set_bucket_metadata(bucket.to_string(), BucketMetadata::new(bucket)) + .await + .unwrap(); + } + #[tokio::test] async fn test_apply_cors_headers_unsupported_method() { // Test with unsupported HTTP method diff --git a/scripts/s3-tests/excluded_tests.txt b/scripts/s3-tests/excluded_tests.txt new file mode 100644 index 00000000..7616b134 --- /dev/null +++ b/scripts/s3-tests/excluded_tests.txt @@ -0,0 +1,425 @@ +# Excluded S3 feature tests +# ========================= +# +# These tests are intentionally excluded from RustFS compatibility gating. +# +# Exclusion buckets: +# - Vendor-specific behavior not required for RustFS S3 compatibility +# - Intentionally unsupported by product decision (for example ACL authorization) + +# Vendor-specific / non-portable tests +test_100_continue_error_retry +test_account_usage +test_atomic_conditional_write_1mb +test_atomic_dual_conditional_write_1mb +test_atomic_write_bucket_gone +test_bucket_get_location +test_bucket_head_extended +test_bucket_header_acl_grants +test_bucket_list_return_data +test_bucket_list_return_data_versioning +test_bucket_logging_bucket_acl_required +test_bucket_logging_bucket_auth_type +test_bucket_logging_cleanup_bucket_concurrent_deletion_j +test_bucket_logging_cleanup_bucket_concurrent_deletion_j_single +test_bucket_logging_cleanup_bucket_concurrent_deletion_s +test_bucket_logging_cleanup_bucket_concurrent_deletion_s_single +test_bucket_logging_cleanup_bucket_deletion_j +test_bucket_logging_cleanup_bucket_deletion_j_single +test_bucket_logging_cleanup_bucket_deletion_s +test_bucket_logging_cleanup_bucket_deletion_s_single +test_bucket_logging_cleanup_concurrent_disabling_j +test_bucket_logging_cleanup_concurrent_disabling_j_single +test_bucket_logging_cleanup_concurrent_disabling_s +test_bucket_logging_cleanup_concurrent_disabling_s_single +test_bucket_logging_cleanup_concurrent_updating_j +test_bucket_logging_cleanup_concurrent_updating_j_single +test_bucket_logging_cleanup_concurrent_updating_s +test_bucket_logging_cleanup_concurrent_updating_s_single +test_bucket_logging_cleanup_disabling_j +test_bucket_logging_cleanup_disabling_j_single +test_bucket_logging_cleanup_disabling_s +test_bucket_logging_cleanup_disabling_s_single +test_bucket_logging_cleanup_updating_j +test_bucket_logging_cleanup_updating_j_single +test_bucket_logging_cleanup_updating_s +test_bucket_logging_cleanup_updating_s_single +test_bucket_logging_concurrent_flush_j +test_bucket_logging_concurrent_flush_j_single +test_bucket_logging_concurrent_flush_s +test_bucket_logging_concurrent_flush_s_single +test_bucket_logging_conf_concurrent_updating_pfx_j +test_bucket_logging_conf_concurrent_updating_pfx_s +test_bucket_logging_conf_concurrent_updating_roll_j +test_bucket_logging_conf_concurrent_updating_roll_s +test_bucket_logging_conf_updating_pfx_j +test_bucket_logging_conf_updating_pfx_s +test_bucket_logging_conf_updating_roll_j +test_bucket_logging_conf_updating_roll_s +test_bucket_logging_copy_objects +test_bucket_logging_copy_objects_bucket +test_bucket_logging_copy_objects_bucket_versioned +test_bucket_logging_copy_objects_versioned +test_bucket_logging_delete_objects +test_bucket_logging_delete_objects_versioned +test_bucket_logging_event_type_j +test_bucket_logging_event_type_s +test_bucket_logging_flush_empty +test_bucket_logging_flush_j +test_bucket_logging_flush_j_single +test_bucket_logging_flush_s +test_bucket_logging_flush_s_single +test_bucket_logging_get_objects +test_bucket_logging_get_objects_versioned +test_bucket_logging_head_objects +test_bucket_logging_head_objects_versioned +test_bucket_logging_key_filter_j +test_bucket_logging_key_filter_s +test_bucket_logging_mpu_copy +test_bucket_logging_mpu_copy_versioned +test_bucket_logging_mpu_j +test_bucket_logging_mpu_s +test_bucket_logging_mpu_versioned_j +test_bucket_logging_mpu_versioned_s +test_bucket_logging_mtime +test_bucket_logging_multi_delete +test_bucket_logging_multi_delete_versioned +test_bucket_logging_multiple_prefixes +test_bucket_logging_notupdating_j +test_bucket_logging_notupdating_j_single +test_bucket_logging_notupdating_s +test_bucket_logging_notupdating_s_single +test_bucket_logging_object_acl_required +test_bucket_logging_object_meta +test_bucket_logging_part_cleanup_concurrent_deletion_j +test_bucket_logging_part_cleanup_concurrent_deletion_s +test_bucket_logging_part_cleanup_concurrent_disabling_j +test_bucket_logging_part_cleanup_concurrent_disabling_s +test_bucket_logging_part_cleanup_concurrent_updating_j +test_bucket_logging_part_cleanup_concurrent_updating_s +test_bucket_logging_part_cleanup_deletion_j +test_bucket_logging_part_cleanup_deletion_s +test_bucket_logging_part_cleanup_disabling_j +test_bucket_logging_part_cleanup_disabling_s +test_bucket_logging_part_cleanup_updating_j +test_bucket_logging_part_cleanup_updating_s +test_bucket_logging_partitioned_key +test_bucket_logging_permission_change_j +test_bucket_logging_permission_change_s +test_bucket_logging_put_and_flush +test_bucket_logging_put_concurrency +test_bucket_logging_put_objects +test_bucket_logging_put_objects_versioned +test_bucket_logging_roll_time +test_bucket_logging_simple_key +test_bucket_logging_single_prefix +test_bucket_logging_target_cleanup_j +test_bucket_logging_target_cleanup_j_single +test_bucket_logging_target_cleanup_s +test_bucket_logging_target_cleanup_s_single +test_bucket_policy_get_obj_acl_existing_tag +test_bucket_policy_get_obj_existing_tag +test_bucket_policy_get_obj_tagging_existing_tag +test_bucket_policy_put_obj_copy_source +test_bucket_policy_put_obj_copy_source_meta +test_bucket_policy_put_obj_kms_noenc +test_bucket_policy_put_obj_request_obj_tag +test_bucket_policy_put_obj_s3_incorrect_algo_sse_s3 +test_bucket_policy_put_obj_s3_noenc +test_bucket_policy_put_obj_tagging_existing_tag +test_bucket_policy_set_condition_operator_end_with_IfExists +test_bucket_policy_upload_part_copy +test_bucket_recreate_new_acl +test_bucket_recreate_overwrite_acl +test_copy_object_ifmatch_failed +test_copy_object_ifmatch_good +test_copy_object_ifnonematch_failed +test_copy_object_ifnonematch_good +test_cors_presigned_get_object_tenant_v2 +test_cors_presigned_get_object_v2 +test_cors_presigned_put_object_tenant_v2 +test_cors_presigned_put_object_v2 +test_create_bucket_bucket_owner_enforced +test_create_bucket_bucket_owner_preferred +test_create_bucket_object_writer +test_delete_marker_expiration +test_delete_marker_nonversioned +test_delete_marker_suspended +test_delete_marker_versioned +test_delete_object_current_if_match +test_delete_object_current_if_match_last_modified_time +test_delete_object_current_if_match_size +test_delete_object_if_match +test_delete_object_if_match_last_modified_time +test_delete_object_if_match_size +test_delete_object_version_if_match +test_delete_object_version_if_match_last_modified_time +test_delete_object_version_if_match_size +test_delete_objects_current_if_match +test_delete_objects_current_if_match_last_modified_time +test_delete_objects_current_if_match_size +test_delete_objects_if_match +test_delete_objects_if_match_last_modified_time +test_delete_objects_if_match_size +test_delete_objects_version_if_match +test_delete_objects_version_if_match_last_modified_time +test_delete_objects_version_if_match_size +test_delete_tags_obj_public +test_encrypted_transfer_13b +test_encrypted_transfer_1MB +test_encrypted_transfer_1b +test_encrypted_transfer_1kb +test_encryption_sse_c_deny_algo_with_bucket_policy +test_encryption_sse_c_enforced_with_bucket_policy +test_encryption_sse_c_multipart_invalid_chunks_1 +test_encryption_sse_c_multipart_invalid_chunks_2 +test_encryption_sse_c_multipart_upload +test_encryption_sse_c_post_object_authenticated_request +test_encryption_sse_c_unaligned_multipart_upload +test_expected_bucket_owner +test_get_multipart_checksum_object_attributes +test_get_multipart_object_attributes +test_get_obj_tagging +test_get_object_attributes +test_get_paginated_multipart_object_attributes +test_get_single_multipart_object_attributes +test_get_sse_c_encrypted_object_attributes +test_get_tags_acl_public +test_head_bucket_usage +test_lifecycle_cloud_multiple_transition +test_lifecycle_cloud_transition +test_lifecycle_cloud_transition_large_obj +test_lifecycle_deletemarker_expiration +test_lifecycle_deletemarker_expiration_with_days_tag +test_lifecycle_expiration +test_lifecycle_expiration_date +test_lifecycle_expiration_header_and_tags_head +test_lifecycle_expiration_header_head +test_lifecycle_expiration_header_tags_head +test_lifecycle_expiration_newer_noncurrent +test_lifecycle_expiration_noncur_tags1 +test_lifecycle_expiration_size_gt +test_lifecycle_expiration_size_lt +test_lifecycle_expiration_tags1 +test_lifecycle_expiration_tags2 +test_lifecycle_expiration_versioned_tags2 +test_lifecycle_expiration_versioning_enabled +test_lifecycle_multipart_expiration +test_lifecycle_noncur_cloud_transition +test_lifecycle_noncur_expiration +test_lifecycle_noncur_transition +test_lifecycle_transition +test_lifecycle_transition_single_rule_multi_trans +test_lifecyclev2_expiration +test_list_buckets_anonymous +test_list_buckets_paginated +test_list_multipart_upload +test_list_multipart_upload_owner +test_multipart_checksum_sha256 +test_multipart_copy_multiple_sizes +test_multipart_copy_versioned +test_multipart_get_part +test_multipart_put_current_object_if_match +test_multipart_put_current_object_if_none_match +test_multipart_put_object_if_match +test_multipart_single_get_part +test_multipart_sse_c_get_part +test_multipart_upload +test_multipart_upload_contents +test_multipart_upload_resend_part +test_multipart_upload_small +test_multipart_use_cksum_helper_crc32 +test_multipart_use_cksum_helper_crc32c +test_multipart_use_cksum_helper_crc64nvme +test_multipart_use_cksum_helper_sha1 +test_multipart_use_cksum_helper_sha256 +test_non_multipart_get_part +test_non_multipart_sse_c_get_part +test_object_copy_canned_acl +test_object_header_acl_grants +test_object_lock_changing_mode_from_compliance +test_object_lock_changing_mode_from_governance_with_bypass +test_object_lock_changing_mode_from_governance_without_bypass +test_object_lock_delete_multipart_object_with_legal_hold_on +test_object_lock_delete_multipart_object_with_retention +test_object_lock_delete_object_with_legal_hold_off +test_object_lock_delete_object_with_legal_hold_on +test_object_lock_delete_object_with_retention +test_object_lock_delete_object_with_retention_and_marker +test_object_lock_get_legal_hold +test_object_lock_get_obj_lock +test_object_lock_get_obj_metadata +test_object_lock_get_obj_retention +test_object_lock_get_obj_retention_iso8601 +test_object_lock_multi_delete_object_with_retention +test_object_lock_put_legal_hold +test_object_lock_put_legal_hold_invalid_status +test_object_lock_put_obj_lock +test_object_lock_put_obj_lock_invalid_days +test_object_lock_put_obj_lock_invalid_mode +test_object_lock_put_obj_lock_invalid_status +test_object_lock_put_obj_lock_invalid_years +test_object_lock_put_obj_lock_with_days_and_years +test_object_lock_put_obj_retention +test_object_lock_put_obj_retention_increase_period +test_object_lock_put_obj_retention_invalid_mode +test_object_lock_put_obj_retention_override_default_retention +test_object_lock_put_obj_retention_shorten_period +test_object_lock_put_obj_retention_shorten_period_bypass +test_object_lock_put_obj_retention_versionid +test_object_lock_suspend_versioning +test_object_lock_uploading_obj +test_object_raw_get_x_amz_expires_not_expired +test_object_raw_get_x_amz_expires_not_expired_tenant +test_object_raw_get_x_amz_expires_out_max_range +test_object_raw_get_x_amz_expires_out_positive_range +test_object_raw_put_authenticated_expired +test_object_read_unreadable +test_object_requestid_matches_header_on_error +test_object_set_get_unicode_metadata +test_object_write_with_chunked_transfer_encoding +test_post_object_invalid_date_format +test_post_object_invalid_request_field_value +test_post_object_missing_policy_condition +test_post_object_request_missing_policy_specified_field +test_post_object_set_key_from_filename +test_post_object_success_redirect_action +test_post_object_tags_anonymous_request +test_post_object_wrong_bucket +test_put_bucket_logging_account_j +test_put_bucket_logging_account_s +test_put_bucket_logging_extensions +test_put_bucket_logging_policy_wildcard_objects +test_put_bucket_logging_tenant_j +test_put_bucket_logging_tenant_s +test_put_bucket_ownership_bucket_owner_enforced +test_put_bucket_ownership_bucket_owner_preferred +test_put_bucket_ownership_object_writer +test_put_current_object_if_match +test_put_current_object_if_none_match +test_put_delete_tags +test_put_max_tags +test_put_modify_tags +test_put_obj_with_tags +test_put_object_current_if_match +test_put_object_if_match +test_put_tags_acl_public +test_ranged_big_request_response_code +test_ranged_request_response_code +test_ranged_request_return_trailing_bytes_response_code +test_ranged_request_skip_leading_bytes_response_code +test_read_through +test_restore_noncur_obj +test_restore_object_permanent +test_restore_object_temporary +test_sse_kms_default_post_object_authenticated_request +test_sse_kms_default_upload_1b +test_sse_kms_default_upload_1kb +test_sse_kms_default_upload_1mb +test_sse_kms_default_upload_8mb +test_sse_kms_method_head +test_sse_kms_multipart_invalid_chunks_1 +test_sse_kms_multipart_invalid_chunks_2 +test_sse_kms_multipart_upload +test_sse_kms_post_object_authenticated_request +test_sse_kms_present +test_sse_kms_transfer_13b +test_sse_kms_transfer_1MB +test_sse_kms_transfer_1b +test_sse_kms_transfer_1kb +test_sse_s3_default_method_head +test_sse_s3_default_multipart_upload +test_sse_s3_default_post_object_authenticated_request +test_sse_s3_default_upload_1b +test_sse_s3_default_upload_1kb +test_sse_s3_default_upload_1mb +test_sse_s3_default_upload_8mb +test_sse_s3_encrypted_upload_1b +test_sse_s3_encrypted_upload_1kb +test_sse_s3_encrypted_upload_1mb +test_sse_s3_encrypted_upload_8mb +test_versioned_object_acl_no_version_specified +test_versioning_copy_obj_version +test_versioning_multi_object_delete_with_marker_create +test_versioning_obj_create_overwrite_multipart +test_versioning_obj_suspended_copy +test_versioning_stack_delete_merkers + +# Intentionally unsupported by design: ACL-related tests +test_object_raw_get +test_object_raw_get_bucket_gone +test_object_raw_get_object_gone +test_object_copy_not_owned_object_bucket +test_100_continue +test_cors_origin_response +test_cors_origin_wildcard +test_bucket_policy_put_obj_grant +test_bucket_list_objects_anonymous +test_bucket_listv2_objects_anonymous +test_post_object_anonymous_request +test_post_object_set_success_code +test_post_object_set_invalid_success_code +test_access_bucket_private_object_private +test_access_bucket_private_object_publicread +test_access_bucket_private_object_publicreadwrite +test_access_bucket_private_objectv2_private +test_access_bucket_private_objectv2_publicread +test_access_bucket_private_objectv2_publicreadwrite +test_access_bucket_publicread_object_private +test_access_bucket_publicread_object_publicread +test_access_bucket_publicread_object_publicreadwrite +test_access_bucket_publicreadwrite_object_private +test_access_bucket_publicreadwrite_object_publicread +test_access_bucket_publicreadwrite_object_publicreadwrite +test_object_anon_put_write_access +test_get_public_acl_bucket_policy_status +test_get_authpublic_acl_bucket_policy_status +test_get_publicpolicy_acl_bucket_policy_status +test_get_nonpublicpolicy_acl_bucket_policy_status +test_block_public_put_bucket_acls +test_block_public_object_canned_acls +test_ignore_public_acls +test_bucket_policy_acl +test_bucketv2_policy_acl +test_bucket_policy_put_obj_acl +test_object_presigned_put_object_with_acl +test_object_put_acl_mtime +test_versioned_object_acl +test_object_presigned_put_object_with_acl_tenant +test_bucket_acl_canned +test_bucket_acl_canned_authenticatedread +test_bucket_acl_canned_during_create +test_bucket_acl_canned_private_to_private +test_bucket_acl_canned_publicreadwrite +test_bucket_acl_default +test_bucket_acl_grant_email +test_bucket_acl_grant_email_not_exist +test_bucket_acl_grant_nonexist_user +test_bucket_acl_grant_userid_fullcontrol +test_bucket_acl_grant_userid_read +test_bucket_acl_grant_userid_readacp +test_bucket_acl_grant_userid_write +test_bucket_acl_grant_userid_writeacp +test_bucket_acl_revoke_all +test_bucket_concurrent_set_canned_acl +test_object_acl +test_object_acl_canned +test_object_acl_canned_authenticatedread +test_object_acl_canned_bucketownerfullcontrol +test_object_acl_canned_bucketownerread +test_object_acl_canned_during_create +test_object_acl_canned_publicreadwrite +test_object_acl_default +test_object_acl_full_control_verify_attributes +test_object_acl_full_control_verify_owner +test_object_acl_read +test_object_acl_readacp +test_object_acl_write +test_object_acl_writeacp +test_put_bucket_acl_grant_group_read +test_object_raw_authenticated_bucket_acl +test_object_raw_authenticated_object_acl +test_object_raw_get_bucket_acl +test_object_raw_get_object_acl +test_cors_presigned_put_object_with_acl +test_cors_presigned_put_object_tenant_with_acl diff --git a/scripts/s3-tests/implemented_tests.txt b/scripts/s3-tests/implemented_tests.txt index bf9f43a2..0ce1c619 100644 --- a/scripts/s3-tests/implemented_tests.txt +++ b/scripts/s3-tests/implemented_tests.txt @@ -3,6 +3,7 @@ # # These tests SHOULD PASS on RustFS for standard S3 API compatibility. # Run these tests to verify RustFS S3 compatibility. +# Intentionally excluded tests (for example ACL authorization tests) are tracked in excluded_tests.txt. # # Covered operations: # - Bucket: Create, Delete, List, Head, GetLocation @@ -23,7 +24,6 @@ # - SSE-KMS: KMS-related edge cases # - Bucket Policy: Multipart upload authorization, SSE condition keys, grant header conditions # - Versioning: Concurrent multi-object delete -# - ACL: Bucket and Object ACL # - Object Copy: Same/cross bucket, metadata, versioning # - POST Object: HTML form upload # - Raw requests: Authenticated and anonymous @@ -36,7 +36,6 @@ # - DeleteObject: Proper NoSuchBucket for deleted buckets # - Multipart Copy: InvalidRange when CopySourceRange exceeds source size # -# Total: 396 tests test_basic_key_count test_bucket_create_naming_bad_short_one @@ -120,13 +119,6 @@ test_bucketv2_notexist test_bucketv2_policy_another_bucket test_get_bucket_policy_status test_get_nonpublicpolicy_principal_bucket_policy_status -test_get_public_acl_bucket_policy_status -test_get_authpublic_acl_bucket_policy_status -test_get_publicpolicy_acl_bucket_policy_status -test_get_nonpublicpolicy_acl_bucket_policy_status -test_block_public_put_bucket_acls -test_block_public_object_canned_acls -test_ignore_public_acls test_set_get_del_bucket_policy test_get_object_ifmatch_good test_get_object_ifmodifiedsince_good @@ -146,7 +138,6 @@ test_multipart_upload_overwrite_existing_object test_multipart_upload_size_too_small test_object_copy_bucket_not_found test_object_copy_key_not_found -test_object_copy_not_owned_object_bucket test_object_head_zero_bytes test_get_obj_head_tagging test_object_metadata_replaced_on_put @@ -223,17 +214,10 @@ test_get_undefined_public_block # Bucket policy tests test_bucketv2_policy -test_bucket_policy_acl -test_bucketv2_policy_acl test_bucket_policy_another_bucket test_bucket_policy_allow_notprincipal -test_bucket_policy_multipart -test_bucket_policy_put_obj_acl -test_bucket_policy_put_obj_grant test_bucket_policy_put_obj_kms_s3 test_bucket_policy_put_obj_s3_kms -test_object_presigned_put_object_with_acl -test_object_put_acl_mtime # Object ownership test_create_bucket_no_ownership_controls @@ -276,7 +260,6 @@ test_object_lock_put_obj_retention_invalid_bucket # Versioning tests test_get_versioned_object_attributes test_versioned_concurrent_object_create_and_remove -test_versioned_object_acl test_versioning_bucket_create_suspend test_versioning_bucket_atomic_upload_return_version_id test_versioning_bucket_multipart_upload_return_version_id @@ -294,54 +277,8 @@ test_versioning_obj_plain_null_version_removal test_versioning_obj_suspend_versions # Tenant and presigned tests -test_object_presigned_put_object_with_acl_tenant -# ACL tests (bucket and object) -test_access_bucket_private_object_private -test_access_bucket_private_object_publicread -test_access_bucket_private_object_publicreadwrite -test_access_bucket_private_objectv2_private -test_access_bucket_private_objectv2_publicread -test_access_bucket_private_objectv2_publicreadwrite -test_access_bucket_publicread_object_private -test_access_bucket_publicread_object_publicread -test_access_bucket_publicread_object_publicreadwrite -test_access_bucket_publicreadwrite_object_private -test_access_bucket_publicreadwrite_object_publicread -test_access_bucket_publicreadwrite_object_publicreadwrite -test_bucket_acl_canned -test_bucket_acl_canned_authenticatedread -test_bucket_acl_canned_during_create -test_bucket_acl_canned_private_to_private -test_bucket_acl_canned_publicreadwrite -test_bucket_acl_default -test_bucket_acl_grant_email -test_bucket_acl_grant_email_not_exist -test_bucket_acl_grant_nonexist_user -test_bucket_acl_grant_userid_fullcontrol -test_bucket_acl_grant_userid_read -test_bucket_acl_grant_userid_readacp -test_bucket_acl_grant_userid_write -test_bucket_acl_grant_userid_writeacp -test_bucket_acl_revoke_all -test_bucket_concurrent_set_canned_acl -test_object_acl -test_object_acl_canned -test_object_acl_canned_authenticatedread -test_object_acl_canned_bucketownerfullcontrol -test_object_acl_canned_bucketownerread -test_object_acl_canned_during_create -test_object_acl_canned_publicreadwrite -test_object_acl_default -test_object_acl_full_control_verify_attributes -test_object_acl_full_control_verify_owner -test_object_acl_read -test_object_acl_readacp -test_object_acl_write -test_object_acl_writeacp test_object_anon_put -test_object_anon_put_write_access -test_put_bucket_acl_grant_group_read # Bucket creation and naming test_bucket_create_exists @@ -356,7 +293,6 @@ test_bucket_list_delimiter_prefix test_bucket_list_delimiter_prefix_underscore test_bucket_list_many test_bucket_list_maxkeys_one -test_bucket_list_objects_anonymous test_bucket_list_objects_anonymous_fail test_bucket_list_unordered test_bucket_listv2_both_continuationtoken_startafter @@ -364,14 +300,12 @@ test_bucket_listv2_delimiter_prefix test_bucket_listv2_delimiter_prefix_underscore test_bucket_listv2_many test_bucket_listv2_maxkeys_one -test_bucket_listv2_objects_anonymous test_bucket_listv2_objects_anonymous_fail test_bucket_listv2_unordered # Object copy tests test_object_copy_16m test_object_copy_diff_bucket -test_object_copy_not_owned_bucket test_object_copy_replacing_metadata test_object_copy_retaining_metadata test_object_copy_same_bucket @@ -388,20 +322,12 @@ test_object_content_encoding_aws_chunked # Raw request tests test_object_raw_authenticated -test_object_raw_authenticated_bucket_acl test_object_raw_authenticated_bucket_gone -test_object_raw_authenticated_object_acl test_object_raw_authenticated_object_gone -test_object_raw_get -test_object_raw_get_bucket_acl -test_object_raw_get_bucket_gone -test_object_raw_get_object_acl -test_object_raw_get_object_gone test_object_raw_get_x_amz_expires_out_range_zero test_object_raw_response_headers # POST Object tests -test_post_object_anonymous_request test_post_object_authenticated_no_content_type test_post_object_authenticated_request test_post_object_authenticated_request_bad_access_key @@ -420,8 +346,6 @@ test_post_object_missing_content_length_argument test_post_object_missing_expires_condition test_post_object_missing_signature test_post_object_no_key_specified -test_post_object_set_invalid_success_code -test_post_object_set_success_code test_post_object_upload_larger_than_chunk test_post_object_upload_size_below_minimum test_post_object_upload_size_limit_exceeded @@ -474,18 +398,13 @@ test_object_checksum_crc64nvme # CORS tests test_set_cors -test_cors_origin_response -test_cors_origin_wildcard test_cors_header_option test_cors_presigned_get_object test_cors_presigned_get_object_tenant test_cors_presigned_put_object -test_cors_presigned_put_object_with_acl test_cors_presigned_put_object_tenant -test_cors_presigned_put_object_tenant_with_acl # HTTP and DeleteObject tests -test_100_continue test_abort_multipart_upload_not_found test_post_object_tags_authenticated_request test_object_delete_key_bucket_gone diff --git a/scripts/s3-tests/non_standard_tests.txt b/scripts/s3-tests/non_standard_tests.txt index d2b22a34..a04f5e36 100644 --- a/scripts/s3-tests/non_standard_tests.txt +++ b/scripts/s3-tests/non_standard_tests.txt @@ -1,8 +1,9 @@ -# Non-standard S3 tests (Ceph/RGW/MinIO specific) -# ================================================ +# Legacy non-standard S3 test list (compatibility fallback) +# ========================================================= # +# This file is kept for backward compatibility. +# Active exclusion classification now uses excluded_tests.txt. # These tests use vendor-specific extensions not part of AWS S3 API. -# They are PERMANENTLY EXCLUDED from RustFS compatibility testing. # # Exclusion reasons: # - fails_on_aws marker: Ceph-specific features diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index 2769c7be..29c41439 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -61,9 +61,9 @@ log_error() { # Test Classification Files # ============================================================================= # Tests are classified into three categories stored in text files: -# - non_standard_tests.txt: Ceph/RGW specific tests (permanently excluded) -# - unimplemented_tests.txt: Standard S3 features not yet implemented # - implemented_tests.txt: Tests that should pass on RustFS +# - unimplemented_tests.txt: Standard S3 features planned but not yet implemented +# - excluded_tests.txt: Tests intentionally excluded from RustFS gating # # By default, only tests listed in implemented_tests.txt are run. # Use TESTEXPR env var to override and run custom test selection. @@ -72,8 +72,9 @@ log_error() { # Test list files location TEST_LISTS_DIR="${SCRIPT_DIR}" IMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/implemented_tests.txt" -NON_STANDARD_TESTS_FILE="${TEST_LISTS_DIR}/non_standard_tests.txt" UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt" +EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt" +LEGACY_NON_STANDARD_TESTS_FILE="${TEST_LISTS_DIR}/non_standard_tests.txt" # ============================================================================= # build_testexpr_from_file: Read test names from file and build pytest -k expr @@ -110,7 +111,7 @@ build_testexpr_from_file() { # MARKEXPR: pytest marker expression (safety net for marker-based filtering) # ============================================================================= # Even though we use file-based test selection, we keep marker exclusions -# as a safety net to ensure no non-standard tests slip through. +# as a safety net to ensure excluded tests do not slip through. # ============================================================================= if [[ -z "${MARKEXPR:-}" ]]; then # Minimal marker exclusions as safety net (file-based filtering is primary) @@ -121,7 +122,7 @@ fi # TESTEXPR: pytest -k expression to select specific tests # ============================================================================= # By default, builds an inclusion expression from implemented_tests.txt, -# combined with an exclusion expression from non_standard_tests.txt and +# combined with an exclusion expression from excluded_tests.txt and # unimplemented_tests.txt to prevent substring-matching collisions. # # For example, "test_object_raw_get" in the include list would also match @@ -144,10 +145,16 @@ if [[ -z "${TESTEXPR:-}" ]]; then TEST_COUNT=$(grep -v '^#' "${IMPLEMENTED_TESTS_FILE}" | grep -v '^[[:space:]]*$' | wc -l | xargs) log_info "Loaded ${TEST_COUNT} tests from implemented_tests.txt" - # Build exclusion expression from non-standard and unimplemented lists + # Build exclusion expression from excluded and unimplemented lists # to guard against pytest -k substring matching false positives EXCLUDE_EXPR="" - for exclude_file in "${NON_STANDARD_TESTS_FILE}" "${UNIMPLEMENTED_TESTS_FILE}"; do + EXCLUDE_FILES=("${EXCLUDED_TESTS_FILE}" "${UNIMPLEMENTED_TESTS_FILE}") + if [[ ! -f "${EXCLUDED_TESTS_FILE}" && -f "${LEGACY_NON_STANDARD_TESTS_FILE}" ]]; then + log_warn "excluded_tests.txt not found, fallback to legacy non_standard_tests.txt" + EXCLUDE_FILES=("${LEGACY_NON_STANDARD_TESTS_FILE}" "${UNIMPLEMENTED_TESTS_FILE}") + fi + + for exclude_file in "${EXCLUDE_FILES[@]}"; do if [[ -f "${exclude_file}" ]]; then FILE_EXPR=$(build_testexpr_from_file "${exclude_file}") if [[ -n "${FILE_EXPR}" ]]; then @@ -161,7 +168,7 @@ if [[ -z "${TESTEXPR:-}" ]]; then if [[ -n "${EXCLUDE_EXPR}" ]]; then TESTEXPR="(${INCLUDE_EXPR}) and not (${EXCLUDE_EXPR})" - log_info "Added exclusion guard from non_standard + unimplemented lists" + log_info "Added exclusion guard from excluded + unimplemented lists" else TESTEXPR="${INCLUDE_EXPR}" fi @@ -246,9 +253,9 @@ Environment Variables: Final path: \${DATA_ROOT}/test-data/\${CONTAINER_NAME} Test Classification Files (in scripts/s3-tests/): - implemented_tests.txt - Tests that should pass (run by default) - unimplemented_tests.txt - Standard S3 features not yet implemented - non_standard_tests.txt - Ceph/RGW specific tests (permanently excluded) + implemented_tests.txt - Implemented tests (run by default) + unimplemented_tests.txt - Standard S3 tests planned but not implemented + excluded_tests.txt - Tests intentionally excluded from RustFS gating Notes: - Tests are loaded from implemented_tests.txt by default diff --git a/scripts/s3-tests/unimplemented_tests.txt b/scripts/s3-tests/unimplemented_tests.txt index d14d11b9..d67dcf3b 100644 --- a/scripts/s3-tests/unimplemented_tests.txt +++ b/scripts/s3-tests/unimplemented_tests.txt @@ -1,8 +1,8 @@ # Unimplemented S3 feature tests # ============================== # -# These tests cover STANDARD S3 features not yet implemented in RustFS. -# They are TEMPORARILY EXCLUDED and should be enabled as features are added. +# These tests cover STANDARD S3 features planned but not yet implemented in RustFS. +# They are TEMPORARILY EXCLUDED and should move to implemented_tests.txt when done. # # Unimplemented features: # - Bucket Logging: Access logging @@ -12,6 +12,8 @@ # Failed tests test_bucket_create_delete_bucket_ownership test_bucket_logging_owner +test_object_copy_not_owned_bucket +test_bucket_policy_multipart test_post_object_upload_checksum test_put_bucket_logging test_put_bucket_logging_errors