From 55d44622ed4491321c730fa35c8d3e7d36f24b7a Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 18 Nov 2025 21:51:10 +0800 Subject: [PATCH] list object include deleted support (#882) Co-authored-by: houseme --- crates/ahm/src/heal/storage.rs | 2 +- crates/ahm/src/scanner/data_scanner.rs | 1 + crates/ecstore/src/data_usage.rs | 1 + crates/ecstore/src/set_disk.rs | 1 + crates/ecstore/src/sets.rs | 1 + crates/ecstore/src/store.rs | 14 ++++++++++++-- crates/ecstore/src/store_api.rs | 1 + crates/ecstore/src/store_list_objects.rs | 8 ++++++-- crates/utils/src/http/headers.rs | 1 + rustfs/src/storage/ecfs.rs | 6 ++++++ 10 files changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/ahm/src/heal/storage.rs b/crates/ahm/src/heal/storage.rs index dddada2c..cdee2ebe 100644 --- a/crates/ahm/src/heal/storage.rs +++ b/crates/ahm/src/heal/storage.rs @@ -498,7 +498,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self .ecstore .clone() - .list_objects_v2(bucket, prefix, None, None, 1000, false, None) + .list_objects_v2(bucket, prefix, None, None, 1000, false, None, false) .await { Ok(list_info) => { diff --git a/crates/ahm/src/scanner/data_scanner.rs b/crates/ahm/src/scanner/data_scanner.rs index e4b2ab1a..e895af0a 100644 --- a/crates/ahm/src/scanner/data_scanner.rs +++ b/crates/ahm/src/scanner/data_scanner.rs @@ -1005,6 +1005,7 @@ impl Scanner { 100, // max_keys - small limit for performance false, // fetch_owner None, // start_after + false, // incl_deleted ) .await { diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index 001a054f..822aaa38 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -277,6 +277,7 @@ pub async fn compute_bucket_usage(store: Arc, bucket_name: &str) -> Res 1000, // max_keys false, // fetch_owner None, // start_after + false, // incl_deleted ) .await?; diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 54a31dd3..2580b68d 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -4483,6 +4483,7 @@ impl StorageAPI for SetDisks { _max_keys: i32, _fetch_owner: bool, _start_after: Option, + _incl_deleted: bool, ) -> Result { unimplemented!() } diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 51c92560..976fcd56 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -440,6 +440,7 @@ impl StorageAPI for Sets { _max_keys: i32, _fetch_owner: bool, _start_after: Option, + _incl_deleted: bool, ) -> Result { unimplemented!() } diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index a39cf9e1..fbaa19c0 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -1338,9 +1338,19 @@ impl StorageAPI for ECStore { max_keys: i32, fetch_owner: bool, start_after: Option, + incl_deleted: bool, ) -> Result { - self.inner_list_objects_v2(bucket, prefix, continuation_token, delimiter, max_keys, fetch_owner, start_after) - .await + self.inner_list_objects_v2( + bucket, + prefix, + continuation_token, + delimiter, + max_keys, + fetch_owner, + start_after, + incl_deleted, + ) + .await } #[instrument(skip(self))] diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index 499f80c0..dbad3911 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -1224,6 +1224,7 @@ pub trait StorageAPI: ObjectIO + Debug { max_keys: i32, fetch_owner: bool, start_after: Option, + incl_deleted: bool, ) -> Result; // ListObjectVersions TODO: FIXME: async fn list_object_versions( diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index 3c370eb2..676d43cf 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -225,6 +225,7 @@ impl ECStore { max_keys: i32, _fetch_owner: bool, start_after: Option, + incl_deleted: bool, ) -> Result { let marker = { if continuation_token.is_none() { @@ -234,7 +235,9 @@ impl ECStore { } }; - let loi = self.list_objects_generic(bucket, prefix, marker, delimiter, max_keys).await?; + let loi = self + .list_objects_generic(bucket, prefix, marker, delimiter, max_keys, incl_deleted) + .await?; Ok(ListObjectsV2Info { is_truncated: loi.is_truncated, continuation_token, @@ -251,6 +254,7 @@ impl ECStore { marker: Option, delimiter: Option, max_keys: i32, + incl_deleted: bool, ) -> Result { let opts = ListPathOptions { bucket: bucket.to_owned(), @@ -258,7 +262,7 @@ impl ECStore { separator: delimiter.clone(), limit: max_keys_plus_one(max_keys, marker.is_some()), marker, - incl_deleted: false, + incl_deleted, ask_disks: "strict".to_owned(), //TODO: from config ..Default::default() }; diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index 3eb75a38..856ae62f 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -163,6 +163,7 @@ pub const AMZ_TAGGING_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov"; pub const RUSTFS_FORCE_DELETE: &str = "X-Rustfs-Force-Delete"; +pub const RUSTFS_INCLUDE_DELETED: &str = "X-Rustfs-Include-Deleted"; pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-Status"; pub const RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size"; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index a59bdd3f..fc5bfb97 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -2225,6 +2225,11 @@ impl S3 for FS { let store = get_validated_store(&bucket).await?; + let incl_deleted = req + .headers + .get(rustfs_utils::http::headers::RUSTFS_INCLUDE_DELETED) + .is_some_and(|v| v.to_str().unwrap_or_default() == "true"); + let object_infos = store .list_objects_v2( &bucket, @@ -2234,6 +2239,7 @@ impl S3 for FS { max_keys, fetch_owner.unwrap_or_default(), start_after, + incl_deleted, ) .await .map_err(ApiError::from)?;