diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 61a26f22..b6905157 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -18,8 +18,8 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M pub const NULL_VERSION_ID: &str = "null"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; -pub const TIER_FV_ID: &str = "tier-free-versionID"; -pub const TIER_FV_MARKER: &str = "tier-free-marker"; +pub const TIER_FV_ID: &str = "tier-free-versionID"; +pub const TIER_FV_MARKER: &str = "tier-free-marker"; pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] @@ -309,7 +309,8 @@ impl FileInfo { } pub fn set_tier_free_version_id(&mut self, version_id: &str) { - self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); } pub fn tier_free_version_id(&self) -> String { @@ -317,19 +318,23 @@ impl FileInfo { } pub fn set_tier_free_version(&mut self) { - self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); } pub fn set_skip_tier_free_version(&mut self) { - self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); } pub fn skip_tier_free_version(&self) -> bool { - self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) + self.metadata + .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) } pub fn tier_free_version(&self) -> bool { - self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) + self.metadata + .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) } pub fn set_inline_data(&mut self) { diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index e65b0758..b389e635 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -47,7 +47,7 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; pub const TRANSITION_TIER: &str = "transition-tier"; -const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; +const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; // type ScanHeaderVersionFn = Box Result<()>>; @@ -486,11 +486,13 @@ impl FileMeta { return Err(Error::other("attempted to add invalid version")); } let encoded = ver.marshal_msg()?; - - if self.versions.len()+1 > 100 { - return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object")); + + if self.versions.len() + 1 > 100 { + return Err(Error::other( + "You've exceeded the limit on the number of versions you can create on this object", + )); } - + self.versions.push(FileMetaShallowVersion { header: FileMetaVersionHeader { mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), @@ -498,15 +500,15 @@ impl FileMeta { }, ..Default::default() }); - + let len = self.versions.len(); for (i, existing) in self.versions.iter().enumerate() { if existing.header.mod_time.unwrap().nanosecond() <= mod_time { - let vers = self.versions[i..len-1].to_vec(); - self.versions[i+1..].clone_from_slice(vers.as_slice()); + let vers = self.versions[i..len - 1].to_vec(); + self.versions[i + 1..].clone_from_slice(vers.as_slice()); self.versions[i] = FileMetaShallowVersion { header: ver.header(), - meta: encoded, + meta: encoded, }; return Ok(()); } @@ -564,7 +566,7 @@ impl FileMeta { ver.object.as_mut().unwrap().reset_inline_data(); self.set_idx(i, ver.clone())?; } else { - let vers = self.versions[i+1..].to_vec(); + let vers = self.versions[i + 1..].to_vec(); self.versions.extend(vers.iter().cloned()); let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi); if to_free { @@ -1927,10 +1929,22 @@ impl MetaObject { } pub fn set_transition(&mut self, fi: &FileInfo) { - self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec()); - self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec()); - self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec()); - self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec()); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), + fi.transition_status.as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), + fi.transitioned_objname.as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), + fi.transition_version_id.unwrap().as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), + fi.transition_tier.as_bytes().to_vec(), + ); } pub fn remove_restore_hdrs(&mut self) { @@ -1977,25 +1991,39 @@ impl MetaObject { if fi.skip_tier_free_version() { return (FileMetaVersion::default(), false); } - if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) { + if let Some(status) = self + .meta_sys + .get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) + { if *status == TRANSITION_COMPLETE.as_bytes().to_vec() { let vid = Uuid::parse_str(&fi.tier_free_version_id()); if let Err(err) = vid { - panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string()); + panic!( + "Invalid Tier Object delete marker versionId {} {}", + fi.tier_free_version_id(), + err.to_string() + ); } let vid = vid.unwrap(); - let mut free_entry = FileMetaVersion { + let mut free_entry = FileMetaVersion { version_type: VersionType::Delete, write_version: 0, ..Default::default() }; free_entry.delete_marker = Some(MetaDeleteMarker { version_id: Some(vid), - mod_time: self.mod_time, - meta_sys: Some(HashMap::>::new()), + mod_time: self.mod_time, + meta_sys: Some(HashMap::>::new()), }); - free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); + free_entry + .delete_marker + .as_mut() + .unwrap() + .meta_sys + .as_mut() + .unwrap() + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER); let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME); let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID); @@ -2003,7 +2031,14 @@ impl MetaObject { let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; for (k, v) in &self.meta_sys { if aa.contains(&k) { - free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone()); + free_entry + .delete_marker + .as_mut() + .unwrap() + .meta_sys + .as_mut() + .unwrap() + .insert(k.clone(), v.clone()); } } return (free_entry, true); diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index a07dc797..4125bb7d 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -122,7 +122,9 @@ pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result Builder lazy_static! { static ref SUPPORTED_QUERY_VALUES: HashMap = { let mut m = HashMap::new(); - m.insert("attributes".to_string(), true); - m.insert("partNumber".to_string(), true); - m.insert("versionId".to_string(), true); - m.insert("response-cache-control".to_string(), true); + m.insert("attributes".to_string(), true); + m.insert("partNumber".to_string(), true); + m.insert("versionId".to_string(), true); + m.insert("response-cache-control".to_string(), true); m.insert("response-content-disposition".to_string(), true); - m.insert("response-content-encoding".to_string(), true); - m.insert("response-content-language".to_string(), true); - m.insert("response-content-type".to_string(), true); - m.insert("response-expires".to_string(), true); + m.insert("response-content-encoding".to_string(), true); + m.insert("response-content-language".to_string(), true); + m.insert("response-content-type".to_string(), true); + m.insert("response-expires".to_string(), true); m }; - static ref SUPPORTED_HEADERS: HashMap = { let mut m = HashMap::new(); - m.insert("content-type".to_string(), true); - m.insert("cache-control".to_string(), true); - m.insert("content-encoding".to_string(), true); - m.insert("content-disposition".to_string(), true); - m.insert("content-language".to_string(), true); - m.insert("x-amz-website-redirect-location".to_string(), true); - m.insert("x-amz-object-lock-mode".to_string(), true); - m.insert("x-amz-metadata-directive".to_string(), true); + m.insert("content-type".to_string(), true); + m.insert("cache-control".to_string(), true); + m.insert("content-encoding".to_string(), true); + m.insert("content-disposition".to_string(), true); + m.insert("content-language".to_string(), true); + m.insert("x-amz-website-redirect-location".to_string(), true); + m.insert("x-amz-object-lock-mode".to_string(), true); + m.insert("x-amz-metadata-directive".to_string(), true); m.insert("x-amz-object-lock-retain-until-date".to_string(), true); - m.insert("expires".to_string(), true); - m.insert("x-amz-replication-status".to_string(), true); + m.insert("expires".to_string(), true); + m.insert("x-amz-replication-status".to_string(), true); m }; - static ref SSE_HEADERS: HashMap = { let mut m = HashMap::new(); - m.insert("x-amz-server-side-encryption".to_string(), true); - m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); - m.insert("x-amz-server-side-encryption-context".to_string(), true); + m.insert("x-amz-server-side-encryption".to_string(), true); + m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); + m.insert("x-amz-server-side-encryption-context".to_string(), true); m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); m }; } @@ -201,7 +201,11 @@ pub fn is_sse_header(header_key: &str) -> bool { pub fn is_amz_header(header_key: &str) -> bool { let key = header_key.to_lowercase(); - key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-") + key.starts_with("x-amz-meta-") + || key.starts_with("x-amz-grant-") + || key == "x-amz-acl" + || is_sse_header(header_key) + || key.starts_with("x-amz-checksum-") } pub fn is_rustfs_header(header_key: &str) -> bool { diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs index 06e5000e..98411f4d 100644 --- a/crates/utils/src/retry.rs +++ b/crates/utils/src/retry.rs @@ -1,8 +1,4 @@ -//use tokio_stream::Stream; -use std::future::Future; -use std::time::{Duration, Instant}; -use std::pin::Pin; -use std::task::{Context, Poll}; +use std::time::Duration; // MaxRetry is the maximum number of retries before stopping. pub const MAX_RETRY: i64 = 10; diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs index cc7ffc1f..565d9975 100644 --- a/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs @@ -24,9 +24,6 @@ pub struct LcAuditEvent { impl LcAuditEvent { pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self { - Self { - event, - source, - } + Self { event, source } } -} \ No newline at end of file +} diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 1f764023..32942e1a 100644 --- a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -1,53 +1,49 @@ +use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; +use futures::Future; +use http::HeaderMap; +use lazy_static::lazy_static; +use s3s::Body; +use sha2::{Digest, Sha256}; use std::any::{Any, TypeId}; +use std::collections::HashMap; use std::env; use std::io::{Cursor, Write}; use std::pin::Pin; use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; use std::sync::{Arc, Mutex}; -use futures::Future; -use lazy_static::lazy_static; -use s3s::Body; -use std::collections::HashMap; -use tracing::{error, info, warn}; -use sha2::{Digest, Sha256}; -use xxhash_rust::xxh64; -use uuid::Uuid; -use http::HeaderMap; use tokio::select; use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::{mpsc, RwLock}; -use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender}; +use tokio::sync::{RwLock, mpsc}; +use tracing::{error, info, warn}; +use uuid::Uuid; +use xxhash_rust::xxh64; -use s3s::dto::BucketLifecycleConfiguration; +use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}; +use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; +use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; +use super::tier_sweeper::{Jentry, delete_object_from_remote_tier}; +use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys}; +use crate::client::object_api_utils::new_getobjectreader; use crate::error::Error; -use crate::event::name::EventName; -use crate::store::ECStore; -use crate::store_api::StorageAPI; -use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,}; use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; +use crate::event::name::EventName; +use crate::event_notification::{EventArgs, send_event}; +use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; -use crate::client::object_api_utils::{new_getobjectreader,}; -use crate::event_notification::{send_event, EventArgs}; use crate::heal::{ + data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object}, data_scanner_metric::ScannerMetrics, - data_scanner::{ - apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects, - }, data_usage_cache::TierStats, }; -use crate::global::GLOBAL_LocalNodeName; -use crate::bucket::{ - metadata_sys::get_lifecycle_config, - versioning_sys::BucketVersioningSys, -}; +use crate::store::ECStore; +use crate::store_api::StorageAPI; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete}; use crate::tier::warm_backend::WarmBackendGetOpts; -use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; -use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats}; -use super::tier_sweeper::{delete_object_from_remote_tier, Jentry}; -use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent}; +use s3s::dto::BucketLifecycleConfiguration; pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; -pub type TraceFn = Arc) -> Pin + Send>> + Send + Sync + 'static>; +pub type TraceFn = + Arc) -> Pin + Send>> + Send + Sync + 'static>; pub type ExpiryOpType = Box; static XXHASH_SEED: u64 = 0; @@ -74,8 +70,7 @@ impl LifecycleSys { Some(lc) } - pub fn trace(oi: &ObjectInfo) -> TraceFn - { + pub fn trace(oi: &ObjectInfo) -> TraceFn { todo!(); } } @@ -132,11 +127,11 @@ pub trait ExpiryOp: 'static { #[derive(Debug, Default, Clone)] pub struct TransitionedObject { - pub name: String, - pub version_id: String, - pub tier: String, + pub name: String, + pub version_id: String, + pub tier: String, pub free_version: bool, - pub status: String, + pub status: String, } struct FreeVersionTask(ObjectInfo); @@ -181,8 +176,6 @@ pub struct ExpiryState { stats: Option, } - - impl ExpiryState { #[allow(clippy::new_ret_no_self)] pub fn new() -> Arc> { @@ -203,7 +196,7 @@ impl ExpiryState { if rxs.len() == 0 { return 0; } - let mut tasks=0; + let mut tasks = 0; for rx in rxs.iter() { tasks += rx.lock().await.len(); } @@ -244,7 +237,11 @@ impl ExpiryState { } pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { - let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()}; + let task = ExpiryTask { + obj_info: oi.clone(), + event: event.clone(), + src: src.clone(), + }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; @@ -265,7 +262,11 @@ impl ExpiryState { return; } - let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event}; + let task = NewerNoncurrentTask { + bucket: String::from(bucket), + versions: versions, + event: lc_event, + }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; @@ -285,7 +286,7 @@ impl ExpiryState { if self.tasks_tx.len() == 0 { return None; } - Some(self.tasks_tx[h as usize %self.tasks_tx.len()].clone()) + Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone()) } pub async fn resize_workers(n: usize, api: Arc) { @@ -311,10 +312,10 @@ impl ExpiryState { let mut l = state.tasks_tx.len(); while l > n { - let worker = state.tasks_tx[l-1].clone(); + let worker = state.tasks_tx[l - 1].clone(); worker.send(None).await.unwrap_or(()); - state.tasks_tx.remove(l-1); - state.tasks_rx.remove(l-1); + state.tasks_tx.remove(l - 1); + state.tasks_rx.remove(l - 1); *state.stats.as_mut().expect("err").workers.get_mut() -= 1; l -= 1; } @@ -357,7 +358,7 @@ impl ExpiryState { else if v.as_any().is::() { let v = v.as_any().downcast_ref::().expect("err!"); let oi = v.0.clone(); - + } else { //info!("Invalid work type - {:?}", v); @@ -422,7 +423,11 @@ impl TransitionState { } pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { - let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()}; + let task = TransitionTask { + obj_info: oi.clone(), + src: src.clone(), + event: event.clone(), + }; select! { //_ -> t.ctx.Done() => (), _ = self.transition_tx.send(Some(task)) => (), @@ -438,8 +443,8 @@ impl TransitionState { } pub async fn init(api: Arc) { - let mut n = 10;//globalAPIConfig.getTransitionWorkers(); - let tw = 10;//globalILMConfig.getTransitionWorkers(); + let mut n = 10; //globalAPIConfig.getTransitionWorkers(); + let tw = 10; //globalILMConfig.getTransitionWorkers(); if tw > 0 { n = tw; } @@ -512,8 +517,10 @@ impl TransitionState { pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { let mut tier_stats = self.last_day_stats.lock().unwrap(); - tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts)) - .or_insert(LastDayTierStats::default()); + tier_stats + .entry(tier.to_string()) + .and_modify(|e| e.add_stats(ts)) + .or_insert(LastDayTierStats::default()); } pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { @@ -574,7 +581,10 @@ impl AuditTierOp { } pub fn string(&self) -> String { - format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error) + format!( + "tier:{},respNS:{},tx:{},err:{}", + self.tier, self.time_to_responsens, self.output_bytes, self.error + ) } } @@ -636,16 +646,21 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { } GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await; } - _ => () + _ => (), } } } -pub async fn expire_transitioned_object(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result { +pub async fn expire_transitioned_object( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> Result { //let traceFn = GLOBAL_LifecycleSys.trace(oi); let mut opts = ObjectOptions { - versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, - expiration: ExpirationOptions {expire: true}, + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + expiration: ExpirationOptions { expire: true }, ..Default::default() }; if lc_event.action == IlmAction::DeleteVersionAction { @@ -660,10 +675,15 @@ pub async fn expire_transitioned_object(api: Arc, oi: &ObjectInfo, lc_e return Ok(dobj); } Err(err) => return Err(std::io::Error::other(err)), - } + } } - let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await; + let ret = delete_object_from_remote_tier( + &oi.transitioned_object.name, + &oi.transitioned_object.version_id, + &oi.transitioned_object.tier, + ) + .await; if ret.is_ok() { opts.skip_decommissioned = true; } else { @@ -679,8 +699,8 @@ pub async fn expire_transitioned_object(api: Arc, oi: &ObjectInfo, lc_e event_name = EventName::ObjectRemovedDeleteMarkerCreated; } let obj_info = ObjectInfo { - name: oi.name.clone(), - version_id: oi.version_id, + name: oi.name.clone(), + version_id: oi.version_id, delete_marker: oi.delete_marker, ..Default::default() }; @@ -712,15 +732,15 @@ pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditE let opts = ObjectOptions { transition: TransitionOptions { status: lifecycle::TRANSITION_PENDING.to_string(), - tier: lae.event.storage_class, - etag: oi.etag.clone().expect("err").to_string(), + tier: lae.event.storage_class, + etag: oi.etag.clone().expect("err").to_string(), ..Default::default() }, //lifecycle_audit_event: lae, - version_id: Some(oi.version_id.expect("err").to_string()), - versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, - version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, - mod_time: oi.mod_time, + version_id: Some(oi.version_id.expect("err").to_string()), + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, + mod_time: oi.mod_time, ..Default::default() }; time_ilm(1); @@ -731,7 +751,14 @@ pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn { todo!(); } -pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result { +pub async fn get_transitioned_object_reader( + bucket: &str, + object: &str, + rs: HTTPRangeSpec, + h: HeaderMap, + oi: ObjectInfo, + opts: &ObjectOptions, +) -> Result { let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { Ok(d) => d, @@ -752,7 +779,9 @@ pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTP //return Ok(HttpFileReader::new(rs, &oi, opts, &h)); //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length) - let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?; + let reader = tgt_client + .get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts) + .await?; Ok(get_fn(reader, h)) } @@ -771,18 +800,18 @@ pub trait LifecycleOps { impl LifecycleOps for ObjectInfo { fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts { lifecycle::ObjectOpts { - name: self.name.clone(), - user_tags: self.user_tags.clone(), - version_id: self.version_id.expect("err").to_string(), - mod_time: self.mod_time, - size: self.size, - is_latest: self.is_latest, - num_versions: self.num_versions, - delete_marker: self.delete_marker, + name: self.name.clone(), + user_tags: self.user_tags.clone(), + version_id: self.version_id.expect("err").to_string(), + mod_time: self.mod_time, + size: self.size, + is_latest: self.is_latest, + num_versions: self.num_versions, + delete_marker: self.delete_marker, successor_mod_time: self.successor_mod_time, //restore_ongoing: self.restore_ongoing, //restore_expires: self.restore_expires, - transition_status: self.transitioned_object.status.clone(), + transition_status: self.transitioned_object.status.clone(), ..Default::default() } } @@ -790,9 +819,9 @@ impl LifecycleOps for ObjectInfo { #[derive(Debug, Default, Clone)] pub struct S3Location { - pub bucketname: String, + pub bucketname: String, //pub encryption: Encryption, - pub prefix: String, + pub prefix: String, pub storage_class: String, //pub tagging: Tags, pub user_metadata: HashMap, @@ -803,13 +832,12 @@ pub struct OutputLocation(pub S3Location); #[derive(Debug, Default, Clone)] pub struct RestoreObjectRequest { - pub days: i64, - pub ror_type: String, - pub tier: String, - pub description: String, + pub days: i64, + pub ror_type: String, + pub tier: String, + pub description: String, //pub select_parameters: SelectParameters, - pub output_location: OutputLocation, + pub output_location: OutputLocation, } const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; - diff --git a/ecstore/src/bucket/lifecycle/lifecycle.rs b/ecstore/src/bucket/lifecycle/lifecycle.rs index 759fde7a..94d3ce5e 100644 --- a/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -1,12 +1,12 @@ +use s3s::dto::{ + BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition, + ObjectLockConfiguration, ObjectLockEnabled, Transition, +}; use std::cmp::Ordering; use std::env; use std::fmt::Display; -use s3s::dto::{ - BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration, - ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition, -}; use time::macros::{datetime, offset}; -use time::{self, OffsetDateTime, Duration}; +use time::{self, Duration, OffsetDateTime}; use crate::bucket::lifecycle::rule::TransitionOps; @@ -16,10 +16,11 @@ pub const TRANSITION_COMPLETE: &str = "complete"; pub const TRANSITION_PENDING: &str = "pending"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules"; -const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; -const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; -const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; -const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; +const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; +const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; +const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; +const ERR_LIFECYCLE_BUCKET_LOCKED: &str = + "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; #[derive(Debug, Clone, PartialEq, Eq)] pub enum IlmAction { @@ -52,7 +53,10 @@ impl IlmAction { if self.delete_restored() { return true; } - *self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction + *self == Self::DeleteVersionAction + || *self == Self::DeleteAction + || *self == Self::DeleteAllVersionsAction + || *self == Self::DelMarkerDeleteAllVersionsAction } } @@ -204,7 +208,10 @@ impl Lifecycle for BucketLifecycleConfiguration { return true; } let rule_expiration = rule.expiration.as_ref().expect("err!"); - if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { + if !rule_expiration.date.is_none() + && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() + < OffsetDateTime::now_utc().unix_timestamp() + { return true; } if !rule_expiration.date.is_none() { @@ -213,9 +220,12 @@ impl Lifecycle for BucketLifecycleConfiguration { if rule_expiration.expired_object_delete_marker.expect("err!") { return true; } - let rule_transitions: &[Transition]= &rule.transitions.as_ref().expect("err!"); + let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!"); let rule_transitions_0 = rule_transitions[0].clone(); - if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { + if !rule_transitions_0.date.is_none() + && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() + < OffsetDateTime::now_utc().unix_timestamp() + { return true; } if !rule.transitions.is_none() { @@ -242,18 +252,18 @@ impl Lifecycle for BucketLifecycleConfiguration { return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); } } /*else { - if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED { - return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED)); - } + if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED { + return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED)); + } }*/ } } } - for (i,_) in self.rules.iter().enumerate() { - if i == self.rules.len()-1 { + for (i, _) in self.rules.iter().enumerate() { + if i == self.rules.len() - 1 { break; } - let other_rules = &self.rules[i+1..]; + let other_rules = &self.rules[i + 1..]; for other_rule in other_rules { if self.rules[i].id == other_rule.id { return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); @@ -281,7 +291,7 @@ impl Lifecycle for BucketLifecycleConfiguration { continue; }*/ //if !obj.delete_marker && !rule.filter.BySize(obj.size) { - if !obj.delete_marker && false{ + if !obj.delete_marker && false { continue; } rules.push(rule.clone()); @@ -306,9 +316,9 @@ impl Lifecycle for BucketLifecycleConfiguration { action = IlmAction::DeleteRestoredVersionAction; } - events.push(Event{ + events.push(Event { action: action, - due: Some(now), + due: Some(now), rule_id: "".into(), noncurrent_days: 0, newer_noncurrent_versions: 0, @@ -322,10 +332,10 @@ impl Lifecycle for BucketLifecycleConfiguration { if obj.expired_object_deletemarker() { if let Some(expiration) = rule.expiration.as_ref() { if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { - events.push(Event{ - action: IlmAction::DeleteVersionAction, + events.push(Event { + action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().expect("err!"), - due: Some(now), + due: Some(now), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -336,12 +346,12 @@ impl Lifecycle for BucketLifecycleConfiguration { if let Some(expiration) = rule.expiration.as_ref() { if let Some(days) = expiration.days { - let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days/*, date*/); + let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/); if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { - events.push(Event{ - action: IlmAction::DeleteVersionAction, + events.push(Event { + action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().expect("err!"), - due: Some(expected_expiry), + due: Some(expected_expiry), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -359,10 +369,10 @@ impl Lifecycle for BucketLifecycleConfiguration { let due = expiration.next_due(obj); if let Some(due) = due { if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() { - events.push(Event{ + events.push(Event { action: IlmAction::DelMarkerDeleteAllVersionsAction, rule_id: rule.id.clone().expect("err!"), - due: Some(due), + due: Some(due), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -392,10 +402,10 @@ impl Lifecycle for BucketLifecycleConfiguration { if let Some(successor_mod_time) = obj.successor_mod_time { let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { - events.push(Event{ - action: IlmAction::DeleteVersionAction, + events.push(Event { + action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().expect("err!"), - due: Some(expected_expiry), + due: Some(expected_expiry), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -413,12 +423,19 @@ impl Lifecycle for BucketLifecycleConfiguration { if storage_class.as_str() != "" { if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE { let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); - if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) { + if due.is_some() + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) + { events.push(Event { - action: IlmAction::TransitionVersionAction, - rule_id: rule.id.clone().expect("err!"), + action: IlmAction::TransitionVersionAction, + rule_id: rule.id.clone().expect("err!"), due, - storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(), + storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0] + .storage_class + .clone() + .unwrap() + .as_str() + .to_string(), ..Default::default() }); } @@ -434,10 +451,10 @@ impl Lifecycle for BucketLifecycleConfiguration { let date0 = OffsetDateTime::from(date.clone()); if date0.unix_timestamp() != 0 { if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() { - events.push(Event{ - action: IlmAction::DeleteAction, + events.push(Event { + action: IlmAction::DeleteAction, rule_id: rule.id.clone().expect("err!"), - due: Some(date0), + due: Some(date0), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -448,10 +465,10 @@ impl Lifecycle for BucketLifecycleConfiguration { if days != 0 { let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days); if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { - let mut event = Event{ - action: IlmAction::DeleteAction, + let mut event = Event { + action: IlmAction::DeleteAction, rule_id: rule.id.clone().expect("err!"), - due: Some(expected_expiry), + due: Some(expected_expiry), noncurrent_days: 0, newer_noncurrent_versions: 0, storage_class: "".into(), @@ -469,10 +486,12 @@ impl Lifecycle for BucketLifecycleConfiguration { if let Some(ref transitions) = rule.transitions { let due = transitions[0].next_due(obj); if let Some(due) = due { - if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) { - events.push(Event{ - action: IlmAction::TransitionAction, - rule_id: rule.id.clone().expect("err!"), + if due.unix_timestamp() > 0 + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) + { + events.push(Event { + action: IlmAction::TransitionAction, + rule_id: rule.id.clone().expect("err!"), due: Some(due), storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(), noncurrent_days: 0, @@ -488,20 +507,27 @@ impl Lifecycle for BucketLifecycleConfiguration { if events.len() > 0 { events.sort_by(|a, b| { - if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() { + if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() + && now.unix_timestamp() > b.due.expect("err").unix_timestamp() + || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() + { match a.action { - IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction - | IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { + IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction + | IlmAction::DeleteVersionAction => { return Ordering::Less; } - _ => () + _ => (), } match b.action { - IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction - | IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { + IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction + | IlmAction::DeleteVersionAction => { return Ordering::Greater; } - _ => () + _ => (), } return Ordering::Less; } @@ -526,18 +552,18 @@ impl Lifecycle for BucketLifecycleConfiguration { continue; } return Event { - action: IlmAction::DeleteVersionAction, - rule_id: rule.id.clone().expect("err"), - noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, newer_noncurrent_versions: newer_noncurrent_versions as usize, due: Some(OffsetDateTime::UNIX_EPOCH), storage_class: "".into(), }; } else { return Event { - action: IlmAction::DeleteVersionAction, - rule_id: rule.id.clone().expect("err"), - noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, newer_noncurrent_versions: 0, due: Some(OffsetDateTime::UNIX_EPOCH), storage_class: "".into(), @@ -601,7 +627,9 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi if days == 0 { return mod_time; } - let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug + let t = mod_time + .to_offset(offset!(-0:00:00)) + .saturating_add(Duration::days(0 /*days as i64*/)); //debug let mut hour = 3600; if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") { if let Ok(num_hour) = env_ilm_hour.parse::() { @@ -661,7 +689,7 @@ impl Default for Event { #[derive(Debug, Clone, Default)] pub struct ExpirationOptions { - pub expire: bool + pub expire: bool, } impl ExpirationOptions { diff --git a/ecstore/src/bucket/lifecycle/mod.rs b/ecstore/src/bucket/lifecycle/mod.rs index 85715016..689f49ed 100644 --- a/ecstore/src/bucket/lifecycle/mod.rs +++ b/ecstore/src/bucket/lifecycle/mod.rs @@ -1,6 +1,6 @@ -pub mod rule; -pub mod lifecycle; -pub mod tier_sweeper; -pub mod tier_last_day_stats; +pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_ops; -pub mod bucket_lifecycle_audit; \ No newline at end of file +pub mod lifecycle; +pub mod rule; +pub mod tier_last_day_stats; +pub mod tier_sweeper; diff --git a/ecstore/src/bucket/lifecycle/rule.rs b/ecstore/src/bucket/lifecycle/rule.rs index d31b35cf..f0c660c8 100644 --- a/ecstore/src/bucket/lifecycle/rule.rs +++ b/ecstore/src/bucket/lifecycle/rule.rs @@ -1,10 +1,9 @@ -use s3s::dto::{ - LifecycleRuleFilter, Transition, -}; +use s3s::dto::{LifecycleRuleFilter, Transition}; -const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; -const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; -const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; +const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; +const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; +const ERR_TRANSITION_INVALID: &str = + "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT"; pub trait Filter { @@ -39,7 +38,6 @@ impl TransitionOps for Transition { } } - #[cfg(test)] mod test { use super::*; diff --git a/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index 5582db69..85755be7 100644 --- a/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -1,9 +1,9 @@ use sha2::Sha256; use std::collections::HashMap; +use std::ops::Sub; use time::OffsetDateTime; use tracing::{error, warn}; -use std::ops::Sub; use crate::heal::data_usage_cache::TierStats; @@ -79,8 +79,5 @@ impl LastDayTierStats { } } - #[cfg(test)] -mod test { - -} +mod test {} diff --git a/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 17237bb1..541ec30e 100644 --- a/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -1,11 +1,11 @@ use sha2::{Digest, Sha256}; -use xxhash_rust::xxh64; use std::any::Any; use std::io::{Cursor, Write}; +use xxhash_rust::xxh64; -use crate::global::GLOBAL_TierConfigMgr; use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject}; use super::lifecycle::{self, ObjectOpts}; +use crate::global::GLOBAL_TierConfigMgr; static XXHASH_SEED: u64 = 0; @@ -44,9 +44,9 @@ impl ObjSweeper { } pub fn get_opts(&self) -> lifecycle::ObjectOpts { - let mut opts = ObjectOpts{ - version_id: self.version_id.clone(), - versioned: self.versioned, + let mut opts = ObjectOpts { + version_id: self.version_id.clone(), + versioned: self.versioned, version_suspended: self.suspended, ..Default::default() }; @@ -69,16 +69,18 @@ impl ObjSweeper { } let mut del_tier = false; - if !self.versioned || self.suspended { // 1, 2.a, 2.b + if !self.versioned || self.suspended { + // 1, 2.a, 2.b del_tier = true; - } else if self.versioned && self.version_id != "" { // 3.a + } else if self.versioned && self.version_id != "" { + // 3.a del_tier = true; } if del_tier { return Some(Jentry { - obj_name: self.remote_object.clone(), + obj_name: self.remote_object.clone(), version_id: self.transition_version_id.clone(), - tier_name: self.transition_tier.clone(), + tier_name: self.transition_tier.clone(), }); } None @@ -123,8 +125,5 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na w.remove(obj_name, rv_id).await } - #[cfg(test)] -mod test { - -} +mod test {} diff --git a/ecstore/src/bucket/mod.rs b/ecstore/src/bucket/mod.rs index e9ce0b7c..bbe4bdaa 100644 --- a/ecstore/src/bucket/mod.rs +++ b/ecstore/src/bucket/mod.rs @@ -1,4 +1,5 @@ pub mod error; +pub mod lifecycle; pub mod metadata; pub mod metadata_sys; pub mod object_lock; @@ -10,4 +11,3 @@ pub mod target; pub mod utils; pub mod versioning; pub mod versioning_sys; -pub mod lifecycle; \ No newline at end of file diff --git a/ecstore/src/bucket/object_lock/objectlock.rs b/ecstore/src/bucket/object_lock/objectlock.rs index 0771d54e..032ad130 100644 --- a/ecstore/src/bucket/object_lock/objectlock.rs +++ b/ecstore/src/bucket/object_lock/objectlock.rs @@ -3,13 +3,8 @@ use std::collections::HashMap; use time::{OffsetDateTime, format_description}; use tracing::{error, warn}; -use s3s::dto::{ - ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold, - Date, -}; -use s3s::header::{ - X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, -}; +use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode}; +use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; //const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; //const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; @@ -17,12 +12,14 @@ use s3s::header::{ //const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold"; const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config"; -const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; -const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; -const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; -const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; -const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; -const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; +const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; +const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; +const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; +const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = + "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; +const ERR_OBJECTLOCK_INVALID_HEADERS: &str = + "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; +const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; pub fn utc_now_ntp() -> OffsetDateTime { return OffsetDateTime::now_utc(); @@ -39,7 +36,10 @@ pub fn get_object_retention_meta(meta: HashMap) -> ObjectLockRet if let Some(mode_str) = mode_str { mode = parse_ret_mode(mode_str.as_str()); } else { - return ObjectLockRetention {mode: None, retain_until_date: None}; + return ObjectLockRetention { + mode: None, + retain_until_date: None, + }; } let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str()); @@ -49,10 +49,13 @@ pub fn get_object_retention_meta(meta: HashMap) -> ObjectLockRet if let Some(till_str) = till_str { let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT); if t.is_err() { - retain_until_date = Date::from(t.expect("err")); //TODO: utc + retain_until_date = Date::from(t.expect("err")); //TODO: utc } } - ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)} + ObjectLockRetention { + mode: Some(mode), + retain_until_date: Some(retain_until_date), + } } pub fn get_object_legalhold_meta(meta: HashMap) -> ObjectLockLegalHold { @@ -61,9 +64,11 @@ pub fn get_object_legalhold_meta(meta: HashMap) -> ObjectLockLeg hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]); } if let Some(hold_str) = hold_str { - return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))}; + return ObjectLockLegalHold { + status: Some(parse_legalhold_status(hold_str)), + }; } - ObjectLockLegalHold {status: None} + ObjectLockLegalHold { status: None } } pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode { @@ -75,7 +80,7 @@ pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode { "COMPLIANCE" => { mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE); } - _ => unreachable!() + _ => unreachable!(), } mode } @@ -89,7 +94,7 @@ pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus { "OFF" => { st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF); } - _ => unreachable!() + _ => unreachable!(), } st } diff --git a/ecstore/src/bucket/object_lock/objectlock_sys.rs b/ecstore/src/bucket/object_lock/objectlock_sys.rs index 59eda77b..5c71a093 100644 --- a/ecstore/src/bucket/object_lock/objectlock_sys.rs +++ b/ecstore/src/bucket/object_lock/objectlock_sys.rs @@ -3,12 +3,10 @@ use std::sync::Arc; use time::OffsetDateTime; use tracing::{error, warn}; -use s3s::dto::{ - DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus, -}; +use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; -use crate::store_api::ObjectInfo; use crate::bucket::metadata_sys::get_object_lock_config; +use crate::store_api::ObjectInfo; use super::objectlock; @@ -21,7 +19,12 @@ impl BucketObjectLockSys { } pub async fn get(bucket: &str) -> Option { - if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule { + if let Some(object_lock_rule) = get_object_lock_config(bucket) + .await + .expect("get_object_lock_config err!") + .0 + .rule + { return object_lock_rule.default_retention; } None @@ -35,10 +38,10 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool { let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err")); match lhold.status { - Some(st) if st.as_str()==ObjectLockLegalHoldStatus::ON => { + Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => { return true; } - _ => () + _ => (), } let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err")); @@ -49,7 +52,7 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool { return true; } } - _ => () + _ => (), } false } diff --git a/ecstore/src/checksum.rs b/ecstore/src/checksum.rs index dad39390..80d53313 100644 --- a/ecstore/src/checksum.rs +++ b/ecstore/src/checksum.rs @@ -1,23 +1,17 @@ #![allow(clippy::map_entry)] -use std::{collections::HashMap, sync::Arc}; -use std::ops::{BitAnd, BitOr}; use lazy_static::lazy_static; +use std::ops::{BitAnd, BitOr}; +use std::{collections::HashMap, sync::Arc}; +use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; +use crate::{disk::DiskAPI, store_api::GetObjectReader}; use reader::hasher::{Hasher, Sha256}; +use rustfs_utils::crypto::{base64_decode, base64_encode}; use s3s::header::{ X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, }; -use crate::client::{ - api_put_object::PutObjectOptions, - api_s3_datatypes::ObjectPart, -}; -use rustfs_utils::crypto::{base64_decode, base64_encode}; -use crate::{ - disk::DiskAPI, - store_api::GetObjectReader, -}; -use enumset::{enum_set, EnumSet, EnumSetType}; +use enumset::{EnumSet, EnumSetType, enum_set}; #[derive(Debug, EnumSetType, Default)] #[enumset(repr = "u8")] @@ -38,8 +32,10 @@ lazy_static! { s.remove(ChecksumMode::ChecksumFullObject); s }; - static ref C_ChecksumFullObjectCRC32: EnumSet = enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); - static ref C_ChecksumFullObjectCRC32C: EnumSet = enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); + static ref C_ChecksumFullObjectCRC32: EnumSet = + enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); + static ref C_ChecksumFullObjectCRC32C: EnumSet = + enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); } const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme"; @@ -49,24 +45,12 @@ impl ChecksumMode { pub fn base(&self) -> ChecksumMode { let s = EnumSet::from(*self).intersection(*C_ChecksumMask); match s.as_u8() { - 1_u8 => { - ChecksumMode::ChecksumNone - } - 2_u8 => { - ChecksumMode::ChecksumSHA256 - } - 4_u8 => { - ChecksumMode::ChecksumSHA1 - } - 8_u8 => { - ChecksumMode::ChecksumCRC32 - } - 16_u8 => { - ChecksumMode::ChecksumCRC32C - } - 32_u8 => { - ChecksumMode::ChecksumCRC64NVME - } + 1_u8 => ChecksumMode::ChecksumNone, + 2_u8 => ChecksumMode::ChecksumSHA256, + 4_u8 => ChecksumMode::ChecksumSHA1, + 8_u8 => ChecksumMode::ChecksumCRC32, + 16_u8 => ChecksumMode::ChecksumCRC32C, + 32_u8 => ChecksumMode::ChecksumCRC64NVME, _ => panic!("enum err."), } } @@ -119,17 +103,13 @@ impl ChecksumMode { let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8(); if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 { 4 - } - else if u == ChecksumMode::ChecksumSHA1 as u8 { - 4//sha1.size - } - else if u == ChecksumMode::ChecksumSHA256 as u8 { - 4//sha256.size - } - else if u == ChecksumMode::ChecksumCRC64NVME as u8 { - 4//crc64.size - } - else { + } else if u == ChecksumMode::ChecksumSHA1 as u8 { + 4 //sha1.size + } else if u == ChecksumMode::ChecksumSHA256 as u8 { + 4 //sha256.size + } else if u == ChecksumMode::ChecksumCRC64NVME as u8 { + 4 //crc64.size + } else { 0 } } @@ -196,7 +176,7 @@ impl ChecksumMode { ChecksumMode::ChecksumCRC64NVME => { return "CRC64NVME".to_string(); } - _=> { + _ => { return "".to_string(); } } @@ -226,10 +206,13 @@ impl ChecksumMode { } }); let c = self.base(); - let mut crc_bytes = Vec::::with_capacity(p.len()*self.raw_byte_len() as usize); + let mut crc_bytes = Vec::::with_capacity(p.len() * self.raw_byte_len() as usize); let mut h = self.hasher()?; h.write(&crc_bytes); - Ok(Checksum {checksum_type: self.clone(), r: h.sum().as_bytes().to_vec()}) + Ok(Checksum { + checksum_type: self.clone(), + r: h.sum().as_bytes().to_vec(), + }) } pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result { @@ -246,7 +229,10 @@ struct Checksum { impl Checksum { fn new(t: ChecksumMode, b: &[u8]) -> Checksum { if t.is_set() && b.len() == t.raw_byte_len() { - return Checksum {checksum_type: t, r: b.to_vec()}; + return Checksum { + checksum_type: t, + r: b.to_vec(), + }; } Checksum::default() } @@ -257,7 +243,7 @@ impl Checksum { Err(err) => return Err(std::io::Error::other(err.to_string())), }; if t.is_set() && b.len() == t.raw_byte_len() { - return Ok(Checksum {checksum_type: t, r: b}); + return Ok(Checksum { checksum_type: t, r: b }); } Ok(Checksum::default()) } @@ -282,28 +268,30 @@ impl Checksum { } pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) { - opts.user_metadata.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); + opts.user_metadata + .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); if opts.auto_checksum.full_object_requested() { - opts.user_metadata.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + opts.user_metadata + .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); } } pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> { if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) { let crc = opts.auto_checksum.composite_checksum(all_parts)?; - opts.user_metadata = { - let mut hm = HashMap::new(); - hm.insert(opts.auto_checksum.key(), crc.encoded()); - hm - } + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key(), crc.encoded()); + hm + } } else if opts.auto_checksum.can_merge_crc() { let crc = opts.auto_checksum.full_object_checksum(all_parts)?; - opts.user_metadata = { - let mut hm = HashMap::new(); - hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded()); - hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); - hm - } + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded()); + hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + hm + } } Ok(()) diff --git a/ecstore/src/client/admin_handler_utils.rs b/ecstore/src/client/admin_handler_utils.rs index 63c2e583..91e047f6 100644 --- a/ecstore/src/client/admin_handler_utils.rs +++ b/ecstore/src/client/admin_handler_utils.rs @@ -30,4 +30,4 @@ impl AdminError { status_code: StatusCode::INTERNAL_SERVER_ERROR, } } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_bucket_policy.rs b/ecstore/src/client/api_bucket_policy.rs index 65b4abf5..0e5b33aa 100644 --- a/ecstore/src/client/api_bucket_policy.rs +++ b/ecstore/src/client/api_bucket_policy.rs @@ -1,11 +1,11 @@ #![allow(clippy::map_entry)] -use std::collections::HashMap; -use http::{HeaderMap, StatusCode}; use bytes::Bytes; +use http::{HeaderMap, StatusCode}; +use std::collections::HashMap; use crate::client::{ api_error_response::http_resp_to_error_response, - transition_api::{RequestMetadata, TransitionClient, ReaderImpl} + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, }; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; @@ -23,9 +23,9 @@ impl TransitionClient { url_values.insert("policy".to_string(), "".to_string()); let mut req_metadata = RequestMetadata { - bucket_name: bucket_name.to_string(), - query_values: url_values, - content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())), + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())), content_length: policy.len() as i64, object_name: "".to_string(), custom_header: HeaderMap::new(), @@ -43,9 +43,9 @@ impl TransitionClient { let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; //defer closeResponse(resp) //if resp != nil { - if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK { - return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); - } + if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } //} Ok(()) } @@ -54,23 +54,28 @@ impl TransitionClient { let mut url_values = HashMap::new(); url_values.insert("policy".to_string(), "".to_string()); - let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - query_values: url_values, - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - object_name: "".to_string(), - custom_header: HeaderMap::new(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; //defer closeResponse(resp) if resp.status() != StatusCode::NO_CONTENT { @@ -89,23 +94,28 @@ impl TransitionClient { let mut url_values = HashMap::new(); url_values.insert("policy".to_string(), "".to_string()); - let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - query_values: url_values, - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - object_name: "".to_string(), - custom_header: HeaderMap::new(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); Ok(policy) diff --git a/ecstore/src/client/api_error_response.rs b/ecstore/src/client/api_error_response.rs index 1c9d74a3..5cc9e301 100644 --- a/ecstore/src/client/api_error_response.rs +++ b/ecstore/src/client/api_error_response.rs @@ -1,26 +1,25 @@ #![allow(clippy::map_entry)] -use std::fmt::Display; use http::StatusCode; -use serde::{Serialize, Deserialize}; -use serde::{ser::Serializer, de::Deserializer}; +use serde::{Deserialize, Serialize}; +use serde::{de::Deserializer, ser::Serializer}; +use std::fmt::Display; -use s3s::S3ErrorCode; use s3s::Body; +use s3s::S3ErrorCode; const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues."; -#[derive(Serialize, Deserialize)] -#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)] #[serde(default, rename_all = "PascalCase")] pub struct ErrorResponse { #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")] - pub code: S3ErrorCode, - pub message: String, + pub code: S3ErrorCode, + pub message: String, pub bucket_name: String, - pub key: String, - pub resource: String, - pub request_id: String, - pub host_id: String, + pub key: String, + pub resource: String, + pub request_id: String, + pub host_id: String, pub region: String, pub server: String, #[serde(skip)] @@ -29,14 +28,14 @@ pub struct ErrorResponse { fn serialize_code(data: &S3ErrorCode, s: S) -> Result where - S: Serializer + S: Serializer, { s.serialize_str("") } fn deserialize_code<'de, D>(d: D) -> Result where - D: Deserializer<'de> + D: Deserializer<'de>, { Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) } @@ -45,14 +44,14 @@ impl Default for ErrorResponse { fn default() -> Self { ErrorResponse { code: S3ErrorCode::Custom("".into()), - message: Default::default(), + message: Default::default(), bucket_name: Default::default(), - key: Default::default(), - resource: Default::default(), - request_id: Default::default(), - host_id: Default::default(), - region: Default::default(), - server: Default::default(), + key: Default::default(), + resource: Default::default(), + request_id: Default::default(), + host_id: Default::default(), + region: Default::default(), + server: Default::default(), status_code: Default::default(), } } @@ -76,7 +75,12 @@ pub fn to_error_response(err: &std::io::Error) -> ErrorResponse { } } -pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucket_name: &str, object_name: &str) -> ErrorResponse { +pub fn http_resp_to_error_response( + resp: http::Response, + b: Vec, + bucket_name: &str, + object_name: &str, +) -> ErrorResponse { let err_body = String::from_utf8(b).unwrap(); //let err_body = xml_decode_and_body(resp.body, &err_resp); let err_resp_ = serde_xml_rs::from_str::(&err_body); @@ -87,18 +91,18 @@ pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucke if object_name == "" { err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::NoSuchBucket, - message: "The specified bucket does not exist.".to_string(), + code: S3ErrorCode::NoSuchBucket, + message: "The specified bucket does not exist.".to_string(), bucket_name: bucket_name.to_string(), ..Default::default() }; } else { err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::NoSuchKey, - message: "The specified key does not exist.".to_string(), + code: S3ErrorCode::NoSuchKey, + message: "The specified key does not exist.".to_string(), bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() }; } @@ -106,18 +110,18 @@ pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucke StatusCode::FORBIDDEN => { err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::AccessDenied, - message: "Access Denied.".to_string(), + code: S3ErrorCode::AccessDenied, + message: "Access Denied.".to_string(), bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() }; } StatusCode::CONFLICT => { err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::BucketNotEmpty, - message: "Bucket not empty.".to_string(), + code: S3ErrorCode::BucketNotEmpty, + message: "Bucket not empty.".to_string(), bucket_name: bucket_name.to_string(), ..Default::default() }; @@ -125,10 +129,10 @@ pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucke StatusCode::PRECONDITION_FAILED => { err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::PreconditionFailed, - message: "Pre condition failed.".to_string(), + code: S3ErrorCode::PreconditionFailed, + message: "Pre condition failed.".to_string(), bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() }; } @@ -137,10 +141,10 @@ pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucke if err_body.len() > 0 { msg = err_body; } - err_resp = ErrorResponse{ + err_resp = ErrorResponse { status_code: resp.status(), - code: S3ErrorCode::Custom(resp.status().to_string().into()), - message: msg, + code: S3ErrorCode::Custom(resp.status().to_string().into()), + message: msg, bucket_name: bucket_name.to_string(), ..Default::default() }; @@ -188,45 +192,55 @@ pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucke pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse { ErrorResponse { status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::InvalidArgument, - message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.".to_string(), + code: S3ErrorCode::InvalidArgument, + message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’." + .to_string(), bucket_name: bucket_name.to_string(), ..Default::default() } } pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { - let msg = format!("Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", total_size, max_object_size); + let msg = format!( + "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", + total_size, max_object_size + ); ErrorResponse { status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::EntityTooLarge, - message: msg, + code: S3ErrorCode::EntityTooLarge, + message: msg, bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() } } pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { - let msg = format!("Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", total_size); + let msg = format!( + "Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", + total_size + ); ErrorResponse { status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::EntityTooSmall, - message: msg, + code: S3ErrorCode::EntityTooSmall, + message: msg, bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() } } pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { - let msg = format!("Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", total_read, total_size); + let msg = format!( + "Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", + total_read, total_size + ); ErrorResponse { status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::Custom("UnexpectedEOF".into()), - message: msg, + code: S3ErrorCode::Custom("UnexpectedEOF".into()), + message: msg, bucket_name: bucket_name.to_string(), - key: object_name.to_string(), + key: object_name.to_string(), ..Default::default() } } @@ -234,9 +248,9 @@ pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, o pub fn err_invalid_argument(message: &str) -> ErrorResponse { ErrorResponse { status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::InvalidArgument, - message: message.to_string(), - request_id: "rustfs".to_string(), + code: S3ErrorCode::InvalidArgument, + message: message.to_string(), + request_id: "rustfs".to_string(), ..Default::default() } } @@ -244,9 +258,9 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse { pub fn err_api_not_supported(message: &str) -> ErrorResponse { ErrorResponse { status_code: StatusCode::NOT_IMPLEMENTED, - code: S3ErrorCode::Custom("APINotSupported".into()), - message: message.to_string(), - request_id: "rustfs".to_string(), + code: S3ErrorCode::Custom("APINotSupported".into()), + message: message.to_string(), + request_id: "rustfs".to_string(), ..Default::default() } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_get_object.rs b/ecstore/src/client/api_get_object.rs index d443866c..efd5d573 100644 --- a/ecstore/src/client/api_get_object.rs +++ b/ecstore/src/client/api_get_object.rs @@ -1,13 +1,13 @@ #![allow(clippy::map_entry)] use bytes::Bytes; use http::HeaderMap; -use tokio::io::BufReader; use std::io::Cursor; +use tokio::io::BufReader; use crate::client::{ - transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, + transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info}, }; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; @@ -16,24 +16,34 @@ impl TransitionClient { todo!(); } - pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { - let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - object_name: object_name.to_string(), - query_values: opts.to_query_values(), - custom_header: opts.header(), - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + pub async fn get_object_inner( + &self, + bucket_name: &str, + object_name: &str, + opts: &GetObjectOptions, + ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: opts.to_query_values(), + custom_header: opts.header(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; let resp = &resp; let object_stat = to_object_info(bucket_name, object_name, resp.headers())?; @@ -45,20 +55,20 @@ impl TransitionClient { #[derive(Default)] struct GetRequest { - pub buffer: Vec, - pub offset: i64, - pub did_offset_change: bool, - pub been_read: bool, - pub is_read_at: bool, - pub is_read_op: bool, - pub is_first_req: bool, + pub buffer: Vec, + pub offset: i64, + pub did_offset_change: bool, + pub been_read: bool, + pub is_read_at: bool, + pub is_read_op: bool, + pub is_first_req: bool, pub setting_object_info: bool, } struct GetResponse { - pub size: i64, + pub size: i64, //pub error: error, - pub did_read: bool, + pub did_read: bool, pub object_info: ObjectInfo, } @@ -79,9 +89,7 @@ struct Object { impl Object { pub fn new() -> Object { - Self { - ..Default::default() - } + Self { ..Default::default() } } fn do_get_request(&self, request: &GetRequest) -> Result { @@ -121,7 +129,7 @@ impl Object { fn stat(&self) -> Result { if !self.is_started || !self.object_info_set { let _ = self.do_get_request(&GetRequest { - is_first_req: !self.is_started, + is_first_req: !self.is_started, setting_object_info: !self.object_info_set, ..Default::default() })?; @@ -134,12 +142,12 @@ impl Object { self.curr_offset = offset; let mut read_at_req = GetRequest { - is_read_op: true, - is_read_at: true, + is_read_op: true, + is_read_at: true, did_offset_change: true, - been_read: self.been_read, + been_read: self.been_read, offset, - buffer: b.to_vec(), + buffer: b.to_vec(), ..Default::default() }; @@ -160,8 +168,8 @@ impl Object { fn seek(&mut self, offset: i64, whence: i64) -> Result { if !self.is_started || !self.object_info_set { let seek_req = GetRequest { - is_read_op: false, - offset: offset, + is_read_op: false, + offset: offset, is_first_req: true, ..Default::default() }; @@ -195,4 +203,4 @@ impl Object { self.is_closed = true; Ok(()) } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_get_options.rs b/ecstore/src/client/api_get_options.rs index 842a02e0..2faab80a 100644 --- a/ecstore/src/client/api_get_options.rs +++ b/ecstore/src/client/api_get_options.rs @@ -1,6 +1,6 @@ #![allow(clippy::map_entry)] -use std::collections::HashMap; use http::{HeaderMap, HeaderName, HeaderValue}; +use std::collections::HashMap; use time::OffsetDateTime; use tracing::warn; @@ -8,9 +8,9 @@ use crate::client::api_error_response::err_invalid_argument; #[derive(Default)] pub struct AdvancedGetOptions { - replication_deletemarker: bool, + replication_deletemarker: bool, is_replication_ready_for_deletemarker: bool, - replication_proxy_request: String, + replication_proxy_request: String, } pub struct GetObjectOptions { @@ -50,7 +50,7 @@ impl GetObjectOptions { } } if self.checksum { - headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err")); + headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err")); } headers } @@ -96,15 +96,15 @@ impl GetObjectOptions { pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> { if start == 0 && end < 0 { self.set("Range", &format!("bytes={}", end)); - } - else if 0 < start && end == 0 { + } else if 0 < start && end == 0 { self.set("Range", &format!("bytes={}-", start)); - } - else if 0 <= start && start <= end { + } else if 0 <= start && start <= end { self.set("Range", &format!("bytes={}-{}", start, end)); - } - else { - return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end)))); + } else { + return Err(std::io::Error::other(err_invalid_argument(&format!( + "Invalid range specified: start={} end={}", + start, end + )))); } Ok(()) } @@ -124,4 +124,4 @@ impl GetObjectOptions { url_values } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_list.rs b/ecstore/src/client/api_list.rs index 2c81bf14..d4c01107 100644 --- a/ecstore/src/client/api_list.rs +++ b/ecstore/src/client/api_list.rs @@ -1,23 +1,36 @@ #![allow(clippy::map_entry)] -use std::collections::HashMap; use bytes::Bytes; use http::{HeaderMap, StatusCode}; +use std::collections::HashMap; use crate::client::{ api_error_response::http_resp_to_error_response, + api_s3_datatypes::{ + ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart, + }, credentials, - api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart}, - transition_api::{ReaderImpl, TransitionClient, RequestMetadata,}, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, }; -use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use crate::store_api::BucketInfo; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; impl TransitionClient { pub fn list_buckets(&self) -> Result, std::io::Error> { todo!(); } - pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result { + pub async fn list_objects_v2_query( + &self, + bucket_name: &str, + object_prefix: &str, + continuation_token: &str, + fetch_owner: bool, + metadata: bool, + delimiter: &str, + start_after: &str, + max_keys: i64, + headers: HeaderMap, + ) -> Result { let mut url_values = HashMap::new(); url_values.insert("list-type".to_string(), "2".to_string()); @@ -43,23 +56,28 @@ impl TransitionClient { url_values.insert("max-keys".to_string(), max_keys.to_string()); } - let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - object_name: "".to_string(), - query_values: url_values, - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - custom_header: headers, - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let mut resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: "".to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; if resp.status() != StatusCode::OK { return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); } @@ -76,12 +94,12 @@ impl TransitionClient { if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" { return Err(std::io::Error::other(credentials::ErrorResponse { - sts_error: credentials::STSError { - r#type: "".to_string(), - code: "NotImplemented".to_string(), - message: "Truncated response should have continuation token set".to_string(), - }, - request_id: "".to_string(), + sts_error: credentials::STSError { + r#type: "".to_string(), + code: "NotImplemented".to_string(), + message: "Truncated response should have continuation token set".to_string(), + }, + request_id: "".to_string(), })); } @@ -97,7 +115,14 @@ impl TransitionClient { Ok(list_bucket_result) } - pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result { + pub fn list_object_versions_query( + &self, + bucket_name: &str, + opts: &ListObjectsOptions, + key_marker: &str, + version_id_marker: &str, + delimiter: &str, + ) -> Result { /*if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListVersionsResult{}, err } @@ -177,15 +202,36 @@ impl TransitionClient { todo!(); } - pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result { + pub fn list_objects_query( + &self, + bucket_name: &str, + object_prefix: &str, + object_marker: &str, + delimiter: &str, + max_keys: i64, + headers: HeaderMap, + ) -> Result { todo!(); } - pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result { + pub fn list_multipart_uploads_query( + &self, + bucket_name: &str, + key_marker: &str, + upload_id_marker: &str, + prefix: &str, + delimiter: &str, + max_uploads: i64, + ) -> Result { todo!(); } - pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result, std::io::Error> { + pub fn list_object_parts( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + ) -> Result, std::io::Error> { todo!(); } @@ -193,7 +239,14 @@ impl TransitionClient { todo!(); } - pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result { + pub async fn list_object_parts_query( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + part_number_marker: i64, + max_parts: i64, + ) -> Result { todo!(); } } @@ -226,4 +279,4 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result Result<(), std::io::Error> { //if self.checksum.is_set() { - /*if !self.trailing_header_support { - return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled"))); - }*/ - /*else if self.override_signer_type == SignatureType::SignatureV2 { - return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures"))); - }*/ + /*if !self.trailing_header_support { + return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled"))); + }*/ + /*else if self.override_signer_type == SignatureType::SignatureV2 { + return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures"))); + }*/ //} Ok(()) @@ -221,19 +235,37 @@ impl PutObjectOptions { } impl TransitionClient { - pub async fn put_object(self: Arc, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64, - opts: &PutObjectOptions + pub async fn put_object( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + object_size: i64, + opts: &PutObjectOptions, ) -> Result { if object_size < 0 && opts.disable_multipart { return Err(std::io::Error::other("object size must be provided with disable multipart upload")); } - self.put_object_common(bucket_name, object_name, reader, object_size, opts).await + self.put_object_common(bucket_name, object_name, reader, object_size, opts) + .await } - pub async fn put_object_common(self: Arc, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result { + pub async fn put_object_common( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { if size > MAX_MULTIPART_PUT_OBJECT_SIZE { - return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name))); + return Err(std::io::Error::other(err_entity_too_large( + size, + MAX_MULTIPART_PUT_OBJECT_SIZE, + bucket_name, + object_name, + ))); } let mut opts = opts.clone(); opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); @@ -255,19 +287,30 @@ impl TransitionClient { return Err(std::io::Error::other("no length provided and multipart disabled")); } if opts.concurrent_stream_parts && opts.num_threads > 1 { - return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await; + return self + .put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts) + .await; } - return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await; + return self + .put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts) + .await; } if size <= part_size || opts.disable_multipart { return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; } - self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await + self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts) + .await } - pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result { + pub async fn put_object_multipart_stream_no_length( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + opts: &PutObjectOptions, + ) -> Result { let mut total_uploaded_size: i64 = 0; let mut compl_multipart_upload = CompleteMultipartUpload::default(); @@ -295,12 +338,8 @@ impl TransitionClient { while part_number <= total_parts_count { buf = match &mut reader { - ReaderImpl::Body(content_body) => { - content_body.to_vec() - } - ReaderImpl::ObjectBody(content_body) => { - content_body.read_all().await? - } + ReaderImpl::Body(content_body) => content_body.to_vec(), + ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?, }; let length = buf.len(); @@ -354,13 +393,13 @@ impl TransitionClient { let part = parts_info[&i].clone(); all_parts.push(part.clone()); compl_multipart_upload.parts.push(CompletePart { - etag: part.etag, - part_num: part.part_num, - checksum_crc32: part.checksum_crc32, - checksum_crc32c: part.checksum_crc32c, - checksum_sha1: part.checksum_sha1, - checksum_sha256: part.checksum_sha256, - checksum_crc64nvme: part.checksum_crc64nvme, + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, ..Default::default() }); } @@ -374,9 +413,11 @@ impl TransitionClient { }; //apply_auto_checksum(&mut opts, all_parts); - let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; upload_info.size = total_uploaded_size; Ok(upload_info) } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_put_object_common.rs b/ecstore/src/client/api_put_object_common.rs index 7f5c8774..ff63aa55 100644 --- a/ecstore/src/client/api_put_object_common.rs +++ b/ecstore/src/client/api_put_object_common.rs @@ -1,10 +1,10 @@ #![allow(clippy::map_entry)] use crate::client::{ - api_put_object::PutObjectOptions, - constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE}, - transition_api::TransitionClient, - transition_api::ReaderImpl, api_error_response::{err_entity_too_large, err_invalid_argument}, + api_put_object::PutObjectOptions, + constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE}, + transition_api::ReaderImpl, + transition_api::TransitionClient, }; const NULL_VERSION_ID: &str = "null"; @@ -28,27 +28,43 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result< } if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { - return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", ""))); + return Err(std::io::Error::other(err_entity_too_large( + object_size, + MAX_MULTIPART_PUT_OBJECT_SIZE, + "", + "", + ))); } let mut part_size_flt: f64; if configured_part_size > 0 { if configured_part_size as i64 > object_size { - return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", ""))); + return Err(std::io::Error::other(err_entity_too_large( + configured_part_size as i64, + object_size, + "", + "", + ))); } if !unknown_size { if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) { - return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize."))); + return Err(std::io::Error::other(err_invalid_argument( + "Part size * max_parts(10000) is lesser than input objectSize.", + ))); } } if (configured_part_size as i64) < ABS_MIN_PART_SIZE { - return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB."))); + return Err(std::io::Error::other(err_invalid_argument( + "Input part size is smaller than allowed minimum of 5MiB.", + ))); } if configured_part_size as i64 > MAX_PART_SIZE { - return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB."))); + return Err(std::io::Error::other(err_invalid_argument( + "Input part size is bigger than allowed maximum of 5GiB.", + ))); } part_size_flt = configured_part_size as f64; @@ -64,13 +80,18 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result< let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64; let part_size = part_size_flt.ceil() as i64; - let last_part_size = object_size - (total_parts_count-1) * part_size; + let last_part_size = object_size - (total_parts_count - 1) * part_size; Ok((total_parts_count, part_size, last_part_size)) } impl TransitionClient { - pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result { + pub async fn new_upload_id( + &self, + bucket_name: &str, + object_name: &str, + opts: &PutObjectOptions, + ) -> Result { let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?; Ok(init_multipart_upload_result.upload_id) } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_put_object_multipart.rs b/ecstore/src/client/api_put_object_multipart.rs index 2c7547cd..3f0a3f0d 100644 --- a/ecstore/src/client/api_put_object_multipart.rs +++ b/ecstore/src/client/api_put_object_multipart.rs @@ -1,50 +1,61 @@ #![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use s3s::S3ErrorCode; use std::io::Read; use std::{collections::HashMap, sync::Arc}; -use bytes::Bytes; -use s3s::S3ErrorCode; -use time::{format_description, OffsetDateTime}; -use uuid::Uuid; -use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; -use url::form_urlencoded::Serializer; +use time::{OffsetDateTime, format_description}; use tokio_util::sync::CancellationToken; use tracing::warn; use tracing::{error, info}; +use url::form_urlencoded::Serializer; +use uuid::Uuid; -use s3s::{dto::StreamingBlob, Body}; -use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; use reader::hasher::Hasher; +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; +use s3s::{Body, dto::StreamingBlob}; //use crate::disk::{Reader, BufferReader}; use crate::client::{ - transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,}, - api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response}, + api_error_response::{ + err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response, + }, api_put_object::PutObjectOptions, api_put_object_common::optimal_part_info, - api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart}, - constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, }, -}; -use rustfs_utils::{ - path::trim_etag, - crypto::base64_encode, + api_s3_datatypes::{ + CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart, + }, + constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE}, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, }; use crate::{ - disk::DiskAPI, - store_api::{ - GetObjectReader, StorageAPI, - }, checksum::ChecksumMode, + disk::DiskAPI, + store_api::{GetObjectReader, StorageAPI}, }; +use rustfs_utils::{crypto::base64_encode, path::trim_etag}; impl TransitionClient { - pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, - opts: &PutObjectOptions + pub async fn put_object_multipart( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, ) -> Result { - let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await; + let info = self + .put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts) + .await; if let Err(err) = &info { let err_resp = to_error_response(err); if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") { if size > MAX_SINGLE_PUT_OBJECT_SIZE { - return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name))); + return Err(std::io::Error::other(err_entity_too_large( + size, + MAX_SINGLE_PUT_OBJECT_SIZE, + bucket_name, + object_name, + ))); } return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await; } @@ -52,7 +63,13 @@ impl TransitionClient { Ok(info?) } - pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result { + pub async fn put_object_multipart_no_stream( + &self, + bucket_name: &str, + object_name: &str, + reader: &mut ReaderImpl, + opts: &PutObjectOptions, + ) -> Result { let mut total_uploaded_size: i64 = 0; let mut compl_multipart_upload = CompleteMultipartUpload::default(); @@ -91,10 +108,10 @@ impl TransitionClient { let mut sha256_hex: String; //if hash_sums["md5"] != nil { - md5_base64 = base64_encode(&hash_sums["md5"]); + md5_base64 = base64_encode(&hash_sums["md5"]); //} //if hash_sums["sha256"] != nil { - sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); + sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); //} if hash_sums.len() == 0 { let csum; @@ -137,12 +154,12 @@ impl TransitionClient { let part = parts_info[&i].clone(); all_parts.push(part.clone()); compl_multipart_upload.parts.push(CompletePart { - etag: part.etag, - part_num: part.part_num, - checksum_crc32: part.checksum_crc32, - checksum_crc32c: part.checksum_crc32c, - checksum_sha1: part.checksum_sha1, - checksum_sha256: part.checksum_sha256, + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, checksum_crc64nvme: part.checksum_crc64nvme, ..Default::default() }); @@ -151,18 +168,25 @@ impl TransitionClient { compl_multipart_upload.parts.sort(); let mut opts = PutObjectOptions { //server_side_encryption: opts.server_side_encryption, - auto_checksum: opts.auto_checksum, + auto_checksum: opts.auto_checksum, ..Default::default() }; //apply_auto_checksum(&mut opts, all_parts); - let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; upload_info.size = total_uploaded_size; Ok(upload_info) } - pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result { + pub async fn initiate_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + opts: &PutObjectOptions, + ) -> Result { let mut url_values = HashMap::new(); url_values.insert("uploads".to_string(), "".to_string()); @@ -180,9 +204,9 @@ impl TransitionClient { let mut req_metadata = RequestMetadata { bucket_name: bucket_name.to_string(), object_name: object_name.to_string(), - query_values: url_values, + query_values: url_values, custom_header, - content_body: ReaderImpl::Body(Bytes::new()), + content_body: ReaderImpl::Body(Bytes::new()), content_length: 0, content_md5_base64: "".to_string(), content_sha256_hex: "".to_string(), @@ -197,9 +221,9 @@ impl TransitionClient { let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; //if resp.is_none() { - if resp.status() != StatusCode::OK { - return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); - } + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } //} let initiate_multipart_upload_result = InitiateMultipartUploadResult::default(); Ok(initiate_multipart_upload_result) @@ -207,13 +231,20 @@ impl TransitionClient { pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result { if p.size > MAX_PART_SIZE { - return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name))); + return Err(std::io::Error::other(err_entity_too_large( + p.size, + MAX_PART_SIZE, + &p.bucket_name, + &p.object_name, + ))); } if p.size <= -1 { return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name))); } if p.part_number <= 0 { - return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero."))); + return Err(std::io::Error::other(err_invalid_argument( + "Part number cannot be negative or equal to zero.", + ))); } if p.upload_id == "" { return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty."))); @@ -224,55 +255,85 @@ impl TransitionClient { url_values.insert("uploadId".to_string(), p.upload_id.clone()); let buf = match &mut p.reader { - ReaderImpl::Body(content_body) => { - content_body.to_vec() - } - ReaderImpl::ObjectBody(content_body) => { - content_body.read_all().await? - } + ReaderImpl::Body(content_body) => content_body.to_vec(), + ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?, }; let mut req_metadata = RequestMetadata { - bucket_name: p.bucket_name.clone(), - object_name: p.object_name.clone(), - query_values: url_values, - custom_header: p.custom_header.clone(), - content_body: ReaderImpl::Body(Bytes::from(buf)), - content_length: p.size, + bucket_name: p.bucket_name.clone(), + object_name: p.object_name.clone(), + query_values: url_values, + custom_header: p.custom_header.clone(), + content_body: ReaderImpl::Body(Bytes::from(buf)), + content_length: p.size, content_md5_base64: p.md5_base64.clone(), content_sha256_hex: p.sha256_hex.clone(), - stream_sha256: p.stream_sha256, - trailer: p.trailer.clone(), - pre_sign_url: Default::default(), + stream_sha256: p.stream_sha256, + trailer: p.trailer.clone(), + pre_sign_url: Default::default(), add_crc: Default::default(), extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), }; let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; //defer closeResponse(resp) //if resp.is_none() { - if resp.status() != StatusCode::OK { - return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name))); - } + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response( + resp, + vec![], + &p.bucket_name.clone(), + &p.object_name, + ))); + } //} let h = resp.headers(); let mut obj_part = ObjectPart { - checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { + h_checksum_crc32.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { + h_checksum_crc32c.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { + h_checksum_sha1.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { + h_checksum_sha256.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { + h_checksum_crc64nvme.to_str().expect("err").to_string() + } else { + "".to_string() + }, ..Default::default() }; obj_part.size = p.size; obj_part.part_num = p.part_number; - obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() }; + obj_part.etag = if let Some(h_etag) = h.get("ETag") { + h_etag.to_str().expect("err").trim_matches('"').to_string() + } else { + "".to_string() + }; Ok(obj_part) } - pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str, - complete: CompleteMultipartUpload, opts: &PutObjectOptions + pub async fn complete_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + complete: CompleteMultipartUpload, + opts: &PutObjectOptions, ) -> Result { let mut url_values = HashMap::new(); url_values.insert("uploadId".to_string(), upload_id.to_string()); @@ -284,19 +345,19 @@ impl TransitionClient { let mut req_metadata = RequestMetadata { bucket_name: bucket_name.to_string(), object_name: object_name.to_string(), - query_values: url_values, - content_body: ReaderImpl::Body(complete_multipart_upload_buffer), - content_length: 100,//complete_multipart_upload_bytes.len(), - content_sha256_hex: "".to_string(),//hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower), - custom_header: headers, - stream_sha256: Default::default(), - trailer: Default::default(), + query_values: url_values, + content_body: ReaderImpl::Body(complete_multipart_upload_buffer), + content_length: 100, //complete_multipart_upload_bytes.len(), + content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower), + custom_header: headers, + stream_sha256: Default::default(), + trailer: Default::default(), content_md5_base64: "".to_string(), - pre_sign_url: Default::default(), + pre_sign_url: Default::default(), add_crc: Default::default(), extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), }; let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; @@ -307,7 +368,7 @@ impl TransitionClient { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { ( OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), - "".to_string() + "".to_string(), ) } else { (OffsetDateTime::now_utc(), "".to_string()) @@ -315,17 +376,21 @@ impl TransitionClient { let h = resp.headers(); Ok(UploadInfo { - bucket: complete_multipart_upload_result.bucket, - key: complete_multipart_upload_result.key, - etag: trim_etag(&complete_multipart_upload_result.etag), - version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, - location: complete_multipart_upload_result.location, - expiration: exp_time, + bucket: complete_multipart_upload_result.bucket, + key: complete_multipart_upload_result.key, + etag: trim_etag(&complete_multipart_upload_result.etag), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { + h_x_amz_version_id.to_str().expect("err").to_string() + } else { + "".to_string() + }, + location: complete_multipart_upload_result.location, + expiration: exp_time, expiration_rule_id: rule_id, - checksum_sha256: complete_multipart_upload_result.checksum_sha256, - checksum_sha1: complete_multipart_upload_result.checksum_sha1, - checksum_crc32: complete_multipart_upload_result.checksum_crc32, - checksum_crc32c: complete_multipart_upload_result.checksum_crc32c, + checksum_sha256: complete_multipart_upload_result.checksum_sha256, + checksum_sha1: complete_multipart_upload_result.checksum_sha1, + checksum_crc32: complete_multipart_upload_result.checksum_crc32, + checksum_crc32c: complete_multipart_upload_result.checksum_crc32c, checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme, ..Default::default() }) diff --git a/ecstore/src/client/api_put_object_streaming.rs b/ecstore/src/client/api_put_object_streaming.rs index dd12f9bb..ce10bbca 100644 --- a/ecstore/src/client/api_put_object_streaming.rs +++ b/ecstore/src/client/api_put_object_streaming.rs @@ -1,28 +1,28 @@ #![allow(clippy::map_entry)] -use std::sync::RwLock; -use std::{collections::HashMap, sync::Arc}; use bytes::Bytes; use futures::future::join_all; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; -use time::{format_description, OffsetDateTime}; +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc}; +use time::{OffsetDateTime, format_description}; use tokio::{select, sync::mpsc}; use tokio_util::sync::CancellationToken; use tracing::warn; use uuid::Uuid; -use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; -use reader::hasher::Hasher; +use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum}; use crate::client::{ - constants::ISO8601_DATEFORMAT, + api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response}, api_put_object::PutObjectOptions, + api_put_object_common::{is_object, optimal_part_info}, api_put_object_multipart::UploadPartParams, api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, - transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl}, - api_put_object_common::{is_object, optimal_part_info,}, - api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof}, + constants::ISO8601_DATEFORMAT, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, }; +use reader::hasher::Hasher; use rustfs_utils::{crypto::base64_encode, path::trim_etag}; -use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode}; +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; pub struct UploadedPartRes { pub error: std::io::Error, @@ -37,23 +37,39 @@ pub struct UploadPartReq { } impl TransitionClient { - pub async fn put_object_multipart_stream(self: Arc, bucket_name: &str, object_name: &str, - mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + pub async fn put_object_multipart_stream( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, ) -> Result { let info: UploadInfo; if opts.concurrent_stream_parts && opts.num_threads > 1 { - info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?; + info = self + .put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts) + .await?; } else if !is_object(&reader) && !opts.send_content_md5 { - info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?; + info = self + .put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts) + .await?; } else { - info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?; + info = self + .put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts) + .await?; } - + Ok(info) } - pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str, - mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + pub async fn put_object_multipart_stream_from_readat( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, ) -> Result { let ret = optimal_part_info(size, opts.part_size)?; let (total_parts_count, part_size, lastpart_size) = ret; @@ -68,8 +84,13 @@ impl TransitionClient { todo!(); } - pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str, - mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + pub async fn put_object_multipart_stream_optional_checksum( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, ) -> Result { let mut opts = opts.clone(); if opts.checksum.is_set() { @@ -130,7 +151,7 @@ impl TransitionClient { } } - let hooked = ReaderImpl::Body(Bytes::from(buf));//newHook(BufferReader::new(buf), opts.progress); + let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress); let mut p = UploadPartParams { bucket_name: bucket_name.to_string(), object_name: object_name.to_string(), @@ -154,7 +175,12 @@ impl TransitionClient { if size > 0 { if total_uploaded_size != size { - return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name))); + return Err(std::io::Error::other(err_unexpected_eof( + total_uploaded_size, + size, + bucket_name, + object_name, + ))); } } @@ -167,12 +193,12 @@ impl TransitionClient { all_parts.push(part.clone()); compl_multipart_upload.parts.push(CompletePart { - etag: part.etag, - part_num: part.part_num, - checksum_crc32: part.checksum_crc32, - checksum_crc32c: part.checksum_crc32c, - checksum_sha1: part.checksum_sha1, - checksum_sha256: part.checksum_sha256, + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, checksum_crc64nvme: part.checksum_crc64nvme, }); } @@ -181,18 +207,24 @@ impl TransitionClient { let mut opts = PutObjectOptions { //server_side_encryption: opts.server_side_encryption, - auto_checksum: opts.auto_checksum, + auto_checksum: opts.auto_checksum, ..Default::default() }; apply_auto_checksum(&mut opts, &mut all_parts); - let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; upload_info.size = total_uploaded_size; Ok(upload_info) } - pub async fn put_object_multipart_stream_parallel(self: Arc, bucket_name: &str, object_name: &str, - mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions + pub async fn put_object_multipart_stream_parallel( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, /*GetObjectReader*/ + opts: &PutObjectOptions, ) -> Result { let mut opts = opts.clone(); if opts.checksum.is_set() { @@ -239,7 +271,11 @@ impl TransitionClient { } if buf.len() != part_size as usize { - return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size))); + return Err(std::io::Error::other(format!( + "read buffer < {} than expected partSize: {}", + buf.len(), + part_size + ))); } match &mut reader { @@ -286,21 +322,21 @@ impl TransitionClient { //defer wg.Done() let mut p = UploadPartParams { - bucket_name: bucket_name.to_string(), - object_name: object_name.to_string(), + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), upload_id: clone_upload_id, - reader: ReaderImpl::Body(Bytes::from(buf.clone())), + reader: ReaderImpl::Body(Bytes::from(buf.clone())), part_number, md5_base64, - size: length as i64, + size: length as i64, //sse: opts.server_side_encryption, stream_sha256: !opts.disable_content_sha256, custom_header, - sha256_hex: "".to_string(), - trailer: HeaderMap::new(), + sha256_hex: "".to_string(), + trailer: HeaderMap::new(), }; let obj_part = clone_self.upload_part(&mut p).await.expect("err"); - + let mut clone_parts_info = clone_parts_info.write().unwrap(); clone_parts_info.entry(part_number).or_insert(obj_part); @@ -328,12 +364,12 @@ impl TransitionClient { all_parts.push(part.clone()); compl_multipart_upload.parts.push(CompletePart { - etag: part.etag, - part_num: part.part_num, - checksum_crc32: part.checksum_crc32, - checksum_crc32c: part.checksum_crc32c, - checksum_sha1: part.checksum_sha1, - checksum_sha256: part.checksum_sha256, + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, checksum_crc64nvme: part.checksum_crc64nvme, ..Default::default() }); @@ -343,41 +379,60 @@ impl TransitionClient { let mut opts = PutObjectOptions { //server_side_encryption: opts.server_side_encryption, - auto_checksum: opts.auto_checksum, + auto_checksum: opts.auto_checksum, ..Default::default() }; apply_auto_checksum(&mut opts, &mut all_parts); - let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; upload_info.size = total_uploaded_size; Ok(upload_info) } - pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result { + pub async fn put_object_gcs( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { let mut opts = opts.clone(); if opts.checksum.is_set() { opts.send_content_md5 = false; } let mut md5_base64: String = "".to_string(); - let progress_reader = reader;//newHook(reader, opts.progress); + let progress_reader = reader; //newHook(reader, opts.progress); - self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await + self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts) + .await } - pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result { + pub async fn put_object_do( + &self, + bucket_name: &str, + object_name: &str, + reader: ReaderImpl, + md5_base64: &str, + sha256_hex: &str, + size: i64, + opts: &PutObjectOptions, + ) -> Result { let custom_header = opts.header(); let mut req_metadata = RequestMetadata { - bucket_name: bucket_name.to_string(), - object_name: object_name.to_string(), + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), custom_header, - content_body: reader, - content_length: size, + content_body: reader, + content_length: size, content_md5_base64: md5_base64.to_string(), content_sha256_hex: sha256_hex.to_string(), - stream_sha256: !opts.disable_content_sha256, + stream_sha256: !opts.disable_content_sha256, add_crc: Default::default(), bucket_location: Default::default(), pre_sign_url: Default::default(), @@ -386,7 +441,7 @@ impl TransitionClient { expires: Default::default(), trailer: Default::default(), }; - let mut add_crc = false;//self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure); + let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure); let mut opts = opts.clone(); if opts.checksum.is_set() { req_metadata.add_crc = opts.checksum; @@ -422,26 +477,50 @@ impl TransitionClient { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { ( OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), - "".to_string() + "".to_string(), ) } else { (OffsetDateTime::now_utc(), "".to_string()) }; let h = resp.headers(); Ok(UploadInfo { - bucket: bucket_name.to_string(), - key: object_name.to_string(), - etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), - version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, - size: size, - expiration: exp_time, + bucket: bucket_name.to_string(), + key: object_name.to_string(), + etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { + h_x_amz_version_id.to_str().expect("err").to_string() + } else { + "".to_string() + }, + size: size, + expiration: exp_time, expiration_rule_id: rule_id, - checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, - checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { + h_checksum_crc32.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { + h_checksum_crc32c.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { + h_checksum_sha1.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { + h_checksum_sha256.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { + h_checksum_crc64nvme.to_str().expect("err").to_string() + } else { + "".to_string() + }, ..Default::default() }) } -} \ No newline at end of file +} diff --git a/ecstore/src/client/api_remove.rs b/ecstore/src/client/api_remove.rs index 3f37ee8f..9ca5bc51 100644 --- a/ecstore/src/client/api_remove.rs +++ b/ecstore/src/client/api_remove.rs @@ -1,26 +1,24 @@ #![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, HeaderValue, Method, StatusCode}; +use s3s::S3ErrorCode; +use s3s::dto::ReplicationStatus; +use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION; use std::fmt::Display; use std::{collections::HashMap, sync::Arc}; -use bytes::Bytes; -use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION; -use s3s::S3ErrorCode; use time::OffsetDateTime; -use s3s::dto::ReplicationStatus; use tokio::sync::mpsc::{self, Receiver, Sender}; -use http::{HeaderMap, HeaderValue, Method, StatusCode}; -use reader::hasher::{sum_sha256_hex, sum_md5_base64}; -use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use crate::client::{ + api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response}, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, +}; use crate::{ disk::DiskAPI, - store_api::{ - GetObjectReader, ObjectInfo, StorageAPI, - }, -}; -use crate::client::{ - transition_api::{TransitionClient, RequestMetadata, ReaderImpl}, - api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,}, + store_api::{GetObjectReader, ObjectInfo, StorageAPI}, }; +use reader::hasher::{sum_md5_base64, sum_sha256_hex}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; struct RemoveBucketOptions { forced_elete: bool, @@ -62,23 +60,28 @@ impl TransitionClient { headers.insert(rustFSForceDelete, "true"); }*/ - let resp = self.execute_method(Method::DELETE, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - custom_header: headers, - object_name: "".to_string(), - query_values: Default::default(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; { let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); @@ -88,23 +91,28 @@ impl TransitionClient { } pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { - let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - custom_header: Default::default(), - object_name: "".to_string(), - query_values: Default::default(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: Default::default(), + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; { let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); @@ -119,7 +127,12 @@ impl TransitionClient { res.err } - pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result { + pub async fn remove_object_inner( + &self, + bucket_name: &str, + object_name: &str, + opts: RemoveObjectOptions, + ) -> Result { let mut url_values = HashMap::new(); if opts.version_id != "" { @@ -129,49 +142,72 @@ impl TransitionClient { let mut headers = HeaderMap::new(); if opts.governance_bypass { - headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));//amzBypassGovernance + headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance } - let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - object_name: object_name.to_string(), - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - query_values: url_values, - custom_header: headers, - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + query_values: url_values, + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; Ok(RemoveObjectResult { object_name: object_name.to_string(), - object_version_id: opts.version_id, - delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", - delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(), + object_version_id: opts.version_id, + delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", + delete_marker_version_id: resp + .headers() + .get("x-amz-version-id") + .expect("err") + .to_str() + .expect("err") + .to_string(), ..Default::default() }) } - pub async fn remove_objects_with_result(self: Arc, bucket_name: &str, objects_rx: Receiver, opts: RemoveObjectsOptions) -> Receiver { + pub async fn remove_objects_with_result( + self: Arc, + bucket_name: &str, + objects_rx: Receiver, + opts: RemoveObjectsOptions, + ) -> Receiver { let (result_tx, mut result_rx) = mpsc::channel(1); let self_clone = Arc::clone(&self); let bucket_name_owned = bucket_name.to_string(); tokio::spawn(async move { - self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; + self_clone + .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts) + .await; }); result_rx } - pub async fn remove_objects(self: Arc, bucket_name: &str, objects_rx: Receiver, opts: RemoveObjectsOptions) -> Receiver { + pub async fn remove_objects( + self: Arc, + bucket_name: &str, + objects_rx: Receiver, + opts: RemoveObjectsOptions, + ) -> Receiver { let (error_tx, mut error_rx) = mpsc::channel(1); let self_clone = Arc::clone(&self); @@ -179,26 +215,36 @@ impl TransitionClient { let (result_tx, mut result_rx) = mpsc::channel(1); tokio::spawn(async move { - self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; + self_clone + .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts) + .await; }); tokio::spawn(async move { while let Some(res) = result_rx.recv().await { if res.err.is_none() { continue; } - error_tx.send(RemoveObjectError { - object_name: res.object_name, - version_id: res.object_version_id, - err: res.err, - ..Default::default() - }).await; + error_tx + .send(RemoveObjectError { + object_name: res.object_name, + version_id: res.object_version_id, + err: res.err, + ..Default::default() + }) + .await; } }); error_rx } - pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver, result_tx: &Sender, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> { + pub async fn remove_objects_inner( + &self, + bucket_name: &str, + mut objects_rx: Receiver, + result_tx: &Sender, + opts: RemoveObjectsOptions, + ) -> Result<(), std::io::Error> { let max_entries = 1000; let mut finish = false; let mut url_values = HashMap::new(); @@ -213,11 +259,17 @@ impl TransitionClient { while let Some(object) = objects_rx.recv().await { if has_invalid_xml_char(&object.name) { - let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions { - version_id: object.version_id.expect("err").to_string(), - governance_bypass: opts.governance_bypass, - ..Default::default() - }).await?; + let remove_result = self + .remove_object_inner( + bucket_name, + &object.name, + RemoveObjectOptions { + version_id: object.version_id.expect("err").to_string(), + governance_bypass: opts.governance_bypass, + ..Default::default() + }, + ) + .await?; let remove_result_clone = remove_result.clone(); if !remove_result.err.is_none() { match to_error_response(&remove_result.err.expect("err")).code { @@ -241,7 +293,7 @@ impl TransitionClient { } if count == 0 { break; - } + } if count < max_entries { finish = true; } @@ -252,23 +304,28 @@ impl TransitionClient { } let remove_bytes = generate_remove_multi_objects_request(&batch); - let resp = self.execute_method(http::Method::POST, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - query_values: url_values.clone(), - content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), - content_length: remove_bytes.len() as i64, - content_md5_base64: sum_md5_base64(&remove_bytes), - content_sha256_hex: sum_sha256_hex(&remove_bytes), - custom_header: headers, - object_name: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::POST, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values.clone(), + content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), + content_length: remove_bytes.len() as i64, + content_md5_base64: sum_md5_base64(&remove_bytes), + content_sha256_hex: sum_sha256_hex(&remove_bytes), + custom_header: headers, + object_name: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; let body_bytes: Vec = resp.body().bytes().expect("err").to_vec(); process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone()); @@ -285,49 +342,77 @@ impl TransitionClient { Ok(()) } - pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> { + pub async fn abort_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + ) -> Result<(), std::io::Error> { let mut url_values = HashMap::new(); url_values.insert("uploadId".to_string(), upload_id.to_string()); - let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { - bucket_name: bucket_name.to_string(), - object_name: object_name.to_string(), - query_values: url_values, - content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), - custom_header: HeaderMap::new(), - content_body: ReaderImpl::Body(Bytes::new()), - content_length: 0, - content_md5_base64: "".to_string(), - stream_sha256: false, - trailer: HeaderMap::new(), - pre_sign_url: Default::default(), - add_crc: Default::default(), - extra_pre_sign_header: Default::default(), - bucket_location: Default::default(), - expires: Default::default(), - }).await?; + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; //if resp.is_some() { - if resp.status() != StatusCode::NO_CONTENT { - let error_response: ErrorResponse; - match resp.status() { - StatusCode::NOT_FOUND => { - error_response = ErrorResponse { - code: S3ErrorCode::NoSuchUpload, - message: "The specified multipart upload does not exist.".to_string(), - bucket_name: bucket_name.to_string(), - key: object_name.to_string(), - request_id: resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(), - host_id: resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(), - region: resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(), - ..Default::default() - }; - } - _ => { - return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); - } + if resp.status() != StatusCode::NO_CONTENT { + let error_response: ErrorResponse; + match resp.status() { + StatusCode::NOT_FOUND => { + error_response = ErrorResponse { + code: S3ErrorCode::NoSuchUpload, + message: "The specified multipart upload does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + request_id: resp + .headers() + .get("x-amz-request-id") + .expect("err") + .to_str() + .expect("err") + .to_string(), + host_id: resp + .headers() + .get("x-amz-id-2") + .expect("err") + .to_str() + .expect("err") + .to_string(), + region: resp + .headers() + .get("x-amz-bucket-region") + .expect("err") + .to_str() + .expect("err") + .to_string(), + ..Default::default() + }; + } + _ => { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); } - return Err(std::io::Error::other(error_response)); } + return Err(std::io::Error::other(error_response)); + } //} Ok(()) } @@ -336,8 +421,8 @@ impl TransitionClient { #[derive(Debug, Default)] struct RemoveObjectError { object_name: String, - version_id: String, - err: Option, + version_id: String, + err: Option, } impl Display for RemoveObjectError { @@ -367,7 +452,7 @@ impl Clone for RemoveObjectResult { delete_marker_version_id: self.delete_marker_version_id.clone(), err: None, //err } - } + } } pub struct RemoveObjectsOptions { diff --git a/ecstore/src/client/api_s3_datatypes.rs b/ecstore/src/client/api_s3_datatypes.rs index 592f3695..7276f863 100644 --- a/ecstore/src/client/api_s3_datatypes.rs +++ b/ecstore/src/client/api_s3_datatypes.rs @@ -1,12 +1,12 @@ #![allow(clippy::map_entry)] -use std::collections::HashMap; use s3s::dto::Owner; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use time::OffsetDateTime; -use rustfs_utils::crypto::base64_decode; use crate::checksum::ChecksumMode; use crate::client::transition_api::ObjectMultipartInfo; +use rustfs_utils::crypto::base64_decode; use super::transition_api; @@ -31,15 +31,15 @@ pub struct ListBucketV2Result { pub name: String, pub next_continuation_token: String, pub continuation_token: String, - pub prefix: String, + pub prefix: String, pub fetch_owner: String, pub start_after: String, } pub struct Version { - etag: String, - is_latest: bool, - key: String, + etag: String, + is_latest: bool, + key: String, last_modified: OffsetDateTime, owner: Owner, size: i64, @@ -53,15 +53,15 @@ pub struct Version { pub struct ListVersionsResult { versions: Vec, common_prefixes: Vec, - name: String, - prefix: String, - delimiter: String, - max_keys: i64, - encoding_type: String, - is_truncated: bool, - key_marker: String, - version_id_marker: String, - next_key_marker: String, + name: String, + prefix: String, + delimiter: String, + max_keys: i64, + encoding_type: String, + is_truncated: bool, + key_marker: String, + version_id_marker: String, + next_key_marker: String, next_version_id_marker: String, } @@ -77,35 +77,35 @@ pub struct ListBucketResult { delimiter: String, encoding_type: String, is_truncated: bool, - marker: String, - max_keys: i64, - name: String, - next_marker: String, - prefix: String, + marker: String, + max_keys: i64, + name: String, + next_marker: String, + prefix: String, } pub struct ListMultipartUploadsResult { - bucket: String, - key_marker: String, - upload_id_marker: String, - next_key_marker: String, + bucket: String, + key_marker: String, + upload_id_marker: String, + next_key_marker: String, next_upload_id_marker: String, - encoding_type: String, - max_uploads: i64, - is_truncated: bool, - uploads: Vec, - prefix: String, - delimiter: String, - common_prefixes: Vec, + encoding_type: String, + max_uploads: i64, + is_truncated: bool, + uploads: Vec, + prefix: String, + delimiter: String, + common_prefixes: Vec, } pub struct Initiator { - id: String, + id: String, display_name: String, } pub struct CopyObjectResult { - pub etag: String, + pub etag: String, pub last_modified: OffsetDateTime, } @@ -115,10 +115,10 @@ pub struct ObjectPart { pub part_num: i64, pub last_modified: OffsetDateTime, pub size: i64, - pub checksum_crc32: String, - pub checksum_crc32c: String, - pub checksum_sha1: String, - pub checksum_sha256: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, pub checksum_crc64nvme: String, } @@ -129,14 +129,13 @@ impl Default for ObjectPart { part_num: 0, last_modified: OffsetDateTime::now_utc(), size: 0, - checksum_crc32: Default::default(), - checksum_crc32c: Default::default(), - checksum_sha1: Default::default(), - checksum_sha256: Default::default(), + checksum_crc32: Default::default(), + checksum_crc32c: Default::default(), + checksum_sha1: Default::default(), + checksum_sha256: Default::default(), checksum_crc64nvme: Default::default(), } } - } impl ObjectPart { @@ -216,15 +215,15 @@ pub struct CompleteMultipartUploadResult { pub checksum_crc64nvme: String, } -#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] -#[derive(serde::Serialize)] -pub struct CompletePart { //api has +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] +pub struct CompletePart { + //api has pub etag: String, pub part_num: i64, - pub checksum_crc32: String, - pub checksum_crc32c: String, - pub checksum_sha1: String, - pub checksum_sha256: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, pub checksum_crc64nvme: String, } @@ -258,8 +257,7 @@ pub struct CopyObjectPartResult { pub last_modified: OffsetDateTime, } -#[derive(Debug, Default)] -#[derive(serde::Serialize)] +#[derive(Debug, Default, serde::Serialize)] pub struct CompleteMultipartUpload { pub parts: Vec, } @@ -287,12 +285,14 @@ pub struct CreateBucketConfiguration { } #[derive(serde::Serialize)] -pub struct DeleteObject { //api has +pub struct DeleteObject { + //api has pub key: String, pub version_id: String, } -pub struct DeletedObject { //s3s has +pub struct DeletedObject { + //s3s has pub key: String, pub version_id: String, pub deletemarker: bool, @@ -300,15 +300,15 @@ pub struct DeletedObject { //s3s has } pub struct NonDeletedObject { - pub key: String, - pub code: String, - pub message: String, + pub key: String, + pub code: String, + pub message: String, pub version_id: String, } #[derive(serde::Serialize)] pub struct DeleteMultiObjects { - pub quiet: bool, + pub quiet: bool, pub objects: Vec, } @@ -331,6 +331,6 @@ impl DeleteMultiObjects { } pub struct DeleteMultiObjectsResult { - pub deleted_objects: Vec, + pub deleted_objects: Vec, pub undeleted_objects: Vec, } diff --git a/ecstore/src/client/bucket_cache.rs b/ecstore/src/client/bucket_cache.rs index 00f8c22d..6216cd5e 100644 --- a/ecstore/src/client/bucket_cache.rs +++ b/ecstore/src/client/bucket_cache.rs @@ -1,20 +1,20 @@ #![allow(clippy::map_entry)] use http::Request; +use hyper::StatusCode; use hyper::body::Incoming; use std::{collections::HashMap, sync::Arc}; use tracing::warn; -use tracing::{error, info, debug}; -use hyper::StatusCode; +use tracing::{debug, error, info}; -use reader::hasher::{Hasher, Sha256}; -use s3s::S3ErrorCode; -use s3s::Body; use crate::client::{ api_error_response::{http_resp_to_error_response, to_error_response}, - transition_api::{TransitionClient, Document}, + transition_api::{Document, TransitionClient}, }; use crate::signer; +use reader::hasher::{Hasher, Sha256}; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use s3s::Body; +use s3s::S3ErrorCode; use super::constants::UNSIGNED_PAYLOAD; use super::credentials::SignatureType; @@ -25,9 +25,7 @@ pub struct BucketLocationCache { impl BucketLocationCache { pub fn new() -> BucketLocationCache { - BucketLocationCache{ - items: HashMap::new(), - } + BucketLocationCache { items: HashMap::new() } } pub fn get(&self, bucket_name: &str) -> Option { @@ -50,7 +48,7 @@ impl TransitionClient { async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result { if self.region != "" { - return Ok(self.region.clone()) + return Ok(self.region.clone()); } let mut location; @@ -81,11 +79,7 @@ impl TransitionClient { let mut target_url = self.endpoint_url.clone(); let scheme = self.endpoint_url.scheme(); let h = target_url.host().expect("host is none."); - let default_port = if scheme == "https" { - 443 - } else { - 80 - }; + let default_port = if scheme == "https" { 443 } else { 80 }; let p = target_url.port().unwrap_or(default_port); let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name); @@ -128,9 +122,9 @@ impl TransitionClient { } let mut signer_type = value.signer_type.clone(); - let mut access_key_id = value.access_key_id; - let mut secret_access_key = value.secret_access_key; - let mut session_token = value.session_token; + let mut access_key_id = value.access_key_id; + let mut secret_access_key = value.secret_access_key; + let mut session_token = value.session_token; if self.override_signer_type != SignatureType::SignatureDefault { signer_type = self.override_signer_type.clone(); @@ -164,7 +158,10 @@ impl TransitionClient { content_sha256 = UNSIGNED_PAYLOAD.to_string(); } - req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); + req_builder + .headers_mut() + .expect("err") + .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); let req = match req_builder.body(Body::empty()) { Ok(req) => return Ok(req), @@ -177,9 +174,9 @@ impl TransitionClient { async fn process_bucket_location_response(mut resp: http::Response, bucket_name: &str) -> Result { //if resp != nil { - if resp.status() != StatusCode::OK { - let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, ""); - match err_resp.code { + if resp.status() != StatusCode::OK { + let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, ""); + match err_resp.code { S3ErrorCode::NotImplemented => { match err_resp.server.as_str() { "AmazonSnowball" => { @@ -205,13 +202,13 @@ async fn process_bucket_location_response(mut resp: http::Response, bucket return Err(std::io::Error::other(err_resp)); } } - } + } //} let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); let Document(location_constraint) = serde_xml_rs::from_str::(&String::from_utf8(b).unwrap()).unwrap(); - let mut location = location_constraint; + let mut location = location_constraint; if location == "" { location = "us-east-1".to_string(); } @@ -221,4 +218,4 @@ async fn process_bucket_location_response(mut resp: http::Response, bucket } Ok(location) -} \ No newline at end of file +} diff --git a/ecstore/src/client/constants.rs b/ecstore/src/client/constants.rs index 505041e1..42ee4c15 100644 --- a/ecstore/src/client/constants.rs +++ b/ecstore/src/client/constants.rs @@ -1,7 +1,7 @@ #![allow(clippy::map_entry)] -use std::{collections::HashMap, sync::Arc}; -use time::{macros::format_description, format_description::FormatItem}; use lazy_static::lazy_static; +use std::{collections::HashMap, sync::Arc}; +use time::{format_description::FormatItem, macros::format_description}; pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5; pub const MAX_PARTS_COUNT: i64 = 10000; @@ -16,8 +16,9 @@ pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; pub const TOTAL_WORKERS: i64 = 4; -pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; -pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); +pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = + format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts"; const GetObjectAttributesMaxParts: i64 = 1000; diff --git a/ecstore/src/client/credentials.rs b/ecstore/src/client/credentials.rs index 8ccf8e01..f4a13513 100644 --- a/ecstore/src/client/credentials.rs +++ b/ecstore/src/client/credentials.rs @@ -19,8 +19,7 @@ pub struct Credentials { provider: P, } -impl Credentials

-{ +impl Credentials

{ pub fn new(provider: P) -> Self { Self { provider: provider, @@ -109,8 +108,8 @@ impl Provider for Static { #[derive(Debug, Clone, Default)] pub struct STSError { - pub r#type: String, - pub code: String, + pub r#type: String, + pub code: String, pub message: String, } @@ -128,21 +127,21 @@ impl Display for ErrorResponse { impl ErrorResponse { fn error(&self) -> String { - if self.sts_error.message == "" { - return format!("Error response code {}.", self.sts_error.code); - } - return self.sts_error.message.clone(); + if self.sts_error.message == "" { + return format!("Error response code {}.", self.sts_error.code); + } + return self.sts_error.message.clone(); } } struct Error { - code: String, - message: String, + code: String, + message: String, bucket_name: String, - key: String, - resource: String, - request_id: String, - host_id: String, + key: String, + resource: String, + request_id: String, + host_id: String, region: String, server: String, status_code: i64, diff --git a/ecstore/src/client/mod.rs b/ecstore/src/client/mod.rs index 2a060af8..e8a45e8e 100644 --- a/ecstore/src/client/mod.rs +++ b/ecstore/src/client/mod.rs @@ -1,18 +1,18 @@ -pub mod constants; -pub mod transition_api; -pub mod api_list; -pub mod api_error_response; -pub mod api_s3_datatypes; +pub mod admin_handler_utils; pub mod api_bucket_policy; -pub mod api_put_object_common; -pub mod api_get_options; +pub mod api_error_response; pub mod api_get_object; +pub mod api_get_options; +pub mod api_list; pub mod api_put_object; -pub mod api_put_object_streaming; +pub mod api_put_object_common; pub mod api_put_object_multipart; +pub mod api_put_object_streaming; pub mod api_remove; +pub mod api_s3_datatypes; +pub mod bucket_cache; +pub mod constants; +pub mod credentials; pub mod object_api_utils; pub mod object_handlers_common; -pub mod admin_handler_utils; -pub mod credentials; -pub mod bucket_cache; \ No newline at end of file +pub mod transition_api; diff --git a/ecstore/src/client/object_api_utils.rs b/ecstore/src/client/object_api_utils.rs index 494969aa..e824968b 100644 --- a/ecstore/src/client/object_api_utils.rs +++ b/ecstore/src/client/object_api_utils.rs @@ -1,17 +1,14 @@ #![allow(clippy::map_entry)] -use std::{collections::HashMap, sync::Arc}; use http::HeaderMap; -use tokio::io::BufReader; use std::io::Cursor; +use std::{collections::HashMap, sync::Arc}; +use tokio::io::BufReader; -use s3s::S3ErrorCode; -use crate::store_api::{ - GetObjectReader, HTTPRangeSpec, - ObjectInfo, ObjectOptions, -}; +use crate::error::ErrorResponse; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions}; use rustfs_filemeta::fileinfo::ObjectPartInfo; use rustfs_rio::HashReader; -use crate::error::ErrorResponse; +use s3s::S3ErrorCode; //#[derive(Clone)] pub struct PutObjReader { @@ -57,7 +54,11 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option (i64, i64, i64, i64, u64) { @@ -71,7 +72,7 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u for (i, part) in oi.parts.iter().enumerate() { cumulative_actual_size += part.actual_size as i64; if cumulative_actual_size <= offset { - compressed_offset += part.size as i64; + compressed_offset += part.size as i64; } else { first_part_idx = i as i64; skip_length = cumulative_actual_size - part.actual_size as i64; @@ -81,23 +82,32 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u skip_length = offset - skip_length; let parts: &[ObjectPartInfo] = &oi.parts; - if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 { + if skip_length > 0 + && parts.len() > first_part_idx as usize + && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 + { todo!(); } (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) } -pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { +pub fn new_getobjectreader( + rs: HTTPRangeSpec, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, +) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?; let mut is_encrypted = false; - let is_compressed = false;//oi.is_compressed_ok(); + let is_compressed = false; //oi.is_compressed_ok(); let mut get_fn: ObjReaderFn; - let (off, length) = match rs.get_offset_length(oi.size) { - Ok(x) => x, - Err(err) => return Err(ErrorResponse { + let (off, length) = match rs.get_offset_length(oi.size) { + Ok(x) => x, + Err(err) => { + return Err(ErrorResponse { code: S3ErrorCode::InvalidRange, message: err.to_string(), key: None, @@ -105,26 +115,27 @@ pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOpti region: None, request_id: None, host_id: "".to_string(), - }), + }); + } + }; + get_fn = Arc::new(move |input_reader: BufReader>>, _: HeaderMap| { + //Box::pin({ + /*let r = GetObjectReader { + object_info: oi.clone(), + stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)), }; - get_fn = Arc::new(move |input_reader: BufReader>>, _: HeaderMap| { - //Box::pin({ - /*let r = GetObjectReader { - object_info: oi.clone(), - stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)), - }; - r*/ - todo!(); - //}) - }); + r*/ + todo!(); + //}) + }); Ok((get_fn, off as i64, length as i64)) } pub fn extract_etag(metadata: &HashMap) -> String { - if let Some(etag) = metadata.get("etag") { + if let Some(etag) = metadata.get("etag") { etag.clone() } else { - metadata["md5Sum"].clone() - } -} \ No newline at end of file + metadata["md5Sum"].clone() + } +} diff --git a/ecstore/src/client/object_handlers_common.rs b/ecstore/src/client/object_handlers_common.rs index 0e0e7a7d..8e5b8761 100644 --- a/ecstore/src/client/object_handlers_common.rs +++ b/ecstore/src/client/object_handlers_common.rs @@ -1,15 +1,17 @@ -use lock::local_locker::MAX_DELETE_LIST; +use crate::StorageAPI; +use crate::bucket::lifecycle::lifecycle; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::store::ECStore; -use crate::store_api::{ObjectOptions, ObjectToDelete,}; -use crate::StorageAPI; -use crate::bucket::lifecycle::lifecycle; +use crate::store_api::{ObjectOptions, ObjectToDelete}; +use lock::local_locker::MAX_DELETE_LIST; pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) { let mut remaining = to_del; loop { - if remaining.len() <= 0 {break}; + if remaining.len() <= 0 { + break; + }; let mut to_del = remaining; if to_del.len() > MAX_DELETE_LIST { remaining = &to_del[MAX_DELETE_LIST..]; @@ -18,10 +20,14 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object remaining = &[]; } let vc = BucketVersioningSys::get(bucket).await.expect("err!"); - let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions { - //prefix_enabled_fn: vc.prefix_enabled(""), - version_suspended: vc.suspended(), - ..Default::default() - }); + let deleted_objs = api.delete_objects( + bucket, + to_del.to_vec(), + ObjectOptions { + //prefix_enabled_fn: vc.prefix_enabled(""), + version_suspended: vc.suspended(), + ..Default::default() + }, + ); } } diff --git a/ecstore/src/client/transition_api.rs b/ecstore/src/client/transition_api.rs index e48a9627..830feffb 100644 --- a/ecstore/src/client/transition_api.rs +++ b/ecstore/src/client/transition_api.rs @@ -1,61 +1,63 @@ #![allow(clippy::map_entry)] -use std::pin::Pin; use bytes::Bytes; use futures::Future; use http::{HeaderMap, HeaderName}; -use serde::{Serialize, Deserialize}; -use uuid::Uuid; +use http::{ + HeaderValue, Response, StatusCode, + request::{Builder, Request}, +}; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; +use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor}; use rand::Rng; -use std::{collections::HashMap, sync::{Arc, Mutex}}; +use serde::{Deserialize, Serialize}; +use std::io::Cursor; +use std::pin::Pin; use std::sync::atomic::{AtomicI32, Ordering}; use std::task::{Context, Poll}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use time::Duration; use time::OffsetDateTime; -use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; -use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector}; -use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response}; -use tracing::{error, debug}; -use url::{form_urlencoded, Url}; use tokio::io::BufReader; -use std::io::Cursor; +use tracing::{debug, error}; +use url::{Url, form_urlencoded}; +use uuid::Uuid; -use s3s::S3ErrorCode; -use s3s::{dto::Owner, Body}; -use s3s::dto::ReplicationStatus; use crate::client::bucket_cache::BucketLocationCache; -use reader::hasher::{Sha256, MD5,}; +use crate::client::{ + api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response}, + api_get_options::GetObjectOptions, + api_put_object::PutObjectOptions, + api_put_object_multipart::UploadPartParams, + api_s3_datatypes::{ + CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, + ListObjectPartsResult, ObjectPart, + }, + constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER}, + credentials::{CredContext, Credentials, SignatureType, Static}, +}; +use crate::signer; +use crate::{checksum::ChecksumMode, store_api::GetObjectReader}; +use reader::hasher::{MD5, Sha256}; use rustfs_rio::HashReader; use rustfs_utils::{ net::get_endpoint_url, - retry::{new_retry_timer, MAX_RETRY}, -}; -use crate::{ - store_api::GetObjectReader, - checksum::ChecksumMode, -}; -use crate::signer; -use crate::client::{ - constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER}, - credentials::{Credentials, SignatureType, CredContext, Static,}, - api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument}, - api_put_object_multipart::UploadPartParams, - api_put_object::PutObjectOptions, - api_get_options::GetObjectOptions, - api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart}, + retry::{MAX_RETRY, new_retry_timer}, }; +use s3s::S3ErrorCode; +use s3s::dto::ReplicationStatus; +use s3s::{Body, dto::Owner}; const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)"; -const C_USER_AGENT: &str = "RustFS (linux; x86)"; +const C_USER_AGENT: &str = "RustFS (linux; x86)"; -const SUCCESS_STATUS: [StatusCode; 3] = [ - StatusCode::OK, - StatusCode::NO_CONTENT, - StatusCode::PARTIAL_CONTENT, -]; +const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT]; const C_UNKNOWN: i32 = -1; const C_OFFLINE: i32 = 0; -const C_ONLINE: i32 = 1; +const C_ONLINE: i32 = 1; //pub type ReaderImpl = Box; pub enum ReaderImpl { @@ -71,7 +73,7 @@ pub struct TransitionClient { pub override_signer_type: SignatureType, /*app_info: TODO*/ pub secure: bool, - pub http_client: Client, Body>, + pub http_client: Client, Body>, //pub http_trace: Httptrace.ClientTrace, pub bucket_loc_cache: Arc>, pub is_trace_enabled: Arc>, @@ -83,7 +85,7 @@ pub struct TransitionClient { pub random: u64, pub lookup: BucketLookupType, //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType, - pub md5_hasher: Arc>>, + pub md5_hasher: Arc>>, pub sha256_hasher: Option, pub health_status: AtomicI32, pub trailing_header_support: bool, @@ -92,16 +94,16 @@ pub struct TransitionClient { #[derive(Debug, Default)] pub struct Options { - pub creds: Credentials, - pub secure: bool, + pub creds: Credentials, + pub secure: bool, //pub transport: http.RoundTripper, //pub trace: *httptrace.ClientTrace, - pub region: String, + pub region: String, pub bucket_lookup: BucketLookupType, //pub custom_region_via_url: func(u url.URL) string, //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType, pub trailing_headers: bool, - pub custom_md5: Option, + pub custom_md5: Option, pub custom_sha256: Option, pub max_retries: i64, } @@ -136,15 +138,13 @@ impl TransitionClient { //if scheme == "https" { // client = Client::builder(TokioExecutor::new()).build_http(); //} else { - let tls = rustls::ClientConfig::builder() - .with_native_roots()? - .with_no_client_auth(); - let https = hyper_rustls::HttpsConnectorBuilder::new() - .with_tls_config(tls) - .https_or_http() - .enable_http1() - .build(); - client = Client::builder(TokioExecutor::new()).build(https); + let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(tls) + .https_or_http() + .enable_http1() + .build(); + client = Client::builder(TokioExecutor::new()).build(https); //} let mut clnt = TransitionClient { @@ -167,7 +167,7 @@ impl TransitionClient { trailing_header_support: opts.trailing_headers, max_retries: opts.max_retries, }; - + { let mut md5_hasher = clnt.md5_hasher.lock().unwrap(); if md5_hasher.is_none() { @@ -218,7 +218,11 @@ impl TransitionClient { todo!(); } - pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap, HashMap>) { + pub fn hash_materials( + &self, + is_md5_requested: bool, + is_sha256_requested: bool, + ) -> (HashMap, HashMap>) { todo!(); } @@ -227,7 +231,8 @@ impl TransitionClient { } fn mark_offline(&self) { - self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); + self.health_status + .compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); } fn is_offline(&self) -> bool { @@ -261,7 +266,9 @@ impl TransitionClient { debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); resp = http_client.request(req); } - let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from)); + let resp = resp + .await /*.map_err(Into::into)*/ + .map(|res| res.map(Body::from)); debug!("http_client url: {} {}", req_method, req_uri); debug!("http_client headers: {:?}", req_headers); if let Err(err) = resp { @@ -282,7 +289,11 @@ impl TransitionClient { Ok(resp) } - pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result, std::io::Error> { + pub async fn execute_method( + &self, + method: http::Method, + metadata: &mut RequestMetadata, + ) -> Result, std::io::Error> { if self.is_offline() { let mut s = self.endpoint_url.to_string(); s.push_str(" is offline."); @@ -295,16 +306,18 @@ impl TransitionClient { let mut resp: http::Response; //if metadata.content_body != nil { - //body_seeker = BufferReader::new(metadata.content_body.read_all().await?); - retryable = true; - if !retryable { - req_retry = 1; - } + //body_seeker = BufferReader::new(metadata.content_body.read_all().await?); + retryable = true; + if !retryable { + req_retry = 1; + } //} //let mut retry_timer = RetryTimer::new(); //while let Some(v) = retry_timer.next().await { - for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ { + for _ in [1; 1] + /*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ + { let req = self.new_request(method, metadata).await?; resp = self.doit(req).await?; @@ -355,7 +368,11 @@ impl TransitionClient { Err(std::io::Error::other("resp err")) } - async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result, std::io::Error> { + async fn new_request( + &self, + method: http::Method, + metadata: &mut RequestMetadata, + ) -> Result, std::io::Error> { let location = metadata.bucket_location.clone(); if location == "" { if metadata.bucket_name != "" { @@ -366,8 +383,13 @@ impl TransitionClient { let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0; let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket; - let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location, - is_virtual_host, &metadata.query_values)?; + let target_url = self.make_target_url( + &metadata.bucket_name, + &metadata.object_name, + &location, + is_virtual_host, + &metadata.query_values, + )?; let mut req_builder = Request::builder().method(method).uri(target_url.to_string()); @@ -377,10 +399,10 @@ impl TransitionClient { value = creds_provider.get_with_context(Some(self.cred_context()))?; } - let mut signer_type = value.signer_type.clone(); - let access_key_id = value.access_key_id; + let mut signer_type = value.signer_type.clone(); + let access_key_id = value.access_key_id; let secret_access_key = value.secret_access_key; - let session_token = value.session_token; + let session_token = value.session_token; if self.override_signer_type != SignatureType::SignatureDefault { signer_type = self.override_signer_type.clone(); @@ -392,24 +414,39 @@ impl TransitionClient { if metadata.expires != 0 && metadata.pre_sign_url { if signer_type == SignatureType::SignatureAnonymous { - return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials."))); + return Err(std::io::Error::other(err_invalid_argument( + "Presigned URLs cannot be generated with anonymous credentials.", + ))); } if metadata.extra_pre_sign_header.is_some() { if signer_type == SignatureType::SignatureV2 { - return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported."))); + return Err(std::io::Error::other(err_invalid_argument( + "Extra signed headers for Presign with Signature V2 is not supported.", + ))); } for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() { req_builder = req_builder.header(k, v); } } if signer_type == SignatureType::SignatureV2 { - req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); + req_builder = + signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); } else if signer_type == SignatureType::SignatureV4 { - req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc()); + req_builder = signer::pre_sign_v4( + req_builder, + &access_key_id, + &secret_access_key, + &session_token, + &location, + metadata.expires, + OffsetDateTime::now_utc(), + ); } let req = match req_builder.body(Body::empty()) { Ok(req) => req, - Err(err) => { return Err(std::io::Error::other(err)); } + Err(err) => { + return Err(std::io::Error::other(err)); + } }; return Ok(req); } @@ -423,7 +460,10 @@ impl TransitionClient { //req.content_length = metadata.content_length; if metadata.content_length <= -1 { let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); - req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value); + req_builder + .headers_mut() + .expect("err") + .insert(http::header::TRANSFER_ENCODING, chunked_value); } if metadata.content_md5_base64.len() > 0 { @@ -434,15 +474,17 @@ impl TransitionClient { if signer_type == SignatureType::SignatureAnonymous { let req = match req_builder.body(Body::empty()) { Ok(req) => req, - Err(err) => { return Err(std::io::Error::other(err)); } + Err(err) => { + return Err(std::io::Error::other(err)); + } }; return Ok(req); } if signer_type == SignatureType::SignatureV2 { - req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); - } - else if metadata.stream_sha256 && !self.secure { + req_builder = + signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); + } else if metadata.stream_sha256 && !self.secure { if metadata.trailer.len() > 0 { //req.Trailer = metadata.trailer; for (_, v) in &metadata.trailer { @@ -451,8 +493,7 @@ impl TransitionClient { } //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id, // &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher()); - } - else { + } else { let mut sha_header = UNSIGNED_PAYLOAD.to_string(); if metadata.content_sha256_hex != "" { sha_header = metadata.content_sha256_hex.clone(); @@ -462,9 +503,17 @@ impl TransitionClient { } else if metadata.trailer.len() > 0 { sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); } - req_builder = req_builder.header::("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); + req_builder = req_builder + .header::("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); - req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone()); + req_builder = signer::sign_v4_trailer( + req_builder, + &access_key_id, + &secret_access_key, + &session_token, + &location, + metadata.trailer.clone(), + ); } let req; @@ -482,11 +531,9 @@ impl TransitionClient { //req = req_builder.body(s3s::Body::from(metadata.content_body.read_all().await?)); } - match req { + match req { Ok(req) => Ok(req), - Err(err) => { - Err(std::io::Error::other(err)) - } + Err(err) => Err(std::io::Error::other(err)), } } @@ -498,14 +545,17 @@ impl TransitionClient { }*/ } - fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap) -> Result { + fn make_target_url( + &self, + bucket_name: &str, + object_name: &str, + bucket_location: &str, + is_virtual_host_style: bool, + query_values: &HashMap, + ) -> Result { let scheme = self.endpoint_url.scheme(); let host = self.endpoint_url.host().unwrap(); - let default_port = if scheme == "https" { - 443 - } else { - 80 - }; + let default_port = if scheme == "https" { 443 } else { 80 }; let port = self.endpoint_url.port().unwrap_or(default_port); let mut url_str = format!("{scheme}://{host}:{port}/"); @@ -562,7 +612,7 @@ impl TransitionClient { } struct LockedRandSource { - src: u64,//rand.Source, + src: u64, //rand.Source, } impl LockedRandSource { @@ -604,76 +654,159 @@ impl TransitionCore { Ok(Self(Arc::new(client))) } - pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result { + pub fn list_objects( + &self, + bucket: &str, + prefix: &str, + marker: &str, + delimiter: &str, + max_keys: i64, + ) -> Result { let client = self.0.clone(); client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new()) } - pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result { + pub async fn list_objects_v2( + &self, + bucket_name: &str, + object_prefix: &str, + start_after: &str, + continuation_token: &str, + delimiter: &str, + max_keys: i64, + ) -> Result { let client = self.0.clone(); - client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await + client + .list_objects_v2_query( + bucket_name, + object_prefix, + continuation_token, + true, + false, + delimiter, + start_after, + max_keys, + HeaderMap::new(), + ) + .await } /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result { self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts) }*/ - pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str, - part_id: i32, start_offset: i32, length: i64, metadata: HashMap, + pub fn copy_object_part( + &self, + src_bucket: &str, + src_object: &str, + dest_bucket: &str, + dest_object: &str, + upload_id: &str, + part_id: i32, + start_offset: i32, + length: i64, + metadata: HashMap, ) -> Result { //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id, // part_id, start_offset, length, metadata) todo!(); } - pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result { - let hook_reader = data;//newHook(data, opts.progress); + pub async fn put_object( + &self, + bucket: &str, + object: &str, + data: ReaderImpl, + size: i64, + md5_base64: &str, + sha256_hex: &str, + opts: &PutObjectOptions, + ) -> Result { + let hook_reader = data; //newHook(data, opts.progress); let client = self.0.clone(); - client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await + client + .put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts) + .await } - pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result { + pub async fn new_multipart_upload( + &self, + bucket: &str, + object: &str, + opts: PutObjectOptions, + ) -> Result { let client = self.0.clone(); let result = client.initiate_multipart_upload(bucket, object, &opts).await?; Ok(result.upload_id) } - pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result { + pub fn list_multipart_uploads( + &self, + bucket: &str, + prefix: &str, + key_marker: &str, + upload_id_marker: &str, + delimiter: &str, + max_uploads: i64, + ) -> Result { let client = self.0.clone(); client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads) } - pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64, - data: ReaderImpl, size: i64, opts: PutObjectPartOptions + pub async fn put_object_part( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_id: i64, + data: ReaderImpl, + size: i64, + opts: PutObjectPartOptions, ) -> Result { let mut p = UploadPartParams { - bucket_name: bucket.to_string(), - object_name: object.to_string(), - upload_id: upload_id.to_string(), - reader: data, - part_number: part_id, - md5_base64: opts.md5_base64, - sha256_hex: opts.sha256_hex, - size: size, + bucket_name: bucket.to_string(), + object_name: object.to_string(), + upload_id: upload_id.to_string(), + reader: data, + part_number: part_id, + md5_base64: opts.md5_base64, + sha256_hex: opts.sha256_hex, + size: size, //sse: opts.sse, stream_sha256: !opts.disable_content_sha256, custom_header: opts.custom_header, - trailer: opts.trailer, + trailer: opts.trailer, }; let client = self.0.clone(); client.upload_part(&mut p).await } - pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result { + pub async fn list_object_parts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_number_marker: i64, + max_parts: i64, + ) -> Result { let client = self.0.clone(); - client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await + client + .list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts) + .await } - pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result { + pub async fn complete_multipart_upload( + &self, + bucket: &str, + object: &str, + upload_id: &str, + parts: &[CompletePart], + opts: PutObjectOptions, + ) -> Result { let client = self.0.clone(); - let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { - parts: parts.to_vec(), - }, &opts).await?; + let res = client + .complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { parts: parts.to_vec() }, &opts) + .await?; Ok(res) } @@ -692,7 +825,12 @@ impl TransitionCore { client.put_bucket_policy(bucket_name, bucket_policy).await } - pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + pub async fn get_object( + &self, + bucket_name: &str, + object_name: &str, + opts: &GetObjectOptions, + ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { let client = self.0.clone(); client.get_object_inner(bucket_name, object_name, opts).await } @@ -709,8 +847,8 @@ pub struct PutObjectPartOptions { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ObjectInfo { - pub etag: String, - pub name: String, + pub etag: String, + pub name: String, pub mod_time: OffsetDateTime, pub size: usize, pub content_type: Option, @@ -730,16 +868,16 @@ pub struct ObjectInfo { #[serde(skip, default = "replication_status_default")] pub replication_status: ReplicationStatus, pub replication_ready: bool, - pub expiration: OffsetDateTime, + pub expiration: OffsetDateTime, pub expiration_rule_id: String, pub num_versions: usize, - + pub restore: RestoreInfo, - pub checksum_crc32: String, - pub checksum_crc32c: String, - pub checksum_sha1: String, - pub checksum_sha256: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, pub checksum_crc64nvme: String, pub checksum_mode: String, } @@ -767,22 +905,21 @@ impl Default for ObjectInfo { version_id: Uuid::nil(), replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), replication_ready: false, - expiration: OffsetDateTime::now_utc(), + expiration: OffsetDateTime::now_utc(), expiration_rule_id: "".to_string(), num_versions: 0, restore: RestoreInfo::default(), - checksum_crc32: "".to_string(), - checksum_crc32c: "".to_string(), - checksum_sha1: "".to_string(), - checksum_sha256: "".to_string(), + checksum_crc32: "".to_string(), + checksum_crc32c: "".to_string(), + checksum_sha1: "".to_string(), + checksum_sha256: "".to_string(), checksum_crc64nvme: "".to_string(), checksum_mode: "".to_string(), } } } -#[derive(Serialize, Deserialize)] -#[derive(Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct RestoreInfo { ongoing_restore: bool, expiry_time: OffsetDateTime, @@ -809,19 +946,19 @@ pub struct ObjectMultipartInfo { } pub struct UploadInfo { - pub bucket: String, - pub key: String, - pub etag: String, - pub size: i64, + pub bucket: String, + pub key: String, + pub etag: String, + pub size: i64, pub last_modified: OffsetDateTime, - pub location: String, - pub version_id: String, - pub expiration: OffsetDateTime, + pub location: String, + pub version_id: String, + pub expiration: OffsetDateTime, pub expiration_rule_id: String, - pub checksum_crc32: String, - pub checksum_crc32c: String, - pub checksum_sha1: String, - pub checksum_sha256: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, pub checksum_crc64nvme: String, pub checksum_mode: String, } @@ -885,4 +1022,4 @@ impl tower::Service> for SendRequest { } #[derive(Serialize, Deserialize)] -pub struct Document(pub String); \ No newline at end of file +pub struct Document(pub String); diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index d1979ff9..45768e0b 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -1,4 +1,4 @@ -use s3s::{S3ErrorCode, S3Error}; +use s3s::{S3Error, S3ErrorCode}; use rustfs_utils::path::decode_dir_object; @@ -727,16 +727,16 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error { } pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool { - err.contains("Connection closed by foreign host") || - err.contains("TLS handshake timeout") || - err.contains("i/o timeout") || - err.contains("connection timed out") || - err.contains("connection reset by peer") || - err.contains("broken pipe") || - err.to_lowercase().contains("503 service unavailable") || - err.contains("use of closed network connection") || - err.contains("An existing connection was forcibly closed by the remote host") || - err.contains("client error (Connect)") + err.contains("Connection closed by foreign host") + || err.contains("TLS handshake timeout") + || err.contains("i/o timeout") + || err.contains("connection timed out") + || err.contains("connection reset by peer") + || err.contains("broken pipe") + || err.to_lowercase().contains("503 service unavailable") + || err.contains("use of closed network connection") + || err.contains("An existing connection was forcibly closed by the remote host") + || err.contains("client error (Connect)") } #[derive(Debug, Default, PartialEq, Eq)] @@ -834,7 +834,7 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i err = std::io::Error::other(StorageError::BucketNameInvalid(bucket)); } S3ErrorCode::InvalidPart => { - err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object/* , version_id */)); + err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */)); } S3ErrorCode::NoSuchBucket => { err = std::io::Error::other(StorageError::BucketNotFound(bucket)); @@ -848,7 +848,7 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i } S3ErrorCode::NoSuchVersion => { if object != "" { - err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));//, version_id); + err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id); } else { err = std::io::Error::other(StorageError::BucketNotFound(bucket)); } @@ -894,9 +894,15 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { StorageError::InvalidArgument(bucket, object, version_id) => { s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) }*/ - StorageError::MethodNotAllowed => { - S3Error::with_message(S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError {bucket: bucket, object: object, ..Default::default()}).to_string()) - } + StorageError::MethodNotAllowed => S3Error::with_message( + S3ErrorCode::MethodNotAllowed, + ObjectApiError::MethodNotAllowed(GenericError { + bucket: bucket, + object: object, + ..Default::default() + }) + .to_string(), + ), /*StorageError::BucketNotFound(bucket) => { s3_error!(NoSuchBucket, "bucket not found {}", bucket) } diff --git a/ecstore/src/event/mod.rs b/ecstore/src/event/mod.rs index f3cb1623..fc878d34 100644 --- a/ecstore/src/event/mod.rs +++ b/ecstore/src/event/mod.rs @@ -1,3 +1,3 @@ pub mod name; pub mod targetid; -pub mod targetlist; \ No newline at end of file +pub mod targetlist; diff --git a/ecstore/src/event/name.rs b/ecstore/src/event/name.rs index 33fee80f..8ecabe9f 100644 --- a/ecstore/src/event/name.rs +++ b/ecstore/src/event/name.rs @@ -55,7 +55,7 @@ impl EventName { } impl AsRef for EventName { - fn as_ref(&self) -> &str { + fn as_ref(&self) -> &str { match self { EventName::BucketCreated => "s3:BucketCreated:*", EventName::BucketRemoved => "s3:BucketRemoved:*", @@ -103,123 +103,45 @@ impl AsRef for EventName { impl From<&str> for EventName { fn from(s: &str) -> Self { match s { - "s3:BucketCreated:*" => { - EventName::BucketCreated - } - "s3:BucketRemoved:*" => { - EventName::BucketRemoved - } - "s3:ObjectAccessed:*" => { - EventName::ObjectAccessedAll - } - "s3:ObjectAccessed:Get" => { - EventName::ObjectAccessedGet - } - "s3:ObjectAccessed:GetRetention" => { - EventName::ObjectAccessedGetRetention - } - "s3:ObjectAccessed:GetLegalHold" => { - EventName::ObjectAccessedGetLegalHold - } - "s3:ObjectAccessed:Head" => { - EventName::ObjectAccessedHead - } - "s3:ObjectAccessed:Attributes" => { - EventName::ObjectAccessedAttributes - } - "s3:ObjectCreated:*" => { - EventName::ObjectCreatedAll - } - "s3:ObjectCreated:CompleteMultipartUpload" => { - EventName::ObjectCreatedCompleteMultipartUpload - } - "s3:ObjectCreated:Copy" => { - EventName::ObjectCreatedCopy - } - "s3:ObjectCreated:Post" => { - EventName::ObjectCreatedPost - } - "s3:ObjectCreated:Put" => { - EventName::ObjectCreatedPut - } - "s3:ObjectCreated:PutRetention" => { - EventName::ObjectCreatedPutRetention - } - "s3:ObjectCreated:PutLegalHold" => { - EventName::ObjectCreatedPutLegalHold - } - "s3:ObjectCreated:PutTagging" => { - EventName::ObjectCreatedPutTagging - } - "s3:ObjectCreated:DeleteTagging" => { - EventName::ObjectCreatedDeleteTagging - } - "s3:ObjectRemoved:*" => { - EventName::ObjectRemovedAll - } - "s3:ObjectRemoved:Delete" => { - EventName::ObjectRemovedDelete - } - "s3:ObjectRemoved:DeleteMarkerCreated" => { - EventName::ObjectRemovedDeleteMarkerCreated - } - "s3:ObjectRemoved:NoOP" => { - EventName::ObjectRemovedNoOP - } - "s3:ObjectRemoved:DeleteAllVersions" => { - EventName::ObjectRemovedDeleteAllVersions - } - "s3:LifecycleDelMarkerExpiration:Delete" => { - EventName::ILMDelMarkerExpirationDelete - } - "s3:Replication:*" => { - EventName::ObjectReplicationAll - } - "s3:Replication:OperationFailedReplication" => { - EventName::ObjectReplicationFailed - } - "s3:Replication:OperationCompletedReplication" => { - EventName::ObjectReplicationComplete - } - "s3:Replication:OperationMissedThreshold" => { - EventName::ObjectReplicationMissedThreshold - } - "s3:Replication:OperationReplicatedAfterThreshold" => { - EventName::ObjectReplicationReplicatedAfterThreshold - } - "s3:Replication:OperationNotTracked" => { - EventName::ObjectReplicationNotTracked - } - "s3:ObjectRestore:*" => { - EventName::ObjectRestoreAll - } - "s3:ObjectRestore:Post" => { - EventName::ObjectRestorePost - } - "s3:ObjectRestore:Completed" => { - EventName::ObjectRestoreCompleted - } - "s3:ObjectTransition:Failed" => { - EventName::ObjectTransitionFailed - } - "s3:ObjectTransition:Complete" => { - EventName::ObjectTransitionComplete - } - "s3:ObjectTransition:*" => { - EventName::ObjectTransitionAll - } - "s3:Scanner:ManyVersions" => { - EventName::ObjectManyVersions - } - "s3:Scanner:LargeVersions" => { - EventName::ObjectLargeVersions - } - "s3:Scanner:BigPrefix" => { - EventName::PrefixManyFolders - } - _ => { - EventName::Everything - } + "s3:BucketCreated:*" => EventName::BucketCreated, + "s3:BucketRemoved:*" => EventName::BucketRemoved, + "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll, + "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet, + "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention, + "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold, + "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead, + "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes, + "s3:ObjectCreated:*" => EventName::ObjectCreatedAll, + "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload, + "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy, + "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost, + "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut, + "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention, + "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold, + "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging, + "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging, + "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll, + "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete, + "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated, + "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP, + "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions, + "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete, + "s3:Replication:*" => EventName::ObjectReplicationAll, + "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed, + "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete, + "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold, + "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold, + "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked, + "s3:ObjectRestore:*" => EventName::ObjectRestoreAll, + "s3:ObjectRestore:Post" => EventName::ObjectRestorePost, + "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted, + "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed, + "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete, + "s3:ObjectTransition:*" => EventName::ObjectTransitionAll, + "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions, + "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions, + "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders, + _ => EventName::Everything, } } -} \ No newline at end of file +} diff --git a/ecstore/src/event/targetid.rs b/ecstore/src/event/targetid.rs index c27e6eed..a2a01289 100644 --- a/ecstore/src/event/targetid.rs +++ b/ecstore/src/event/targetid.rs @@ -1,6 +1,6 @@ pub struct TargetID { - id: String, - name: String, + id: String, + name: String, } impl TargetID { diff --git a/ecstore/src/event/targetlist.rs b/ecstore/src/event/targetlist.rs index 67559df6..30602c09 100644 --- a/ecstore/src/event/targetlist.rs +++ b/ecstore/src/event/targetlist.rs @@ -4,9 +4,9 @@ use super::targetid::TargetID; #[derive(Default)] pub struct TargetList { - pub current_send_calls: AtomicI64, - pub total_events: AtomicI64, - pub events_skipped: AtomicI64, + pub current_send_calls: AtomicI64, + pub total_events: AtomicI64, + pub events_skipped: AtomicI64, pub events_errors_total: AtomicI64, //pub targets: HashMap, //pub queue: AsyncEvent, @@ -26,6 +26,6 @@ struct TargetStat { } struct TargetIDResult { - id: TargetID, + id: TargetID, err: std::io::Error, } diff --git a/ecstore/src/event_notification.rs b/ecstore/src/event_notification.rs index 4182fbc7..3639f7f4 100644 --- a/ecstore/src/event_notification.rs +++ b/ecstore/src/event_notification.rs @@ -2,11 +2,11 @@ use std::collections::HashMap; use std::sync::Arc; use tokio::sync::RwLock; -use crate::store_api::ObjectInfo; +use crate::bucket::metadata::BucketMetadata; use crate::event::name::EventName; use crate::event::targetlist::TargetList; use crate::store::ECStore; -use crate::bucket::metadata::BucketMetadata; +use crate::store_api::ObjectInfo; pub struct EventNotifier { target_list: TargetList, @@ -16,7 +16,7 @@ pub struct EventNotifier { impl EventNotifier { pub fn new() -> Arc> { Arc::new(RwLock::new(Self { - target_list: TargetList::new(), + target_list: TargetList::new(), //bucket_rules_map: HashMap::new(), })) } @@ -54,8 +54,6 @@ pub struct EventArgs { pub user_agent: String, } -impl EventArgs { -} +impl EventArgs {} -pub fn send_event(args: EventArgs) { -} +pub fn send_event(args: EventArgs) {} diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index d860718d..34c6b4ac 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -9,13 +9,13 @@ use uuid::Uuid; use crate::heal::mrf::MRFState; use crate::{ + bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, disk::DiskStore, endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, - heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, - bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, - tier::tier::TierConfigMgr, event_notification::EventNotifier, + heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, store::ECStore, + tier::tier::TierConfigMgr, }; pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30; @@ -26,29 +26,29 @@ pub const DISK_RESERVE_FRACTION: f64 = 0.15; pub const DEFAULT_PORT: u16 = 9000; lazy_static! { - static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); - pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); - pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); - pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); - pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); - pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); - pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); - pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); - pub static ref GLOBAL_Endpoints: OnceLock = OnceLock::new(); - pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); - pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); - pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); - pub static ref GLOBAL_TierConfigMgr: Arc> = TierConfigMgr::new(); - pub static ref GLOBAL_LifecycleSys: Arc = LifecycleSys::new(); - pub static ref GLOBAL_EventNotifier: Arc> = EventNotifier::new(); - //pub static ref GLOBAL_RemoteTargetTransport - pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); - pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); - static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); - pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); - pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); - pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes()); - pub static ref GLOBAL_NodeNamesHex: HashMap = HashMap::new();} +static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); +pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); +pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); +pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); +pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); +pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); +pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); +pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); +pub static ref GLOBAL_Endpoints: OnceLock = OnceLock::new(); +pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); +pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); +pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); +pub static ref GLOBAL_TierConfigMgr: Arc> = TierConfigMgr::new(); +pub static ref GLOBAL_LifecycleSys: Arc = LifecycleSys::new(); +pub static ref GLOBAL_EventNotifier: Arc> = EventNotifier::new(); +//pub static ref GLOBAL_RemoteTargetTransport +pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); +pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); +static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); +pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); +pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); +pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes()); +pub static ref GLOBAL_NodeNamesHex: HashMap = HashMap::new();} /// Get the global rustfs port pub fn global_rustfs_port() -> u16 { diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index 3d69ba2a..80c01021 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -6,16 +6,15 @@ use std::{ path::{Path, PathBuf}, pin::Pin, sync::{ - atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, Arc, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }, time::{Duration, SystemTime}, }; -use time::{self, OffsetDateTime}; use common::defer; +use time::{self, OffsetDateTime}; -use rustfs_utils::path::encode_dir_object; use super::{ data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend}, @@ -23,28 +22,25 @@ use super::{ heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, }; use crate::bucket::{ - object_lock::objectlock_sys::{ - enforce_retention_for_deletion, - BucketObjectLockSys, - }, utils::is_meta_bucketname, + object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion}, + utils::is_meta_bucketname, }; +use crate::cmd::bucket_replication::queue_replication_heal; +use crate::event::name::EventName; use crate::{ bucket::{ lifecycle::{ bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}, - lifecycle::{self, Lifecycle, ExpirationOptions}, - bucket_lifecycle_ops::{ - self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, - expire_transitioned_object, - }, + bucket_lifecycle_ops::{self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object}, + lifecycle::{self, ExpirationOptions, Lifecycle}, }, - metadata_sys + metadata_sys, }, - event_notification::{send_event, EventArgs}, - global::GLOBAL_LocalNodeName, heal::{data_scanner}, + event_notification::{EventArgs, send_event}, + global::GLOBAL_LocalNodeName, + heal::data_scanner, store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, }; -use crate::cmd::bucket_replication::queue_replication_heal; use crate::{ bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, cmd::bucket_replication::ReplicationStatusType, @@ -70,7 +66,6 @@ use crate::{ peer::is_reserved_or_invalid_bucket, store::ECStore, }; -use crate::event::name::EventName; use crate::{disk::DiskAPI, store_api::ObjectInfo}; use crate::{ disk::error::DiskError, @@ -82,8 +77,12 @@ use lazy_static::lazy_static; use rand::Rng; use rmp_serde::{Deserializer, Serializer}; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_utils::path::encode_dir_object; use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration}; +use s3s::dto::{ + BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, + ReplicationRuleStatus, VersioningConfiguration, +}; use serde::{Deserialize, Serialize}; use tokio::{ sync::{ @@ -561,20 +560,33 @@ impl ScannerItem { lr = BucketObjectLockSys::get(&self.bucket).await; rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await { Some(replication_config) - } else { None }; + } else { + None + }; } let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; if self.debug { if !version_id.is_none() { - info!("lifecycle: {} (version-id={}), Initial scan: {}", self.object_path().to_string_lossy().to_string(), version_id.expect("err"), lc_evt.action); + info!( + "lifecycle: {} (version-id={}), Initial scan: {}", + self.object_path().to_string_lossy().to_string(), + version_id.expect("err"), + lc_evt.action + ); } else { - info!("lifecycle: {} Initial scan: {}", self.object_path().to_string_lossy().to_string(), lc_evt.action); + info!( + "lifecycle: {} Initial scan: {}", + self.object_path().to_string_lossy().to_string(), + lc_evt.action + ); } } match lc_evt.action { - lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + lifecycle::IlmAction::DeleteVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction + | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { size = 0; } lifecycle::IlmAction::DeleteAction => { @@ -582,7 +594,7 @@ impl ScannerItem { size = 0 } } - _ => () + _ => (), } apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; @@ -612,7 +624,9 @@ impl ScannerItem { let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { rcfg.mode.is_some() - } else { false }; + } else { + false + }; let vcfg = BucketVersioningSys::get(&self.bucket).await?; let versioned = match BucketVersioningSys::get(&self.bucket).await { @@ -633,28 +647,43 @@ impl ScannerItem { return Ok(object_infos); } - let event = self.lifecycle.as_ref().expect("lifecycle err.").noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { - name: self.object_path().to_string_lossy().to_string(), - ..Default::default() - }).await; + let event = self + .lifecycle + .as_ref() + .expect("lifecycle err.") + .noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { + name: self.object_path().to_string_lossy().to_string(), + ..Default::default() + }) + .await; let lim = event.newer_noncurrent_versions; - if lim == 0 || fivs.len() <= lim+1 { + if lim == 0 || fivs.len() <= lim + 1 { for fi in fivs.iter() { - object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); + object_infos.push(ObjectInfo::from_file_info( + fi, + &self.bucket, + &self.object_path().to_string_lossy(), + versioned, + )); } return Ok(object_infos); } - let overflow_versions = &fivs[lim+1..]; - for fi in fivs[..lim+1].iter() { - object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); + let overflow_versions = &fivs[lim + 1..]; + for fi in fivs[..lim + 1].iter() { + object_infos.push(ObjectInfo::from_file_info( + fi, + &self.bucket, + &self.object_path().to_string_lossy(), + versioned, + )); } let mut to_del = Vec::::with_capacity(overflow_versions.len()); for fi in overflow_versions.iter() { let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); if lock_enabled && enforce_retention_for_deletion(&obj) { - //if enforce_retention_for_deletion(&obj) { + //if enforce_retention_for_deletion(&obj) { if self.debug { if obj.version_id.is_some() { info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); @@ -666,7 +695,10 @@ impl ScannerItem { continue; } - if OffsetDateTime::now_utc().unix_timestamp() < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32).unix_timestamp() { + if OffsetDateTime::now_utc().unix_timestamp() + < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32) + .unix_timestamp() + { object_infos.push(obj); continue; } @@ -688,7 +720,7 @@ impl ScannerItem { pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, usize) { let done = ScannerMetrics::time(ScannerMetric::Ilm); - + let (action, size) = self.apply_lifecycle(oi).await; info!( @@ -1535,15 +1567,18 @@ pub async fn scan_data_folder( Ok(s.new_cache) } -pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: Option, rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, oi: &ObjectInfo) -> lifecycle::Event { +pub async fn eval_action_from_lifecycle( + lc: &BucketLifecycleConfiguration, + lr: Option, + rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, + oi: &ObjectInfo, +) -> lifecycle::Event { let event = lc.eval(&oi.to_lifecycle_opts()).await; //if serverDebugLog { - info!("lifecycle: Secondary scan: {}", event.action); + info!("lifecycle: Secondary scan: {}", event.action); //} - let lock_enabled = if let Some(lr) = lr { - lr.mode.is_some() - } else { false }; + let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; match event.action { lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { @@ -1557,11 +1592,11 @@ pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: O } if lock_enabled && enforce_retention_for_deletion(oi) { //if serverDebugLog { - if !oi.version_id.is_none() { - info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); - } else { - info!("lifecycle: {} is locked, not deleting", oi.name); - } + if !oi.version_id.is_none() { + info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting", oi.name); + } //} return lifecycle::Event::default(); } @@ -1571,7 +1606,7 @@ pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: O } } } - _ => () + _ => (), } event @@ -1585,7 +1620,12 @@ async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: & true } -pub async fn apply_expiry_on_transitioned_object(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { +pub async fn apply_expiry_on_transitioned_object( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> bool { let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { return false; @@ -1595,9 +1635,14 @@ pub async fn apply_expiry_on_transitioned_object(api: Arc, oi: &ObjectI true } -pub async fn apply_expiry_on_non_transitioned_objects(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { +pub async fn apply_expiry_on_non_transitioned_objects( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> bool { let mut opts = ObjectOptions { - expiration: ExpirationOptions {expire: true}, + expiration: ExpirationOptions { expire: true }, ..Default::default() }; @@ -1617,7 +1662,10 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc, oi: &Ob let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); - let mut dobj = api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await.unwrap(); + let mut dobj = api + .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts) + .await + .unwrap(); if dobj.name == "" { dobj = oi.clone(); } @@ -1630,13 +1678,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc, oi: &Ob event_name = EventName::ObjectRemovedDeleteMarkerCreated; } match lc_event.action { - lifecycle::IlmAction::DeleteAllVersionsAction => { - event_name = EventName::ObjectRemovedDeleteAllVersions - } - lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { - event_name = EventName::ILMDelMarkerExpirationDelete - } - _ => () + lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions, + lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete, + _ => (), } send_event(EventArgs { event_name: event_name.as_ref().to_string(), @@ -1667,20 +1711,22 @@ async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &Obje pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { let mut success = false; match event.action { - lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAction - | lifecycle::IlmAction::DeleteRestoredAction | lifecycle::IlmAction::DeleteRestoredVersionAction - | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + lifecycle::IlmAction::DeleteVersionAction + | lifecycle::IlmAction::DeleteAction + | lifecycle::IlmAction::DeleteRestoredAction + | lifecycle::IlmAction::DeleteRestoredVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction + | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { success = apply_expiry_rule(event, src, oi).await; } lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { success = apply_transition_rule(event, src, oi).await; } - _ => () + _ => (), } success } - #[cfg(test)] mod tests { use std::io::Cursor; diff --git a/ecstore/src/heal/data_scanner_metric.rs b/ecstore/src/heal/data_scanner_metric.rs index f59818af..ced1e6b8 100644 --- a/ecstore/src/heal/data_scanner_metric.rs +++ b/ecstore/src/heal/data_scanner_metric.rs @@ -312,9 +312,7 @@ impl ScannerMetrics { pub fn time_ilm(a: lifecycle::IlmAction) -> Box Box + Send + Sync> { let a_clone = a as usize; if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { - return Box::new(move |_: u64| { - Box::new(move || {}) - }); + return Box::new(move |_: u64| Box::new(move || {})); } let start = SystemTime::now(); Box::new(move |versions: u64| { diff --git a/ecstore/src/heal/data_usage_cache.rs b/ecstore/src/heal/data_usage_cache.rs index 35aac655..a88da15a 100644 --- a/ecstore/src/heal/data_usage_cache.rs +++ b/ecstore/src/heal/data_usage_cache.rs @@ -139,9 +139,9 @@ pub struct TierStats { impl TierStats { pub fn add(&self, u: &TierStats) -> TierStats { TierStats { - total_size: self.total_size + u.total_size, + total_size: self.total_size + u.total_size, num_versions: self.num_versions + u.num_versions, - num_objects: self.num_objects + u.num_objects, + num_objects: self.num_objects + u.num_objects, } } } @@ -153,9 +153,7 @@ struct AllTierStats { impl AllTierStats { pub fn new() -> Self { - Self { - tiers: HashMap::new(), - } + Self { tiers: HashMap::new() } } fn add_sizes(&mut self, tiers: HashMap) { @@ -172,11 +170,14 @@ impl AllTierStats { fn populate_stats(&self, stats: &mut HashMap) { for (tier, st) in &self.tiers { - stats.insert(tier.clone(), TierStats { - total_size: st.total_size.clone(), - num_versions: st.num_versions.clone(), - num_objects: st.num_objects.clone(), - }); + stats.insert( + tier.clone(), + TierStats { + total_size: st.total_size.clone(), + num_versions: st.num_versions.clone(), + num_objects: st.num_objects.clone(), + }, + ); } } } diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 178aa4f4..9ca4dd0e 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -27,11 +27,11 @@ pub mod store_list_objects; mod store_utils; pub mod checksum; +pub mod client; pub mod event; pub mod event_notification; -pub mod client; -pub mod tier; pub mod signer; +pub mod tier; pub use global::new_object_layer_fn; pub use global::set_global_endpoints; diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 548a60b3..e2ab478e 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -1,6 +1,6 @@ -use s3s::header::X_AMZ_RESTORE; -use crate::error::ObjectApiError; use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; +use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl}; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::{ self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, @@ -8,12 +8,15 @@ use crate::disk::{ }; use crate::erasure_coding; use crate::erasure_coding::bitrot_verify; +use crate::error::ObjectApiError; use crate::error::{Error, Result}; use crate::global::GLOBAL_MRFState; +use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr}; use crate::heal::data_usage_cache::DataUsageCache; use crate::heal::heal_ops::{HealEntryFn, HealSequence}; use crate::store_api::ObjectToDelete; use crate::{ + bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, config::{GLOBAL_StorageClass, storageclass}, disk::{ @@ -22,8 +25,8 @@ use crate::{ UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, error::{StorageError, to_object_err}, - event::name::EventName, event_notification::{send_event, EventArgs}, - bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, put_restore_opts, get_transitioned_object_reader,}, + event::name::EventName, + event_notification::{EventArgs, send_event}, global::{ GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure, @@ -44,16 +47,11 @@ use crate::{ }, store_init::load_format_erasure, }; -use crate::client::{ - object_api_utils::extract_etag, - transition_api::ReaderImpl, -}; use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; use crate::{ heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, store_api::ListObjectVersionsInfo, }; -use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use bytesize::ByteSize; use chrono::Utc; use futures::future::join_all; @@ -75,6 +73,7 @@ use rustfs_utils::{ crypto::{base64_decode, base64_encode, hex}, path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, }; +use s3s::header::X_AMZ_RESTORE; use sha2::{Digest, Sha256}; use std::hash::Hash; use std::mem; @@ -100,7 +99,6 @@ use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; use workers::workers::Workers; -use crate::global::{GLOBAL_TierConfigMgr, GLOBAL_LocalNodeName}; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; @@ -3677,7 +3675,13 @@ impl SetDisks { Ok(()) } - pub async fn update_restore_metadata(&self, bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> { + pub async fn update_restore_metadata( + &self, + bucket: &str, + object: &str, + obj_info: &ObjectInfo, + opts: &ObjectOptions, + ) -> Result<()> { let mut oi = obj_info.clone(); oi.metadata_only = true; @@ -3686,13 +3690,23 @@ impl SetDisks { } let version_id = oi.version_id.clone().map(|v| v.to_string()); - let obj = self.copy_object(bucket, object, bucket, object, &mut oi, &ObjectOptions { - version_id: version_id.clone(), - ..Default::default() - }, &ObjectOptions { - version_id: version_id, - ..Default::default() - }).await; + let obj = self + .copy_object( + bucket, + object, + bucket, + object, + &mut oi, + &ObjectOptions { + version_id: version_id.clone(), + ..Default::default() + }, + &ObjectOptions { + version_id: version_id, + ..Default::default() + }, + ) + .await; if let Err(err) = obj { //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) return Err(err); @@ -4169,7 +4183,10 @@ impl StorageAPI for SetDisks { for disk in disks.iter() { futures.push(async move { if let Some(disk) = disk { - match disk.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()).await { + match disk + .delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()) + .await + { Ok(r) => Ok(r), Err(e) => Err(e), } @@ -4392,16 +4409,17 @@ impl StorageAPI for SetDisks { #[tracing::instrument(skip(self))] async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { - GLOBAL_MRFState.add_partial(PartialOperation { - bucket: bucket.to_string(), - object: object.to_string(), - version_id: Some(version_id.to_string()), - queued: Utc::now(), - set_index: self.set_index, - pool_index: self.pool_index, - ..Default::default() - }) - .await; + GLOBAL_MRFState + .add_partial(PartialOperation { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: Some(version_id.to_string()), + queued: Utc::now(), + set_index: self.set_index, + pool_index: self.pool_index, + ..Default::default() + }) + .await; Ok(()) } @@ -4518,7 +4536,9 @@ impl StorageAPI for SetDisks { } return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object])); }*/ - if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() || !(opts.transition.etag == extract_etag(&fi.metadata)) { + if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() + || !(opts.transition.etag == extract_etag(&fi.metadata)) + { return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object])); } if fi.transition_status == TRANSITION_COMPLETE { @@ -4548,7 +4568,10 @@ impl StorageAPI for SetDisks { let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); //let h = HeaderMap::new(); //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi}); - let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: Box::new(pr), object_info: oi}); + let reader = ReaderImpl::ObjectBody(GetObjectReader { + stream: Box::new(pr), + object_info: oi, + }); let cloned_bucket = bucket.to_string(); let cloned_object = object.to_string(); @@ -4557,7 +4580,16 @@ impl StorageAPI for SetDisks { let pool_index = self.pool_index; tokio::spawn(async move { if let Err(e) = Self::get_object_with_fileinfo( - &cloned_bucket, &cloned_object, 0, cloned_fi.size, &mut pw, cloned_fi, meta_arr, &online_disks, set_index, pool_index + &cloned_bucket, + &cloned_object, + 0, + cloned_fi.size, + &mut pw, + cloned_fi, + meta_arr, + &online_disks, + set_index, + pool_index, ) .await { @@ -4565,11 +4597,13 @@ impl StorageAPI for SetDisks { }; }); - let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size as i64, { - let mut m = HashMap::::new(); - m.insert("name".to_string(), object.to_string()); - m - }).await; + let rv = tgt_client + .put_with_meta(&dest_obj, reader, fi.size as i64, { + let mut m = HashMap::::new(); + m.insert("name".to_string(), object.to_string()); + m + }) + .await; //pr.CloseWithError(err); if let Err(err) = rv { //traceFn(ILMTransition, nil, err) @@ -4579,7 +4613,7 @@ impl StorageAPI for SetDisks { fi.transition_status = TRANSITION_COMPLETE.to_string(); fi.transitioned_objname = dest_obj; fi.transition_tier = opts.transition.tier.clone(); - fi.transition_version_id = if rv=="" { None } else { Some(Uuid::parse_str(&rv)?) }; + fi.transition_version_id = if rv == "" { None } else { Some(Uuid::parse_str(&rv)?) }; let mut event_name = EventName::ObjectTransitionComplete.as_ref(); let disks = self.get_disks(0, 0).await?; @@ -4594,7 +4628,8 @@ impl StorageAPI for SetDisks { continue; } } - self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")).await; + self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")) + .await; break; } @@ -4602,9 +4637,9 @@ impl StorageAPI for SetDisks { send_event(EventArgs { event_name: event_name.to_string(), bucket_name: bucket.to_string(), - object: obj_info, - user_agent: "Internal: [ILM-Transition]".to_string(), - host: GLOBAL_LocalNodeName.to_string(), + object: obj_info, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), ..Default::default() }); //let tags = opts.lifecycle_audit_event.tags(); diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 808946e1..c5343757 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,7 +1,6 @@ #![allow(clippy::map_entry)] use std::{collections::HashMap, sync::Arc}; -use rustfs_filemeta::FileInfo; use crate::disk::error_reduce::count_errs; use crate::error::{Error, Result}; use crate::{ @@ -30,6 +29,7 @@ use futures::future::join_all; use http::HeaderMap; use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; use madmin::heal_commands::{HealDriveInfo, HealResultItem}; +use rustfs_filemeta::FileInfo; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use tokio::sync::RwLock; use uuid::Uuid; @@ -591,7 +591,9 @@ impl StorageAPI for Sets { #[tracing::instrument(skip(self))] async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.get_disks_by_key(object).restore_transitioned_object(bucket, object, opts).await + self.get_disks_by_key(object) + .restore_transitioned_object(bucket, object, opts) + .await } #[tracing::instrument(skip(self))] @@ -688,8 +690,6 @@ impl StorageAPI for Sets { .await } - - #[tracing::instrument(skip(self))] async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await diff --git a/ecstore/src/signer/mod.rs b/ecstore/src/signer/mod.rs index 105e92ca..7d87ba04 100644 --- a/ecstore/src/signer/mod.rs +++ b/ecstore/src/signer/mod.rs @@ -1,13 +1,13 @@ -pub mod utils; -pub mod request_signature_v2; -pub mod request_signature_v4; +pub mod ordered_qs; pub mod request_signature_streaming; pub mod request_signature_streaming_unsigned_trailer; -pub mod ordered_qs; +pub mod request_signature_v2; +pub mod request_signature_v4; +pub mod utils; -pub use request_signature_v2::sign_v2; +pub use request_signature_streaming::streaming_sign_v4; pub use request_signature_v2::pre_sign_v2; -pub use request_signature_v4::sign_v4; +pub use request_signature_v2::sign_v2; pub use request_signature_v4::pre_sign_v4; +pub use request_signature_v4::sign_v4; pub use request_signature_v4::sign_v4_trailer; -pub use request_signature_streaming::streaming_sign_v4; \ No newline at end of file diff --git a/ecstore/src/signer/request_signature_streaming.rs b/ecstore/src/signer/request_signature_streaming.rs index 81175b74..90a4fe66 100644 --- a/ecstore/src/signer/request_signature_streaming.rs +++ b/ecstore/src/signer/request_signature_streaming.rs @@ -1,37 +1,37 @@ -use std::pin::Pin; -use std::sync::Mutex; +use bytes::{Bytes, BytesMut}; use futures::prelude::*; use futures::task; -use bytes::{Bytes, BytesMut}; -use http::header::TRAILER; +use http::HeaderMap; use http::Uri; +use http::header::TRAILER; +use http::request::{self, Request}; +use hyper::Method; use lazy_static::lazy_static; use std::collections::HashMap; -use stdx::str::StrExt; use std::fmt::Write; -use tracing::{error, info, warn, debug}; -use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; -use http::request::{self, Request}; -use http::HeaderMap; -use hyper::Method; +use std::pin::Pin; +use std::sync::Mutex; +use stdx::str::StrExt; +use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description}; +use tracing::{debug, error, info, warn}; +use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key}; +use crate::client::constants::UNSIGNED_PAYLOAD; use rustfs_utils::{ crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256}, - hash::EMPTY_STRING_SHA256_HASH + hash::EMPTY_STRING_SHA256_HASH, }; -use crate::client::constants::UNSIGNED_PAYLOAD; -use super::request_signature_v4::{get_scope, get_signing_key, get_signature, SERVICE_TYPE_S3}; -const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; -const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD"; -const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER"; -const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024; -const CHUNK_SIGCONST_LEN: i64 = 17; -const SIGNATURESTR_LEN: i64 = 64; -const CRLF_LEN: i64 = 2; -const TRAILER_KV_SEPARATOR: &str = ":"; -const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature"; +const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER"; +const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024; +const CHUNK_SIGCONST_LEN: i64 = 17; +const SIGNATURESTR_LEN: i64 = 64; +const CRLF_LEN: i64 = 2; +const TRAILER_KV_SEPARATOR: &str = ":"; +const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature"; lazy_static! { static ref ignored_streaming_headers: HashMap = { @@ -55,17 +55,26 @@ fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &st string_to_sign_parts.join("\n") } -fn build_chunk_signature(chunk_check_sum: &str, req_time: OffsetDateTime, region: &str, - previous_signature: &str, secret_access_key: &str +fn build_chunk_signature( + chunk_check_sum: &str, + req_time: OffsetDateTime, + region: &str, + previous_signature: &str, + secret_access_key: &str, ) -> String { - let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, - previous_signature, chunk_check_sum); + let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, previous_signature, chunk_check_sum); let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3); get_signature(signing_key, &chunk_string_to_sign) } -pub fn streaming_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, - region: &str, data_len: i64, req_time: OffsetDateTime/*, sh256: md5simd.Hasher*/ +pub fn streaming_sign_v4( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + region: &str, + data_len: i64, + req_time: OffsetDateTime, /*, sh256: md5simd.Hasher*/ ) -> request::Builder { todo!(); } diff --git a/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs index 3f84aabc..6a635343 100644 --- a/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs +++ b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs @@ -1,6 +1,11 @@ use http::request; use time::OffsetDateTime; -pub fn streaming_unsigned_v4(mut req: request::Builder, session_token: &str, data_len: i64, req_time: OffsetDateTime) -> request::Builder{ +pub fn streaming_unsigned_v4( + mut req: request::Builder, + session_token: &str, + data_len: i64, + req_time: OffsetDateTime, +) -> request::Builder { todo!(); } diff --git a/ecstore/src/signer/request_signature_v2.rs b/ecstore/src/signer/request_signature_v2.rs index 8b4aa763..12aebc1d 100644 --- a/ecstore/src/signer/request_signature_v2.rs +++ b/ecstore/src/signer/request_signature_v2.rs @@ -1,15 +1,15 @@ -use std::collections::HashMap; +use bytes::{Bytes, BytesMut}; use http::request; use hyper::Uri; -use bytes::{Bytes, BytesMut}; +use std::collections::HashMap; use std::fmt::Write; -use time::{OffsetDateTime, macros::format_description, format_description}; +use time::{OffsetDateTime, format_description, macros::format_description}; use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use super::utils::get_host_addr; -const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; const SIGN_V2_ALGORITHM: &str = "AWS"; fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String { @@ -20,7 +20,13 @@ fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String { path } -pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, expires: i64, virtual_host: bool) -> request::Builder { +pub fn pre_sign_v2( + mut req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + expires: i64, + virtual_host: bool, +) -> request::Builder { if access_key_id == "" || secret_access_key == "" { return req; } @@ -50,7 +56,11 @@ pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access let uri = req.uri_ref().unwrap().clone(); let mut parts = req.uri_ref().unwrap().clone().into_parts(); - parts.path_and_query = Some(format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); + parts.path_and_query = Some( + format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature) + .parse() + .unwrap(), + ); let req = req.uri(Uri::from_parts(parts).unwrap()); req @@ -61,7 +71,13 @@ fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> S signature } -pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, virtual_host: bool) -> request::Builder { +pub fn sign_v2( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + virtual_host: bool, +) -> request::Builder { if access_key_id == "" || secret_access_key == "" { return req; } @@ -74,7 +90,14 @@ pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, let date = headers.get("Date").unwrap(); if date.to_str().unwrap() == "" { - headers.insert("Date", d2.format(&format_description::well_known::Rfc2822).unwrap().to_string().parse().unwrap()); + headers.insert( + "Date", + d2.format(&format_description::well_known::Rfc2822) + .unwrap() + .to_string() + .parse() + .unwrap(), + ); } let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id); @@ -130,21 +153,27 @@ fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) { let lk = k.as_str().to_lowercase(); if lk.starts_with("x-amz") { proto_headers.push(lk.clone()); - let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); + let vv = req + .headers_ref() + .expect("err") + .get_all(k) + .iter() + .map(|e| e.to_str().unwrap().to_string()) + .collect(); vals.insert(lk, vv); } } proto_headers.sort(); for k in proto_headers { - buf.write_str(&k); - buf.write_char(':'); - for (idx, v) in vals[&k].iter().enumerate() { - if idx > 0 { - buf.write_char(','); - } - buf.write_str(v); - } - buf.write_char('\n'); + buf.write_str(&k); + buf.write_char(':'); + for (idx, v) in vals[&k].iter().enumerate() { + if idx > 0 { + buf.write_char(','); + } + buf.write_str(v); + } + buf.write_char('\n'); } } @@ -181,7 +210,7 @@ fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virt let mut vals = result.unwrap_or_default(); for resource in INCLUDED_QUERY { let vv = &vals[*resource]; - if vv.len() > 0 { + if vv.len() > 0 { n += 1; match n { 1 => { @@ -199,4 +228,4 @@ fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virt } } } -} \ No newline at end of file +} diff --git a/ecstore/src/signer/request_signature_v4.rs b/ecstore/src/signer/request_signature_v4.rs index 29f055da..6bf91586 100644 --- a/ecstore/src/signer/request_signature_v4.rs +++ b/ecstore/src/signer/request_signature_v4.rs @@ -1,21 +1,21 @@ use bytes::{Bytes, BytesMut}; -use http::header::TRAILER; +use http::HeaderMap; use http::Uri; +use http::header::TRAILER; +use http::request::{self, Request}; use lazy_static::lazy_static; use std::collections::HashMap; use std::fmt::Write; -use tracing::{error, info, warn, debug}; -use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; -use http::request::{self, Request}; -use http::HeaderMap; +use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description}; +use tracing::{debug, error, info, warn}; -use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; -use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; -use crate::client::constants::UNSIGNED_PAYLOAD; use super::ordered_qs::OrderedQs; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::utils::stable_sort_by_first; use super::utils::{get_host_addr, sign_v4_trim_all}; +use crate::client::constants::UNSIGNED_PAYLOAD; +use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SERVICE_TYPE_S3: &str = "s3"; @@ -69,7 +69,7 @@ fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, servic fn get_hashed_payload(req: &request::Builder) -> String { let headers = req.headers_ref().unwrap(); let mut hashed_payload = ""; - if let Some(payload)= headers.get("X-Amz-Content-Sha256") { + if let Some(payload) = headers.get("X-Amz-Content-Sha256") { hashed_payload = payload.to_str().unwrap(); } if hashed_payload == "" { @@ -86,7 +86,13 @@ fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap>::new(); canonical_request.push(req.method_ref().unwrap().to_string()); canonical_request.push(req.uri_ref().unwrap().path().to_string()); @@ -178,7 +184,15 @@ fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: & string_to_sign } -pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, expires: i64, t: OffsetDateTime) -> request::Builder { +pub fn pre_sign_v4( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + expires: i64, + t: OffsetDateTime, +) -> request::Builder { if access_key_id == "" || secret_access_key == "" { return req; } @@ -208,9 +222,13 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key let uri = req.uri_ref().unwrap().clone(); let mut parts = req.uri_ref().unwrap().clone().into_parts(); - parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); let req = req.uri(Uri::from_parts(parts).unwrap()); - + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3); //println!("canonical_request: \n{}\n", canonical_request); @@ -220,7 +238,16 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key let uri = req.uri_ref().unwrap().clone(); let mut parts = req.uri_ref().unwrap().clone().into_parts(); - parts.path_and_query = Some(format!("{}?{}&X-Amz-Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); + parts.path_and_query = Some( + format!( + "{}?{}&X-Amz-Signature={}", + uri.path(), + serde_urlencoded::to_string(&query).unwrap(), + signature + ) + .parse() + .unwrap(), + ); let req = req.uri(Uri::from_parts(parts).unwrap()); req @@ -236,7 +263,16 @@ fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) } -fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, service_type: &str, trailer: HeaderMap) -> request::Builder { +fn sign_v4_inner( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + service_type: &str, + trailer: HeaderMap, +) -> request::Builder { if access_key_id == "" || secret_access_key == "" { return req; } @@ -252,7 +288,7 @@ fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &st headers.insert("X-Amz-Security-Token", session_token.parse().unwrap()); } - if trailer.len() > 0{ + if trailer.len() > 0 { for (k, _) in &trailer { headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); } @@ -276,9 +312,12 @@ fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &st let mut headers = req.headers_mut().expect("err"); - let auth = format!("{} Credential={}, SignedHeaders={}, Signature={}", SIGN_V4_ALGORITHM, credential, signed_headers, signature); + let auth = format!( + "{} Credential={}, SignedHeaders={}, Signature={}", + SIGN_V4_ALGORITHM, credential, signed_headers, signature + ); headers.insert("Authorization", auth.parse().unwrap()); - + if trailer.len() > 0 { //req.Trailer = trailer; for (_, v) in &trailer { @@ -315,12 +354,44 @@ fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: Header streaming_unsigned_v4(req, "", content_len, t); } -pub fn sign_v4(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str) -> request::Builder { - sign_v4_inner(req, content_len, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, HeaderMap::new()) +pub fn sign_v4( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, +) -> request::Builder { + sign_v4_inner( + req, + content_len, + access_key_id, + secret_access_key, + session_token, + location, + SERVICE_TYPE_S3, + HeaderMap::new(), + ) } -pub fn sign_v4_trailer(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, trailer: HeaderMap) -> request::Builder { - sign_v4_inner(req, 0, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, trailer) +pub fn sign_v4_trailer( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + trailer: HeaderMap, +) -> request::Builder { + sign_v4_inner( + req, + 0, + access_key_id, + secret_access_key, + session_token, + location, + SERVICE_TYPE_S3, + trailer, + ) } #[cfg(test)] @@ -338,10 +409,17 @@ mod tests { let service = "s3"; let path = "/"; - let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/?"); + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://examplebucket.s3.amazonaws.com/?"); let mut headers = req.headers_mut().expect("err"); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); - headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); headers.insert("x-amz-date", timestamp.parse().unwrap()); let mut query = >::new(); @@ -349,7 +427,11 @@ mod tests { query.push(("prefix".to_string(), "J".to_string())); let uri = req.uri_ref().unwrap().clone(); let mut parts = req.uri_ref().unwrap().clone().into_parts(); - parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); let req = req.uri(Uri::from_parts(parts).unwrap()); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); @@ -361,7 +443,9 @@ mod tests { "max-keys=2&prefix=J\n", "host:examplebucket.s3.amazonaws.com\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", - "x-amz-date:", "20130524T000000Z", "\n", + "x-amz-date:", + "20130524T000000Z", + "\n", "\n", "host;x-amz-content-sha256;x-amz-date\n", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -373,7 +457,8 @@ mod tests { string_to_sign, concat!( "AWS4-HMAC-SHA256\n", - "20130524T000000Z", "\n", + "20130524T000000Z", + "\n", "20130524/us-east-1/s3/aws4_request\n", "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7", ) @@ -396,17 +481,28 @@ mod tests { let service = "s3"; let path = "/mblock2/"; - let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?"); - + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://192.168.1.11:9020/mblock2/?"); + let mut headers = req.headers_mut().expect("err"); headers.insert("host", "192.168.1.11:9020".parse().unwrap()); - headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); headers.insert("x-amz-date", timestamp.parse().unwrap()); let mut query: Vec<(String, String)> = Vec::new(); let uri = req.uri_ref().unwrap().clone(); let mut parts = req.uri_ref().unwrap().clone().into_parts(); - parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); let req = req.uri(Uri::from_parts(parts).unwrap()); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); @@ -419,7 +515,9 @@ mod tests { "\n", "host:192.168.1.11:9020\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", - "x-amz-date:", "20250505T011054Z", "\n", + "x-amz-date:", + "20250505T011054Z", + "\n", "\n", "host;x-amz-content-sha256;x-amz-date\n", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -432,7 +530,8 @@ mod tests { string_to_sign, concat!( "AWS4-HMAC-SHA256\n", - "20250505T011054Z", "\n", + "20250505T011054Z", + "\n", "20250505/us-east-1/s3/aws4_request\n", "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0", ) @@ -456,10 +555,15 @@ mod tests { let path = "/mblock2/"; let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?list-type=2&encoding-type=url&prefix=mypre&delimiter=%2F&fetch-owner=true&max-keys=1"); - + let mut headers = req.headers_mut().expect("err"); headers.insert("host", "192.168.1.11:9020".parse().unwrap()); - headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); headers.insert("x-amz-date", timestamp.parse().unwrap()); /*let uri = req.uri_ref().unwrap().clone(); @@ -475,7 +579,7 @@ mod tests { let mut parts = req.uri_ref().unwrap().clone().into_parts(); parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());*/ -println!("{:?}", req.uri_ref().unwrap().query()); + println!("{:?}", req.uri_ref().unwrap().query()); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); println!("canonical_request: \n{}\n", canonical_request); assert_eq!( @@ -486,7 +590,9 @@ println!("{:?}", req.uri_ref().unwrap().query()); "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n", "host:192.168.1.11:9020\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", - "x-amz-date:", "20250507T051030Z", "\n", + "x-amz-date:", + "20250507T051030Z", + "\n", "\n", "host;x-amz-content-sha256;x-amz-date\n", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -499,7 +605,8 @@ println!("{:?}", req.uri_ref().unwrap().query()); string_to_sign, concat!( "AWS4-HMAC-SHA256\n", - "20250507T051030Z", "\n", + "20250507T051030Z", + "\n", "20250507/us-east-1/s3/aws4_request\n", "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999", ) @@ -525,8 +632,10 @@ println!("{:?}", req.uri_ref().unwrap().query()); let path = "/"; let session_token = ""; - let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/test.txt"); - + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://examplebucket.s3.amazonaws.com/test.txt"); + let mut headers = req.headers_mut().expect("err"); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); @@ -573,7 +682,7 @@ println!("{:?}", req.uri_ref().unwrap().query()); let session_token = ""; let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/test.txt?delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2"); - + let mut headers = req.headers_mut().expect("err"); headers.insert("host", "192.168.1.11:9020".parse().unwrap()); @@ -604,4 +713,4 @@ println!("{:?}", req.uri_ref().unwrap().query()); ) ); } -} \ No newline at end of file +} diff --git a/ecstore/src/signer/utils.rs b/ecstore/src/signer/utils.rs index dd73b877..61bdf63b 100644 --- a/ecstore/src/signer/utils.rs +++ b/ecstore/src/signer/utils.rs @@ -28,4 +28,4 @@ where T: Ord, { v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0)); -} \ No newline at end of file +} diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index a996a5d3..62fe0a60 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,20 +1,21 @@ #![allow(clippy::map_entry)] -use rustfs_filemeta::FileInfo; +use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry; use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::config::GLOBAL_StorageClass; use crate::config::storageclass; use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; +use crate::error::{Error, Result}; use crate::error::{ StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, to_object_err, }; use crate::global::{ DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, - GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd, - set_global_deployment_id, set_object_layer, GLOBAL_TierConfigMgr, + GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure, + is_erasure_sd, set_global_deployment_id, set_object_layer, }; use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; @@ -27,10 +28,7 @@ use crate::rebalance::RebalanceMeta; use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::{ - bucket::{ - metadata::BucketMetadata, - lifecycle::bucket_lifecycle_ops::TransitionState - }, + bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata}, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk}, endpoints::EndpointServerPools, peer::S3PeerSys, @@ -42,10 +40,6 @@ use crate::{ }, store_init, }; -use rustfs_utils::crypto::base64_decode; -use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; -use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry; -use crate::error::{Error, Result}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use futures::future::join_all; use glob::Pattern; @@ -53,7 +47,10 @@ use http::HeaderMap; use lazy_static::lazy_static; use madmin::heal_commands::HealResultItem; use rand::Rng as _; +use rustfs_filemeta::FileInfo; use rustfs_filemeta::MetaCacheEntry; +use rustfs_utils::crypto::base64_decode; +use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; use std::cmp::Ordering; use std::process::exit; @@ -332,7 +329,7 @@ impl ECStore { TransitionState::init(self.clone()).await; - if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { + if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { info!("TierConfigMgr init error: {}", err); } @@ -1868,7 +1865,9 @@ impl StorageAPI for ECStore { self.pools[0].add_partial(bucket, object.as_str(), version_id).await; } - let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()).await?; + let idx = self + .get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()) + .await?; self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; Ok(()) @@ -2134,7 +2133,9 @@ impl StorageAPI for ECStore { let object = rustfs_utils::path::encode_dir_object(object); if self.single_pool() { - return self.pools[0].delete_object_version(bucket, object.as_str(), fi, force_del_marker).await; + return self.pools[0] + .delete_object_version(bucket, object.as_str(), fi, force_del_marker) + .await; } Ok(()) } diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index b26d4178..2fac857b 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -2,22 +2,19 @@ use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::versioning::VersioningApi as _; use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::error::{Error, Result}; -use rustfs_utils::path::decode_dir_object; use crate::heal::heal_ops::HealSequence; use crate::store_utils::clean_metadata; -use crate::{disk::DiskStore, heal::heal_commands::HealOpts,}; use crate::{ - bucket::lifecycle::{ - lifecycle::TransitionOptions, - bucket_lifecycle_ops::TransitionedObject, - }, bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::ExpirationOptions, + bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions}, }; +use crate::{disk::DiskStore, heal::heal_commands::HealOpts}; use http::{HeaderMap, HeaderValue}; use madmin::heal_commands::HealResultItem; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; use rustfs_rio::{HashReader, Reader}; +use rustfs_utils::path::decode_dir_object; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt::Debug; @@ -444,15 +441,18 @@ impl ObjectInfo { // TODO:expires // TODO:ReplicationState - - let transitioned_object = TransitionedObject { - name: fi.transitioned_objname.clone(), - version_id: if let Some(transition_version_id) = fi.transition_version_id { transition_version_id.to_string() } else { "".to_string() }, - status: fi.transition_status.clone(), - free_version: fi.tier_free_version(), - tier: fi.transition_tier.clone(), - }; + let transitioned_object = TransitionedObject { + name: fi.transitioned_objname.clone(), + version_id: if let Some(transition_version_id) = fi.transition_version_id { + transition_version_id.to_string() + } else { + "".to_string() + }, + status: fi.transition_status.clone(), + free_version: fi.tier_free_version(), + tier: fi.transition_tier.clone(), + }; let metadata = { let mut v = fi.metadata.clone(); diff --git a/ecstore/src/tier/mod.rs b/ecstore/src/tier/mod.rs index 80e99e75..8c54e169 100644 --- a/ecstore/src/tier/mod.rs +++ b/ecstore/src/tier/mod.rs @@ -1,9 +1,9 @@ -pub mod warm_backend_s3; -pub mod warm_backend_minio; -pub mod warm_backend_rustfs; -pub mod warm_backend; +pub mod tier; pub mod tier_admin; pub mod tier_config; -pub mod tier; pub mod tier_gen; -pub mod tier_handlers; \ No newline at end of file +pub mod tier_handlers; +pub mod warm_backend; +pub mod warm_backend_minio; +pub mod warm_backend_rustfs; +pub mod warm_backend_s3; diff --git a/ecstore/src/tier/tier.rs b/ecstore/src/tier/tier.rs index a5afc65b..e8e54003 100644 --- a/ecstore/src/tier/tier.rs +++ b/ecstore/src/tier/tier.rs @@ -1,46 +1,49 @@ -use std::{ - collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,} -}; use bytes::Bytes; -use serde::{Serialize, Deserialize}; -use time::OffsetDateTime; -use tokio::{select, sync::RwLock, time::interval}; -use rand::Rng; -use tracing::{info, debug, warn, error}; use http::status::StatusCode; use lazy_static::lazy_static; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, hash_map::Entry}, + io::Cursor, + sync::Arc, + time::Duration, +}; +use time::OffsetDateTime; use tokio::io::BufReader; +use tokio::{select, sync::RwLock, time::interval}; +use tracing::{debug, error, info, warn}; -use s3s::S3ErrorCode; -use crate::error::{Error, Result, StorageError}; -use rustfs_utils::path::{path_join, SLASH_SEPARATOR}; -use crate::{ - config::com::{read_config, CONFIG_PREFIX}, - disk::RUSTFS_META_BUCKET, - store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI -}; use crate::client::admin_handler_utils::AdminError; -use crate::tier::{ - warm_backend::{check_warm_backend, new_warm_backend}, - tier_handlers::{ - ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, - }, - tier_admin::TierCreds, - tier_config::{TierType, TierConfig,}, -}; +use crate::error::{Error, Result, StorageError}; use crate::new_object_layer_fn; +use crate::tier::{ + tier_admin::TierCreds, + tier_config::{TierConfig, TierType}, + tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND}, + warm_backend::{check_warm_backend, new_warm_backend}, +}; +use crate::{ + StorageAPI, + config::com::{CONFIG_PREFIX, read_config}, + disk::RUSTFS_META_BUCKET, + store::ECStore, + store_api::{ObjectOptions, PutObjReader}, +}; use rustfs_rio::HashReader; +use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; +use s3s::S3ErrorCode; use super::{ - tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND}, + tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR}, warm_backend::WarmBackendImpl, }; const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); -pub const TIER_CONFIG_FILE: &str = "tier-config.json"; -pub const TIER_CONFIG_FORMAT: u16 = 1; -pub const TIER_CONFIG_V1: u16 = 1; +pub const TIER_CONFIG_FILE: &str = "tier-config.json"; +pub const TIER_CONFIG_FORMAT: u16 = 1; +pub const TIER_CONFIG_V1: u16 = 1; pub const TIER_CONFIG_VERSION: u16 = 1; lazy_static! { @@ -50,32 +53,32 @@ lazy_static! { const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError { - code: "XRustFSAdminTierMissingCredentials", - message: "Specified remote credentials are empty", + code: "XRustFSAdminTierMissingCredentials", + message: "Specified remote credentials are empty", status_code: StatusCode::FORBIDDEN, }; pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError { - code: "XRustFSAdminTierBackendInUse", - message: "Specified remote tier is already in use", + code: "XRustFSAdminTierBackendInUse", + message: "Specified remote tier is already in use", status_code: StatusCode::CONFLICT, }; pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError { - code: "XRustFSAdminTierTypeUnsupported", - message: "Specified tier type is unsupported", + code: "XRustFSAdminTierTypeUnsupported", + message: "Specified tier type is unsupported", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError { - code: "XRustFSAdminTierBackendNotEmpty", - message: "Specified remote backend is not empty", + code: "XRustFSAdminTierBackendNotEmpty", + message: "Specified remote backend is not empty", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError { - code: "XRustFSAdminTierInvalidConfig", - message: "Unable to setup remote tier, check tier configuration", + code: "XRustFSAdminTierInvalidConfig", + message: "Unable to setup remote tier, check tier configuration", status_code: StatusCode::BAD_REQUEST, }; @@ -147,22 +150,22 @@ impl TierConfigMgr { if !force { let in_use = d.in_use().await; match in_use { - Ok(b) => { - if b { - return Err(ERR_TIER_BACKEND_IN_USE); - } - } - Err(err) => { - warn!("tier add failed, err: {:?}", err); - if err.to_string().contains("connect") { - return Err(ERR_TIER_CONNECT_ERR); - } else if err.to_string().contains("authorization") { - return Err(ERR_TIER_INVALID_CREDENTIALS); - } else if err.to_string().contains("bucket") { - return Err(ERR_TIER_BUCKET_NOT_FOUND); - } - return Err(ERR_TIER_PERM_ERR); - } + Ok(b) => { + if b { + return Err(ERR_TIER_BACKEND_IN_USE); + } + } + Err(err) => { + warn!("tier add failed, err: {:?}", err); + if err.to_string().contains("connect") { + return Err(ERR_TIER_CONNECT_ERR); + } else if err.to_string().contains("authorization") { + return Err(ERR_TIER_INVALID_CREDENTIALS); + } else if err.to_string().contains("bucket") { + return Err(ERR_TIER_BUCKET_NOT_FOUND); + } + return Err(ERR_TIER_PERM_ERR); + } } } @@ -279,7 +282,7 @@ impl TierConfigMgr { minio.access_key = creds.access_key; minio.secret_key = creds.secret_key; } - _ => () + _ => (), } let d = new_warm_backend(&cfg, true).await?; @@ -290,9 +293,7 @@ impl TierConfigMgr { pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { Ok(match self.driver_cache.entry(tier_name.to_string()) { - Entry::Occupied(e) => { - e.into_mut() - } + Entry::Occupied(e) => e.into_mut(), Entry::Vacant(e) => { let t = self.tiers.get(tier_name); if t.is_none() { @@ -326,7 +327,9 @@ impl TierConfigMgr { #[tracing::instrument(level = "debug", name = "tier_save", skip(self))] pub async fn save(&self) -> std::result::Result<(), std::io::Error> { - let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) }; + let Some(api) = new_object_layer_fn() else { + return Err(std::io::Error::other("errServerNotInitialized")); + }; //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; self.save_tiering_config(api).await @@ -340,7 +343,12 @@ impl TierConfigMgr { self.save_config(api, &config_file, data).await } - pub async fn save_config(&self, api: Arc, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> { + pub async fn save_config( + &self, + api: Arc, + file: &str, + data: Bytes, + ) -> std::result::Result<(), std::io::Error> { self.save_config_with_opts( api, file, @@ -353,7 +361,13 @@ impl TierConfigMgr { .await } - pub async fn save_config_with_opts(&self, api: Arc, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> { + pub async fn save_config_with_opts( + &self, + api: Arc, + file: &str, + data: Bytes, + opts: &ObjectOptions, + ) -> std::result::Result<(), std::io::Error> { debug!("save tier config:{}", file); let _ = api .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) @@ -365,9 +379,7 @@ impl TierConfigMgr { //let r = rand.New(rand.NewSource(time.Now().UnixNano())); let mut rng = rand::rng(); let r = rng.random_range(0.0..1.0); - let rand_interval = || { - Duration::from_secs((r * 60_f64).round() as u64) - }; + let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64); let mut t = interval(TIER_CFG_REFRESH + rand_interval()); loop { @@ -394,10 +406,10 @@ impl TierConfigMgr { async fn new_and_save_tiering_config(api: Arc) -> Result { let mut cfg = TierConfigMgr { - driver_cache: HashMap::new(), - tiers: HashMap::new(), - last_refreshed_at: OffsetDateTime::now_utc(), - }; + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; //lookup_configs(&mut cfg, api.clone()).await; cfg.save_tiering_config(api).await?; @@ -420,7 +432,7 @@ async fn load_tier_config(api: Arc) -> std::result::Result { cfg = match TierConfigMgr::unmarshal(&data.unwrap()) { @@ -440,4 +452,4 @@ async fn load_tier_config(api: Arc) -> std::result::Result bool { matches!(err, StorageError::ObjectNotFound(_, _)) -} \ No newline at end of file +} diff --git a/ecstore/src/tier/tier_admin.rs b/ecstore/src/tier/tier_admin.rs index ee7c2cc5..699e4b26 100644 --- a/ecstore/src/tier/tier_admin.rs +++ b/ecstore/src/tier/tier_admin.rs @@ -1,13 +1,10 @@ -use std::{ - time::{Duration, SystemTime, UNIX_EPOCH}, -}; -use rand::Rng; -use tracing::warn; use http::status::StatusCode; -use serde::{Serialize, Deserialize}; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tracing::warn; -#[derive(Serialize, Deserialize)] -#[derive(Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierCreds { #[serde(rename = "accessKey")] @@ -26,4 +23,4 @@ pub struct TierCreds { //#[serde(rename = "credsJson")] pub creds_json: Vec, -} \ No newline at end of file +} diff --git a/ecstore/src/tier/tier_config.rs b/ecstore/src/tier/tier_config.rs index 518338ec..b6e2f631 100644 --- a/ecstore/src/tier/tier_config.rs +++ b/ecstore/src/tier/tier_config.rs @@ -1,5 +1,5 @@ +use serde::{Deserialize, Serialize}; use std::fmt::Display; -use serde::{Serialize, Deserialize}; use tracing::info; const C_TierConfigVer: &str = "v1"; @@ -9,8 +9,7 @@ const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config"; const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version"; const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type"; -#[derive(Serialize, Deserialize)] -#[derive(Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] pub enum TierType { #[default] Unsupported, @@ -48,35 +47,19 @@ impl Display for TierType { impl TierType { pub fn new(sc_type: &str) -> Self { match sc_type { - "S3" => { - TierType::S3 - } - "RustFS" => { - TierType::RustFS - } - "MinIO" => { - TierType::MinIO - } - _ => { - TierType::Unsupported - } + "S3" => TierType::S3, + "RustFS" => TierType::RustFS, + "MinIO" => TierType::MinIO, + _ => TierType::Unsupported, } } pub fn to_string(&self) -> String { match self { - TierType::S3 => { - "s3".to_string() - } - TierType::RustFS => { - "rustfs".to_string() - } - TierType::MinIO => { - "minio".to_string() - } - _ => { - "unsupported".to_string() - } + TierType::S3 => "s3".to_string(), + TierType::RustFS => "rustfs".to_string(), + TierType::MinIO => "minio".to_string(), + _ => "unsupported".to_string(), } } } @@ -123,17 +106,17 @@ impl Clone for TierConfig { m_.secret_key = "REDACTED".to_string(); m = Some(m_); } - _ => () + _ => (), } TierConfig { version: self.version.clone(), tier_type: self.tier_type.clone(), - name: self.name.clone(), + name: self.name.clone(), s3: s3, //azure: az, //gcs: gcs, - rustfs: r, - minio: m, + rustfs: r, + minio: m, } } } @@ -154,15 +137,9 @@ impl TierConfig { fn endpoint(&self) -> String { match self.tier_type { - TierType::S3 => { - self.s3.as_ref().expect("err").endpoint.clone() - } - TierType::RustFS => { - self.rustfs.as_ref().expect("err").endpoint.clone() - } - TierType::MinIO => { - self.minio.as_ref().expect("err").endpoint.clone() - } + TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(), _ => { info!("unexpected tier type {}", self.tier_type); "".to_string() @@ -172,15 +149,9 @@ impl TierConfig { fn bucket(&self) -> String { match self.tier_type { - TierType::S3 => { - self.s3.as_ref().expect("err").bucket.clone() - } - TierType::RustFS => { - self.rustfs.as_ref().expect("err").bucket.clone() - } - TierType::MinIO => { - self.minio.as_ref().expect("err").bucket.clone() - } + TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(), _ => { info!("unexpected tier type {}", self.tier_type); "".to_string() @@ -190,15 +161,9 @@ impl TierConfig { fn prefix(&self) -> String { match self.tier_type { - TierType::S3 => { - self.s3.as_ref().expect("err").prefix.clone() - } - TierType::RustFS => { - self.rustfs.as_ref().expect("err").prefix.clone() - } - TierType::MinIO => { - self.minio.as_ref().expect("err").prefix.clone() - } + TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(), _ => { info!("unexpected tier type {}", self.tier_type); "".to_string() @@ -208,18 +173,12 @@ impl TierConfig { fn region(&self) -> String { match self.tier_type { - TierType::S3 => { - self.s3.as_ref().expect("err").region.clone() - } - TierType::RustFS => { - self.rustfs.as_ref().expect("err").region.clone() - } - TierType::MinIO => { - self.minio.as_ref().expect("err").region.clone() - } + TierType::S3 => self.s3.as_ref().expect("err").region.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").region.clone(), _ => { info!("unexpected tier type {}", self.tier_type); - "".to_string() + "".to_string() } } } @@ -227,8 +186,7 @@ impl TierConfig { //type S3Options = impl Fn(TierS3) -> Pin>> + Send + Sync + 'static; -#[derive(Serialize, Deserialize)] -#[derive(Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierS3 { pub name: String, @@ -257,17 +215,17 @@ pub struct TierS3 { impl TierS3 { fn new(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec) -> Result where - F: Fn(TierS3) -> Box> + Send + Sync + 'static + F: Fn(TierS3) -> Box> + Send + Sync + 'static, { if name == "" { return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); } let sc = TierS3 { - access_key: access_key.to_string(), - secret_key: secret_key.to_string(), - bucket: bucket.to_string(), - endpoint: "https://s3.amazonaws.com".to_string(), - region: "".to_string(), + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + endpoint: "https://s3.amazonaws.com".to_string(), + region: "".to_string(), storage_class: "".to_string(), ..Default::default() }; @@ -281,49 +239,54 @@ impl TierS3 { Ok(TierConfig { version: C_TierConfigVer.to_string(), tier_type: TierType::S3, - name: name.to_string(), - s3: Some(sc), + name: name.to_string(), + s3: Some(sc), ..Default::default() }) } } -#[derive(Serialize, Deserialize)] -#[derive(Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierRustFS { pub name: String, - pub endpoint: String, + pub endpoint: String, #[serde(rename = "accesskey")] pub access_key: String, #[serde(rename = "secretkey")] pub secret_key: String, - pub bucket: String, - pub prefix: String, - pub region: String, + pub bucket: String, + pub prefix: String, + pub region: String, #[serde(rename = "storageclass")] pub storage_class: String, } -#[derive(Serialize, Deserialize)] -#[derive(Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierMinIO { pub name: String, - pub endpoint: String, + pub endpoint: String, #[serde(rename = "accesskey")] pub access_key: String, #[serde(rename = "secretkey")] pub secret_key: String, - pub bucket: String, - pub prefix: String, - pub region: String, + pub bucket: String, + pub prefix: String, + pub region: String, } impl TierMinIO { - fn new(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec) -> Result + fn new( + name: &str, + endpoint: &str, + access_key: &str, + secret_key: &str, + bucket: &str, + options: Vec, + ) -> Result where - F: Fn(TierMinIO) -> Box> + Send + Sync + 'static + F: Fn(TierMinIO) -> Box> + Send + Sync + 'static, { if name == "" { return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); @@ -331,8 +294,8 @@ impl TierMinIO { let m = TierMinIO { access_key: access_key.to_string(), secret_key: secret_key.to_string(), - bucket: bucket.to_string(), - endpoint: endpoint.to_string(), + bucket: bucket.to_string(), + endpoint: endpoint.to_string(), ..Default::default() }; @@ -345,8 +308,8 @@ impl TierMinIO { Ok(TierConfig { version: C_TierConfigVer.to_string(), tier_type: TierType::MinIO, - name: name.to_string(), - minio: Some(m), + name: name.to_string(), + minio: Some(m), ..Default::default() }) } diff --git a/ecstore/src/tier/tier_handlers.rs b/ecstore/src/tier/tier_handlers.rs index 491dd2ea..1f4ff510 100644 --- a/ecstore/src/tier/tier_handlers.rs +++ b/ecstore/src/tier/tier_handlers.rs @@ -1,51 +1,51 @@ use crate::client::admin_handler_utils::AdminError; -use tracing::warn; use http::status::StatusCode; +use tracing::warn; pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { - code: "XRustFSAdminTierAlreadyExists", - message: "Specified remote tier already exists", + code: "XRustFSAdminTierAlreadyExists", + message: "Specified remote tier already exists", status_code: StatusCode::CONFLICT, }; pub const ERR_TIER_NOT_FOUND: AdminError = AdminError { - code: "XRustFSAdminTierNotFound", - message: "Specified remote tier was not found", + code: "XRustFSAdminTierNotFound", + message: "Specified remote tier was not found", status_code: StatusCode::NOT_FOUND, }; pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError { - code: "XRustFSAdminTierNameNotUpperCase", - message: "Tier name must be in uppercase", + code: "XRustFSAdminTierNameNotUpperCase", + message: "Tier name must be in uppercase", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError { - code: "XRustFSAdminTierBucketNotFound", - message: "Remote tier bucket not found", + code: "XRustFSAdminTierBucketNotFound", + message: "Remote tier bucket not found", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError { - code: "XRustFSAdminTierInvalidCredentials", - message: "Invalid remote tier credentials", + code: "XRustFSAdminTierInvalidCredentials", + message: "Invalid remote tier credentials", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError { - code: "XRustFSAdminTierReserved", - message: "Cannot use reserved tier name", + code: "XRustFSAdminTierReserved", + message: "Cannot use reserved tier name", status_code: StatusCode::BAD_REQUEST, }; pub const ERR_TIER_PERM_ERR: AdminError = AdminError { - code: "TierPermErr", - message: "Tier Perm Err", + code: "TierPermErr", + message: "Tier Perm Err", status_code: StatusCode::OK, }; pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError { - code: "TierConnectErr", - message: "Tier Connect Err", + code: "TierConnectErr", + message: "Tier Connect Err", status_code: StatusCode::OK, -}; \ No newline at end of file +}; diff --git a/ecstore/src/tier/warm_backend.rs b/ecstore/src/tier/warm_backend.rs index 7fea5f3c..4bee0568 100644 --- a/ecstore/src/tier/warm_backend.rs +++ b/ecstore/src/tier/warm_backend.rs @@ -1,20 +1,20 @@ -use std::collections::HashMap; use bytes::Bytes; +use std::collections::HashMap; use crate::client::{ admin_handler_utils::AdminError, - transition_api::{ReadCloser, ReaderImpl,}, + transition_api::{ReadCloser, ReaderImpl}, }; use crate::error::is_err_bucket_not_found; -use tracing::{info, warn}; use crate::tier::{ - tier_config::{TierType, TierConfig}, + tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, + tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR}, - tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,}, - warm_backend_s3::WarmBackendS3, - warm_backend_rustfs::WarmBackendRustFS, warm_backend_minio::WarmBackendMinIO, + warm_backend_rustfs::WarmBackendRustFS, + warm_backend_s3::WarmBackendS3, }; +use tracing::{info, warn}; pub type WarmBackendImpl = Box; @@ -29,7 +29,13 @@ pub struct WarmBackendGetOpts { #[async_trait::async_trait] pub trait WarmBackend { async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result; - async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result; + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result; async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result; async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>; async fn in_use(&self) -> Result; @@ -37,7 +43,9 @@ pub trait WarmBackend { pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { let w = w.expect("err"); - let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await; + let remote_version_id = w + .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5) + .await; if let Err(err) = remote_version_id { return Err(ERR_TIER_PERM_ERR); } @@ -52,7 +60,7 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin return Err(ERR_TIER_MISSING_CREDENTIALS); }*/ //else { - return Err(ERR_TIER_PERM_ERR); + return Err(ERR_TIER_PERM_ERR); //} } if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await { @@ -94,4 +102,4 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result) -> Result { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { let part_size = optimal_part_size(length)?; let client = self.0.client.clone(); - let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { - storage_class: self.0.storage_class.clone(), - part_size: part_size as u64, - disable_content_sha256: true, - user_metadata: meta, - ..Default::default() - }).await?; + let res = client + .put_object( + &self.0.bucket, + &self.0.get_dest(object), + r, + length, + &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }, + ) + .await?; //self.ToObjectError(err, object) Ok(res.version_id) } diff --git a/ecstore/src/tier/warm_backend_rustfs.rs b/ecstore/src/tier/warm_backend_rustfs.rs index 33f463ec..f0e12bb6 100644 --- a/ecstore/src/tier/warm_backend_rustfs.rs +++ b/ecstore/src/tier/warm_backend_rustfs.rs @@ -4,9 +4,9 @@ use tracing::warn; use crate::client::{ admin_handler_utils::AdminError, - transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore}, - credentials::{Credentials, SignatureType, Static, Value}, api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; use crate::tier::{ tier_config::TierRustFS, @@ -15,9 +15,9 @@ use crate::tier::{ }; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; -const MAX_PARTS_COUNT: i64 = 10000; -const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; -const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; +const MAX_PARTS_COUNT: i64 = 10000; +const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; pub struct WarmBackendRustFS(WarmBackendS3); @@ -37,26 +37,23 @@ impl WarmBackendRustFS { }; let creds = Credentials::new(Static(Value { - access_key_id: conf.access_key.clone(), + access_key_id: conf.access_key.clone(), secret_access_key: conf.secret_key.clone(), - session_token: "".to_string(), + session_token: "".to_string(), signer_type: SignatureType::SignatureV4, ..Default::default() })); let opts = Options { - creds: creds, - secure: u.scheme() == "https", + creds: creds, + secure: u.scheme() == "https", //transport: GLOBAL_RemoteTargetTransport, trailing_headers: true, ..Default::default() }; let scheme = u.scheme(); - let default_port = if scheme == "https" { - 443 - } else { - 80 - }; - let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; + let default_port = if scheme == "https" { 443 } else { 80 }; + let client = + TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag); let client = Arc::new(client); @@ -64,8 +61,8 @@ impl WarmBackendRustFS { Ok(Self(WarmBackendS3 { client, core, - bucket: conf.bucket.clone(), - prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), + bucket: conf.bucket.clone(), + prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), storage_class: "".to_string(), })) } @@ -73,16 +70,30 @@ impl WarmBackendRustFS { #[async_trait::async_trait] impl WarmBackend for WarmBackendRustFS { - async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { let part_size = optimal_part_size(length)?; let client = self.0.client.clone(); - let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { - storage_class: self.0.storage_class.clone(), - part_size: part_size as u64, - disable_content_sha256: true, - user_metadata: meta, - ..Default::default() - }).await?; + let res = client + .put_object( + &self.0.bucket, + &self.0.get_dest(object), + r, + length, + &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }, + ) + .await?; //self.ToObjectError(err, object) Ok(res.version_id) } diff --git a/ecstore/src/tier/warm_backend_s3.rs b/ecstore/src/tier/warm_backend_s3.rs index 2d04da51..1dda71ac 100644 --- a/ecstore/src/tier/warm_backend_s3.rs +++ b/ecstore/src/tier/warm_backend_s3.rs @@ -2,21 +2,21 @@ use std::collections::HashMap; use std::sync::Arc; use url::Url; -use crate::error::ErrorResponse; -use crate::error::error_resp_to_object_err; use crate::client::{ api_get_options::GetObjectOptions, - credentials::{Credentials, Static, Value, SignatureType}, - transition_api::{ReaderImpl, ReadCloser}, api_put_object::PutObjectOptions, api_remove::RemoveObjectOptions, - transition_api::{Options, TransitionClient, TransitionCore,}, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, TransitionClient, TransitionCore}, + transition_api::{ReadCloser, ReaderImpl}, }; -use rustfs_utils::path::SLASH_SEPARATOR; +use crate::error::ErrorResponse; +use crate::error::error_resp_to_object_err; use crate::tier::{ tier_config::TierS3, - warm_backend::{WarmBackend, WarmBackendGetOpts,} + warm_backend::{WarmBackend, WarmBackendGetOpts}, }; +use rustfs_utils::path::SLASH_SEPARATOR; pub struct WarmBackendS3 { pub client: Arc, @@ -35,16 +35,22 @@ impl WarmBackendS3 { } }; - if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" { + if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" + || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" + { return Err(std::io::Error::other("both the token file and the role ARN are required")); - } - else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" { + } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" { return Err(std::io::Error::other("both the access and secret keys are required")); - } - else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") { - return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file")); - } - else if conf.bucket == "" { + } else if conf.aws_role + && (conf.aws_role_web_identity_token_file != "" + || conf.aws_role_arn != "" + || conf.access_key != "" + || conf.secret_key != "") + { + return Err(std::io::Error::other( + "AWS Role cannot be activated with static credentials or the web identity token file", + )); + } else if conf.bucket == "" { return Err(std::io::Error::other("no bucket name was provided")); } @@ -53,19 +59,18 @@ impl WarmBackendS3 { if conf.access_key != "" && conf.secret_key != "" { //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, ""); creds = Credentials::new(Static(Value { - access_key_id: conf.access_key.clone(), + access_key_id: conf.access_key.clone(), secret_access_key: conf.secret_key.clone(), - session_token: "".to_string(), + session_token: "".to_string(), signer_type: SignatureType::SignatureV4, ..Default::default() })); - } - else { + } else { return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); } let opts = Options { - creds: creds, - secure: u.scheme() == "https", + creds: creds, + secure: u.scheme() == "https", //transport: GLOBAL_RemoteTargetTransport, ..Default::default() }; @@ -77,8 +82,8 @@ impl WarmBackendS3 { Ok(Self { client, core, - bucket: conf.bucket.clone(), - prefix: conf.prefix.clone().trim_matches('/').to_string(), + bucket: conf.bucket.clone(), + prefix: conf.prefix.clone().trim_matches('/').to_string(), storage_class: conf.storage_class.clone(), }) } @@ -103,14 +108,28 @@ impl WarmBackendS3 { #[async_trait::async_trait] impl WarmBackend for WarmBackendS3 { - async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { let client = self.client.clone(); - let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions { - send_content_md5: true, - storage_class: self.storage_class.clone(), - user_metadata: meta, - ..Default::default() - }).await?; + let res = client + .put_object( + &self.bucket, + &self.get_dest(object), + r, + length, + &PutObjectOptions { + send_content_md5: true, + storage_class: self.storage_class.clone(), + user_metadata: meta, + ..Default::default() + }, + ) + .await?; Ok(res.version_id) } @@ -125,7 +144,7 @@ impl WarmBackend for WarmBackendS3 { gopts.version_id = rv.to_string(); } if opts.start_offset >= 0 && opts.length > 0 { - if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset+opts.length-1) { + if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) { return Err(std::io::Error::other(err)); } } @@ -146,8 +165,11 @@ impl WarmBackend for WarmBackendS3 { } async fn in_use(&self) -> Result { - let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?; + let result = self + .core + .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1) + .await?; Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) } -} \ No newline at end of file +} diff --git a/reader/src/hasher.rs b/reader/src/hasher.rs index 6366a64c..5f86a52f 100644 --- a/reader/src/hasher.rs +++ b/reader/src/hasher.rs @@ -1,7 +1,7 @@ use md5::{Digest as Md5Digest, Md5}; use sha2::{ - digest::{Reset, Update}, Digest, Sha256 as sha_sha256, + digest::{Reset, Update}, }; pub trait Hasher { fn write(&mut self, bytes: &[u8]); @@ -181,4 +181,4 @@ pub fn sum_md5_base64(data: &[u8]) -> String { let mut hash = MD5::new(); hash.write(data); base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) -} \ No newline at end of file +} diff --git a/reader/src/reader.rs b/reader/src/reader.rs index 2f731d52..e68a64ae 100644 --- a/reader/src/reader.rs +++ b/reader/src/reader.rs @@ -13,9 +13,9 @@ use crate::{ }; // use futures::stream::Stream; -use std::io::{Error, Result}; -use super::hasher::{Hasher, Sha256, MD5}; +use super::hasher::{Hasher, MD5, Sha256}; use futures::Stream; +use std::io::{Error, Result}; pin_project_lite::pin_project! { #[derive(Default)] @@ -101,20 +101,8 @@ pin_project_lite::pin_project! { impl HashReader { pub fn new(inner: S, size: usize, md5_hex: Option, sha256_hex: Option, actual_size: usize) -> Self { - let md5 = { - if md5_hex.is_some() { - Some(MD5::new()) - } else { - None - } - }; - let sha256 = { - if sha256_hex.is_some() { - Some(Sha256::new()) - } else { - None - } - }; + let md5 = { if md5_hex.is_some() { Some(MD5::new()) } else { None } }; + let sha256 = { if sha256_hex.is_some() { Some(Sha256::new()) } else { None } }; Self { inner, size, diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index 1e2caf17..31a21581 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -63,9 +63,9 @@ pub mod pools; pub mod rebalance; pub mod service_account; pub mod sts; +pub mod tier; pub mod trace; pub mod user; -pub mod tier; use urlencoding::decode; #[derive(Debug, Serialize, Default)] diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 456a64d8..a11a92e7 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -3,28 +3,28 @@ use std::str::from_utf8; use http::{HeaderMap, StatusCode}; use iam::get_global_action_cred; use matchit::Params; -use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; use serde_urlencoded::from_bytes; -use tracing::{warn, debug, info}; +use tracing::{debug, info, warn}; -use crypto::{encrypt_data, decrypt_data}; use crate::{ admin::{router::Operation, utils::has_space_be}, auth::{check_key_valid, get_session_token}, }; +use crypto::{decrypt_data, encrypt_data}; use ecstore::{ client::admin_handler_utils::AdminError, config::storageclass, global::GLOBAL_TierConfigMgr, tier::{ + tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS}, tier_admin::TierCreds, tier_config::{TierConfig, TierType}, tier_handlers::{ - ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, ERR_TIER_CONNECT_ERR, - ERR_TIER_INVALID_CREDENTIALS, + ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, + ERR_TIER_NOT_FOUND, }, - tier::{ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_BACKEND_IN_USE}, }, }; @@ -92,29 +92,44 @@ impl Operation for AddTier { } match args.name.as_str() { storageclass::STANDARD | storageclass::RRS => { - warn!("tier reserved name, args.name: {}", args.name); - return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); + warn!("tier reserved name, args.name: {}", args.name); + return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); } - &_ => () + &_ => (), } let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; //tier_config_mgr.reload(api); match tier_config_mgr.add(args, force).await { Err(ERR_TIER_ALREADY_EXISTS) => { - return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameAlreadyExist".into()), "tier name already exists!")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameAlreadyExist".into()), + "tier name already exists!", + )); } Err(ERR_TIER_NAME_NOT_UPPERCASE) => { - return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameNotUppercase".into()), "tier name not uppercase!")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameNotUppercase".into()), + "tier name not uppercase!", + )); } Err(ERR_TIER_BACKEND_IN_USE) => { - return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse!".into()), "tier name backend in use!")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameBackendInUse!".into()), + "tier name backend in use!", + )); } Err(ERR_TIER_CONNECT_ERR) => { - return Err(S3Error::with_message(S3ErrorCode::Custom("TierConnectError".into()), "tier connect error!")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierConnectError".into()), + "tier connect error!", + )); } Err(ERR_TIER_INVALID_CREDENTIALS) => { - return Err(S3Error::with_message(S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), ERR_TIER_INVALID_CREDENTIALS.message)); + return Err(S3Error::with_message( + S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), + ERR_TIER_INVALID_CREDENTIALS.message, + )); } Err(e) => { warn!("tier_config_mgr add failed, e: {:?}", e); @@ -182,7 +197,10 @@ impl Operation for EditTier { return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); } Err(ERR_TIER_MISSING_CREDENTIALS) => { - return Err(S3Error::with_message(S3ErrorCode::Custom("TierMissingCredentials".into()), "tier missing credentials!")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierMissingCredentials".into()), + "tier missing credentials!", + )); } Err(e) => { warn!("tier_config_mgr edit failed, e: {:?}", e); @@ -281,7 +299,10 @@ impl Operation for RemoveTier { } Err(e) => { warn!("tier_config_mgr remove failed, e: {:?}", e); - return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier remove failed")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierRemoveFailed".into()), + "tier remove failed", + )); } Ok(_) => (), } diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 05ae8e13..3489d5e3 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -7,7 +7,7 @@ pub mod utils; use handlers::{ group, policys, pools, rebalance, service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount}, - sts, user, tier, + sts, tier, user, }; use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler}; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 0385b53b..8456404b 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -49,11 +49,11 @@ use ecstore::store_api::ObjectToDelete; use ecstore::store_api::PutObjReader; use ecstore::store_api::StorageAPI; // use ecstore::store_api::RESERVED_METADATA_PREFIX; +use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; use ecstore::bucket::utils::serialize; use ecstore::cmd::bucket_replication::ReplicationStatusType; use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; -use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; use futures::pin_mut; use futures::{Stream, StreamExt}; use http::HeaderMap; @@ -96,10 +96,7 @@ use transform_stream::AsyncTryStream; use uuid::Uuid; use ecstore::bucket::{ - lifecycle::{ - lifecycle::Lifecycle, - bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS - }, + lifecycle::{bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS, lifecycle::Lifecycle}, object_lock::objectlock_sys::BucketObjectLockSys, }; @@ -1602,7 +1599,10 @@ impl S3 for FS { let rcfg = metadata_sys::get_object_lock_config(&bucket).await; if rcfg.is_err() { - return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockIsNotExist".into()), "bucket lock is not exist.")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("BucketLockIsNotExist".into()), + "bucket lock is not exist.", + )); } let rcfg = rcfg.expect("get_lifecycle_config err!").0; @@ -1612,12 +1612,18 @@ impl S3 for FS { if let Err(err) = input_cfg.validate(&rcfg).await { //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed.")); - return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), format!("{}", err.to_string()))); + return Err(S3Error::with_message( + S3ErrorCode::Custom("ValidateFailed".into()), + format!("{}", err.to_string()), + )); } - + if let Err(err) = validate_transition_tier(&input_cfg).await { //warn!("lifecycle_configuration add failed, err: {:?}", err); - return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), format!("{}", err.to_string()))); + return Err(S3Error::with_message( + S3ErrorCode::Custom("CustomError".into()), + format!("{}", err.to_string()), + )); } let data = try_!(serialize(&input_cfg)); @@ -2070,7 +2076,10 @@ impl S3 for FS { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - if let Err(e) = store.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()).await { + if let Err(e) = store + .get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()) + .await + { return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); }