diff --git a/.vscode/launch.json b/.vscode/launch.json index 28578442..0fb590e8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,33 +20,19 @@ } }, "env": { - "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug", - "RUSTFS_VOLUMES": "./target/volume/test{0...3}", - "RUSTFS_ADDRESS": "[::]:9000", - "RUSTFS_CONSOLE_ENABLE": "true", - "RUSTFS_CONSOLE_ADDRESS": "[::]:9002", - "RUSTFS_SERVER_DOMAINS": "localhost:9000", - "RUSTFS_TLS_PATH": "./deploy/certs", - "RUSTFS_OBS_CONFIG": "./deploy/config/obs.example.toml", - "RUSTFS__OBSERVABILITY__ENDPOINT": "http://localhost:4317", - "RUSTFS__OBSERVABILITY__USE_STDOUT": "true", - "RUSTFS__OBSERVABILITY__SAMPLE_RATIO": "2.0", - "RUSTFS__OBSERVABILITY__METER_INTERVAL": "30", - "RUSTFS__OBSERVABILITY__SERVICE_NAME": "rustfs", - "RUSTFS__OBSERVABILITY__SERVICE_VERSION": "0.1.0", - "RUSTFS__OBSERVABILITY__ENVIRONMENT": "develop", - "RUSTFS__OBSERVABILITY__LOGGER_LEVEL": "info", - "RUSTFS__SINKS__FILE__ENABLED": "true", - "RUSTFS__SINKS__FILE__PATH": "./deploy/logs/rustfs.log", - "RUSTFS__SINKS__WEBHOOK__ENABLED": "false", - "RUSTFS__SINKS__WEBHOOK__ENDPOINT": "", - "RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN": "", - "RUSTFS__SINKS__KAFKA__ENABLED": "false", - "RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS": "", - "RUSTFS__SINKS__KAFKA__TOPIC": "", - "RUSTFS__LOGGER__QUEUE_CAPACITY": "10" - + "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug" }, + "args": [ + "--access-key", + "AKEXAMPLERUSTFS", + "--secret-key", + "SKEXAMPLERUSTFS", + "--address", + "0.0.0.0:9010", + "--domain-name", + "127.0.0.1:9010", + "./target/volume/test{0...4}" + ], "cwd": "${workspaceFolder}" }, { @@ -86,6 +72,19 @@ }, "args": [], "cwd": "${workspaceFolder}" + }, + { + "name": "Debug executable target/debug/rustfs", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/target/debug/rustfs", + "args": [], + "cwd": "${workspaceFolder}", + //"stopAtEntry": false, + //"preLaunchTask": "cargo build", + "sourceLanguages": [ + "rust" + ], } ] } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 09812efc..f4722112 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3612,6 +3612,7 @@ dependencies = [ name = "ecstore" version = "0.0.1" dependencies = [ + "async-channel", "async-trait", "aws-sdk-s3", "backon", @@ -3624,18 +3625,25 @@ dependencies = [ "common", "crc32fast", "criterion", + "enumset", "flatbuffers 25.2.10", "futures", "glob", "hex-simd", "highway", + "hmac 0.12.1", "http 1.3.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls 0.27.7", + "hyper-util", "lazy_static", "lock", "madmin", "md-5", "netif", "nix 0.30.1", + "num", "num_cpus", "once_cell", "path-absolutize", @@ -3644,6 +3652,7 @@ dependencies = [ "policy", "protos", "rand 0.9.1", + "reader", "reed-solomon-erasure", "reed-solomon-simd", "regex", @@ -3655,13 +3664,18 @@ dependencies = [ "rustfs-rio", "rustfs-rsc", "rustfs-utils", + "rustls 0.23.27", "s3s", "serde", + "serde-xml-rs 0.8.1", "serde_json", - "sha2 0.11.0-pre.5", + "serde_urlencoded", + "sha1 0.10.6", + "sha2 0.10.9", "shadow-rs", "siphasher 1.0.1", "smallvec", + "std-next", "tempfile", "thiserror 2.0.12", "time", @@ -3669,6 +3683,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tonic", + "tower", "tracing", "tracing-error", "transform-stream", @@ -4852,6 +4867,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", + "log", "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", @@ -7832,6 +7848,25 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "reader" +version = "0.0.1" +dependencies = [ + "async-trait", + "base64-simd", + "bytes", + "common", + "futures", + "hex-simd", + "md-5", + "pin-project-lite", + "s3s", + "sha2 0.11.0-pre.5", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "readme-rustdocifier" version = "0.1.1" @@ -8362,6 +8397,7 @@ dependencies = [ "rmp", "rmp-serde", "rustfs-utils", + "s3s", "serde", "thiserror 2.0.12", "time", @@ -8473,7 +8509,7 @@ dependencies = [ "regex", "reqwest", "serde", - "serde-xml-rs", + "serde-xml-rs 0.6.0", "sha2 0.10.9", "urlencoding", ] @@ -8484,9 +8520,13 @@ version = "0.0.1" dependencies = [ "base64-simd", "blake3", + "common", "crc32fast", "hex-simd", "highway", + "hmac 0.12.1", + "hyper 1.6.0", + "hyper-util", "lazy_static", "local-ip-address", "md-5", @@ -8497,7 +8537,9 @@ dependencies = [ "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", + "s3s", "serde", + "sha1 0.10.6", "sha2 0.10.9", "siphasher 1.0.1", "tempfile", @@ -8908,6 +8950,18 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "serde-xml-rs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53630160a98edebde0123eb4dfd0fce6adff091b2305db3154a9e920206eb510" +dependencies = [ + "log", + "serde", + "thiserror 1.0.69", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.219" diff --git a/Cargo.toml b/Cargo.toml index 96a1b07f..04e38435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ members = [ "crates/zip", "crates/filemeta", "crates/rio", - + "reader", ] resolver = "2" @@ -61,6 +61,7 @@ rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" } rustfs-disk = { path = "crates/disk", version = "0.0.1" } rustfs-error = { path = "crates/error", version = "0.0.1" } workers = { path = "./common/workers", version = "0.0.1" } +reader = { path = "./reader", version = "0.0.1" } aes-gcm = { version = "0.10.3", features = ["std"] } arc-swap = "1.7.1" argon2 = { version = "0.5.3", features = ["std"] } @@ -105,6 +106,7 @@ hyper-util = { version = "0.1.14", features = [ "server-auto", "server-graceful", ] } +hyper-rustls = "0.27.5" http = "1.3.1" http-body = "1.0.1" humantime = "2.2.0" @@ -189,9 +191,13 @@ scopeguard = "1.2.0" shadow-rs = { version = "1.1.1", default-features = false } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +serde-xml-rs = "0.8.1" serde_urlencoded = "0.7.1" serde_with = "3.12.0" +sha1 = "0.10.6" sha2 = "0.10.9" +hmac = "0.12.1" +std-next = "0.1.8" siphasher = "1.0.1" smallvec = { version = "1.15.1", features = ["serde"] } snafu = "0.8.6" @@ -215,6 +221,7 @@ tokio-rustls = { version = "0.26.2", default-features = false } tokio-stream = { version = "0.1.17" } tokio-tar = "0.3.1" tokio-util = { version = "0.7.15", features = ["io", "compat"] } +async-channel = "2.3.1" tower = { version = "0.5.2", features = ["timeout"] } tower-http = { version = "0.6.6", features = ["cors"] } tracing = "0.1.41" diff --git a/common/lock/src/local_locker.rs b/common/lock/src/local_locker.rs index 3e50d0a2..3ab4c737 100644 --- a/common/lock/src/local_locker.rs +++ b/common/lock/src/local_locker.rs @@ -7,7 +7,7 @@ use std::{ use crate::{Locker, lock_args::LockArgs}; -const MAX_DELETE_LIST: usize = 1000; +pub const MAX_DELETE_LIST: usize = 1000; #[derive(Clone, Debug)] struct LockRequesterInfo { diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index 7f92441e..0e96bbd0 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -21,6 +21,8 @@ byteorder = "1.5.0" tracing.workspace = true thiserror.workspace = true +s3s.workspace = true + [dev-dependencies] criterion = { version = "0.5", features = ["html_reports"] } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 1ae3c5af..61a26f22 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -18,6 +18,10 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M pub const NULL_VERSION_ID: &str = "null"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; +pub const TIER_FV_ID: &str = "tier-free-versionID"; +pub const TIER_FV_MARKER: &str = "tier-free-marker"; +pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; + #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct ObjectPartInfo { pub etag: String, @@ -147,11 +151,10 @@ pub struct FileInfo { pub version_id: Option, pub is_latest: bool, pub deleted: bool, - // Transition related fields - pub transition_status: Option, - pub transitioned_obj_name: Option, - pub transition_tier: Option, - pub transition_version_id: Option, + pub transition_status: String, + pub transitioned_objname: String, + pub transition_tier: String, + pub transition_version_id: Option, pub expire_restored: bool, pub data_dir: Option, pub mod_time: Option, @@ -220,7 +223,11 @@ impl FileInfo { } pub fn get_etag(&self) -> Option { - self.metadata.get("etag").cloned() + if let Some(meta) = self.metadata.get("etag") { + Some(meta.clone()) + } else { + None + } } pub fn write_quorum(&self, quorum: usize) -> usize { @@ -301,6 +308,30 @@ impl FileInfo { self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string()); } + pub fn set_tier_free_version_id(&mut self, version_id: &str) { + self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); + } + + pub fn tier_free_version_id(&self) -> String { + self.metadata[&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID)].clone() + } + + pub fn set_tier_free_version(&mut self) { + self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); + } + + pub fn set_skip_tier_free_version(&mut self) { + self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); + } + + pub fn skip_tier_free_version(&self) -> bool { + self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) + } + + pub fn tier_free_version(&self) -> bool { + self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) + } + pub fn set_inline_data(&mut self) { self.metadata .insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned()); @@ -319,7 +350,7 @@ impl FileInfo { /// Check if the object is remote (transitioned to another tier) pub fn is_remote(&self) -> bool { - !self.transition_tier.as_ref().is_none_or(|s| s.is_empty()) + !self.transition_tier.is_empty() } /// Get the data directory for this object @@ -375,7 +406,7 @@ impl FileInfo { pub fn transition_info_equals(&self, other: &FileInfo) -> bool { self.transition_status == other.transition_status && self.transition_tier == other.transition_tier - && self.transitioned_obj_name == other.transitioned_obj_name + && self.transitioned_objname == other.transitioned_objname && self.transition_version_id == other.transition_version_id } diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 5ef8d3e2..e65b0758 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -7,6 +7,7 @@ use crate::headers::{ }; use byteorder::ByteOrder; use rmp::Marker; +use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::hash::Hasher; @@ -36,6 +37,19 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; const META_DATA_READ_DEFAULT: usize = 4 << 10; const MSGP_UINT32_SIZE: usize = 5; +pub const TRANSITION_COMPLETE: &str = "complete"; +pub const TRANSITION_PENDING: &str = "pending"; + +pub const FREE_VERSION: &str = "free-version"; + +pub const TRANSITION_STATUS: &str = "transition-status"; +pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; +pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; +pub const TRANSITION_TIER: &str = "transition-tier"; + +const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; +const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; + // type ScanHeaderVersionFn = Box Result<()>>; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -466,6 +480,40 @@ impl FileMeta { Err(Error::other("add_version failed")) } + pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> { + let mod_time = ver.get_mod_time().unwrap().nanosecond(); + if !ver.valid() { + return Err(Error::other("attempted to add invalid version")); + } + let encoded = ver.marshal_msg()?; + + if self.versions.len()+1 > 100 { + return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object")); + } + + self.versions.push(FileMetaShallowVersion { + header: FileMetaVersionHeader { + mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), + ..Default::default() + }, + ..Default::default() + }); + + let len = self.versions.len(); + for (i, existing) in self.versions.iter().enumerate() { + if existing.header.mod_time.unwrap().nanosecond() <= mod_time { + let vers = self.versions[i..len-1].to_vec(); + self.versions[i+1..].clone_from_slice(vers.as_slice()); + self.versions[i] = FileMetaShallowVersion { + header: ver.header(), + meta: encoded, + }; + return Ok(()); + } + } + Err(Error::other("addVersion: Internal error, unable to add version")) + } + // delete_version deletes version, returns data_dir pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { let mut ventry = FileMetaVersion::default(); @@ -501,6 +549,42 @@ impl FileMeta { } } + for (i, version) in self.versions.iter().enumerate() { + if version.header.version_type != VersionType::Object || version.header.version_id != fi.version_id { + continue; + } + + let mut ver = self.get_idx(i)?; + + if fi.expire_restored { + ver.object.as_mut().unwrap().remove_restore_hdrs(); + let _ = self.set_idx(i, ver.clone()); + } else if fi.transition_status == TRANSITION_COMPLETE { + ver.object.as_mut().unwrap().set_transition(fi); + ver.object.as_mut().unwrap().reset_inline_data(); + self.set_idx(i, ver.clone())?; + } else { + let vers = self.versions[i+1..].to_vec(); + self.versions.extend(vers.iter().cloned()); + let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi); + if to_free { + self.add_version_filemata(free_version)?; + } + } + + if fi.deleted { + self.add_version_filemata(ventry)?; + } + if self.shared_data_dir_count(ver.object.as_ref().unwrap().version_id, ver.object.as_ref().unwrap().data_dir) > 0 { + return Ok(None); + } + return Ok(ver.object.as_ref().unwrap().data_dir); + } + + if fi.deleted { + self.add_version_filemata(ventry)?; + } + Err(Error::FileVersionNotFound) } @@ -1842,10 +1926,17 @@ impl MetaObject { } } - /// Set transition metadata - pub fn set_transition(&mut self, _fi: &FileInfo) { - // Implementation for object lifecycle transitions - // This would handle storage class transitions + pub fn set_transition(&mut self, fi: &FileInfo) { + self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec()); + self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec()); + self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec()); + self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec()); + } + + pub fn remove_restore_hdrs(&mut self) { + self.meta_user.remove(X_AMZ_RESTORE.as_str()); + self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS); + self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE); } pub fn uses_data_dir(&self) -> bool { @@ -1881,6 +1972,45 @@ impl MetaObject { let bytes = hash.to_le_bytes(); [bytes[0], bytes[1], bytes[2], bytes[3]] } + + pub fn init_free_version(&self, fi: &FileInfo) -> (FileMetaVersion, bool) { + if fi.skip_tier_free_version() { + return (FileMetaVersion::default(), false); + } + if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) { + if *status == TRANSITION_COMPLETE.as_bytes().to_vec() { + let vid = Uuid::parse_str(&fi.tier_free_version_id()); + if let Err(err) = vid { + panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string()); + } + let vid = vid.unwrap(); + let mut free_entry = FileMetaVersion { + version_type: VersionType::Delete, + write_version: 0, + ..Default::default() + }; + free_entry.delete_marker = Some(MetaDeleteMarker { + version_id: Some(vid), + mod_time: self.mod_time, + meta_sys: Some(HashMap::>::new()), + }); + + free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); + let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER); + let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME); + let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID); + + let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; + for (k, v) in &self.meta_sys { + if aa.contains(&k) { + free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone()); + } + } + return (free_entry, true); + } + } + (FileMetaVersion::default(), false) + } } impl From for MetaObject { diff --git a/crates/filemeta/src/lib.rs b/crates/filemeta/src/lib.rs index 7f003680..55237001 100644 --- a/crates/filemeta/src/lib.rs +++ b/crates/filemeta/src/lib.rs @@ -1,5 +1,5 @@ mod error; -mod fileinfo; +pub mod fileinfo; mod filemeta; mod filemeta_inline; pub mod headers; diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index d2cd4393..c208772a 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -23,12 +23,18 @@ rustls = { workspace = true, optional = true } rustls-pemfile = { workspace = true, optional = true } rustls-pki-types = { workspace = true, optional = true } serde = { workspace = true, optional = true } -sha2 = { workspace = true, optional = true } siphasher = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } tokio = { workspace = true, optional = true, features = ["io-util", "macros"] } tracing = { workspace = true } url = { workspace = true , optional = true} +hyper.workspace = true +hyper-util.workspace = true +common.workspace = true +sha1 = { workspace = true } +sha2 = { workspace = true, optional = true } +hmac.workspace = true +s3s.workspace = true [dev-dependencies] diff --git a/crates/utils/src/crypto.rs b/crates/utils/src/crypto.rs index a075cbe3..0c271adf 100644 --- a/crates/utils/src/crypto.rs +++ b/crates/utils/src/crypto.rs @@ -1,3 +1,8 @@ +use std::mem::MaybeUninit; + +use hex_simd::{AsOut, AsciiCase}; +use hyper::body::Bytes; + pub fn base64_encode(input: &[u8]) -> String { base64_simd::URL_SAFE_NO_PAD.encode_to_string(input) } @@ -24,6 +29,80 @@ pub fn hex(data: impl AsRef<[u8]>) -> String { // h.finish().unwrap() // } +/// verify sha256 checksum string +pub fn is_sha256_checksum(s: &str) -> bool { + // TODO: optimize + let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f'); + s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex) +} + +/// `hmac_sha1(key, data)` +pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] { + use hmac::{Hmac, Mac}; + use sha1::Sha1; + + let mut m = >::new_from_slice(key.as_ref()).unwrap(); + m.update(data.as_ref()); + m.finalize().into_bytes().into() +} + +/// `hmac_sha256(key, data)` +pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let mut m = Hmac::::new_from_slice(key.as_ref()).unwrap(); + m.update(data.as_ref()); + m.finalize().into_bytes().into() +} + +/// `f(hex(src))` +fn hex_bytes32(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64]; + let ans = hex_simd::encode_as_str(src.as_ref(), buf.as_out(), AsciiCase::Lower); + f(ans) +} + +#[cfg(not(all(feature = "openssl", not(windows))))] +fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> { + use sha2::{Digest, Sha256}; + ::digest(data) +} + +#[cfg(all(feature = "openssl", not(windows)))] +fn sha256(data: &[u8]) -> impl AsRef<[u8]> { + use openssl::hash::{Hasher, MessageDigest}; + let mut h = Hasher::new(MessageDigest::sha256()).unwrap(); + h.update(data).unwrap(); + h.finish().unwrap() +} + +#[cfg(not(all(feature = "openssl", not(windows))))] +fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> { + use sha2::{Digest, Sha256}; + let mut h = ::new(); + chunk.iter().for_each(|data| h.update(data)); + h.finalize() +} + +#[cfg(all(feature = "openssl", not(windows)))] +fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8]> { + use openssl::hash::{Hasher, MessageDigest}; + let mut h = Hasher::new(MessageDigest::sha256()).unwrap(); + chunk.iter().for_each(|data| h.update(data).unwrap()); + h.finish().unwrap() +} + +/// `f(hex(sha256(data)))` +pub fn hex_sha256(data: &[u8], f: impl FnOnce(&str) -> R) -> R { + hex_bytes32(sha256(data).as_ref(), f) +} + +/// `f(hex(sha256(chunk)))` +pub fn hex_sha256_chunk(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R { + hex_bytes32(sha256_chunk(chunk).as_ref(), f) +} + #[test] fn test_base64_encoding_decoding() { let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000"; diff --git a/crates/utils/src/hash.rs b/crates/utils/src/hash.rs index 796e7a90..2f526f1b 100644 --- a/crates/utils/src/hash.rs +++ b/crates/utils/src/hash.rs @@ -61,6 +61,8 @@ impl HashAlgorithm { use crc32fast::Hasher; use siphasher::sip::SipHasher; +pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize { // 你的密钥，必须是 16 字节 diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index bafc06b0..2f2aff60 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -7,6 +7,8 @@ pub mod net; #[cfg(feature = "net")] pub use net::*; +pub mod retry; + #[cfg(feature = "io")] pub mod io; diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index 79944ff2..a07dc797 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -1,11 +1,17 @@ +use hyper::client::conn::http2::Builder; +use hyper_util::rt::TokioExecutor; use lazy_static::lazy_static; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fmt::Display, net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs}, }; +use url::{Host, Url}; +//use hyper::{client::conn::http2::Builder, rt::Executor}; +//use tonic::{SharedExec, UserAgent}; +//use hyper_util::rt::TokioTimer; -use url::Host; +use s3s::header::X_AMZ_STORAGE_CLASS; lazy_static! { static ref LOCAL_IPS: Vec = must_get_local_ips().unwrap(); @@ -105,6 +111,107 @@ pub fn must_get_local_ips() -> std::io::Result> { } } +pub fn get_default_location(u: Url, region_override: &str) -> String { + todo!(); +} + +pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result { + let mut scheme = "https"; + if !secure { + scheme = "http"; + } + + let endpoint_url_str = format!("{scheme}://{endpoint}"); + let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { return Err(std::io::Error::other("url parse error.")); }; + + //is_valid_endpoint_url(endpoint_url)?; + Ok(endpoint_url) +} + +pub const DEFAULT_DIAL_TIMEOUT: i64 = 5; + +pub fn new_remotetarget_http_transport(insecure: bool) -> Builder { + todo!(); +} + +lazy_static! { + static ref SUPPORTED_QUERY_VALUES: HashMap = { + let mut m = HashMap::new(); + m.insert("attributes".to_string(), true); + m.insert("partNumber".to_string(), true); + m.insert("versionId".to_string(), true); + m.insert("response-cache-control".to_string(), true); + m.insert("response-content-disposition".to_string(), true); + m.insert("response-content-encoding".to_string(), true); + m.insert("response-content-language".to_string(), true); + m.insert("response-content-type".to_string(), true); + m.insert("response-expires".to_string(), true); + m + }; + + static ref SUPPORTED_HEADERS: HashMap = { + let mut m = HashMap::new(); + m.insert("content-type".to_string(), true); + m.insert("cache-control".to_string(), true); + m.insert("content-encoding".to_string(), true); + m.insert("content-disposition".to_string(), true); + m.insert("content-language".to_string(), true); + m.insert("x-amz-website-redirect-location".to_string(), true); + m.insert("x-amz-object-lock-mode".to_string(), true); + m.insert("x-amz-metadata-directive".to_string(), true); + m.insert("x-amz-object-lock-retain-until-date".to_string(), true); + m.insert("expires".to_string(), true); + m.insert("x-amz-replication-status".to_string(), true); + m + }; + + static ref SSE_HEADERS: HashMap = { + let mut m = HashMap::new(); + m.insert("x-amz-server-side-encryption".to_string(), true); + m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); + m.insert("x-amz-server-side-encryption-context".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); + m + }; +} + +pub fn is_standard_query_value(qs_key: &str) -> bool { + SUPPORTED_QUERY_VALUES[qs_key] +} + +const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-"; + +pub fn is_custom_query_value(qs_key: &str) -> bool { + qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX) +} + +pub fn is_storageclass_header(header_key: &str) -> bool { + header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase() +} + +pub fn is_standard_header(header_key: &str) -> bool { + *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) +} + +pub fn is_sse_header(header_key: &str) -> bool { + *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) +} + +pub fn is_amz_header(header_key: &str) -> bool { + let key = header_key.to_lowercase(); + key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-") +} + +pub fn is_rustfs_header(header_key: &str) -> bool { + header_key.to_lowercase().starts_with("x-rustfs-") +} + +pub fn is_rustfs_header(header_key: &str) -> bool { + header_key.to_lowercase().starts_with("x-rustfs-") +} + #[derive(Debug, Clone)] pub struct XHost { pub name: String, diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index 0c63b960..f706d8cb 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -253,6 +253,11 @@ pub fn dir(path: &str) -> String { let (a, _) = split(path); clean(a) } + +pub fn trim_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs new file mode 100644 index 00000000..06e5000e --- /dev/null +++ b/crates/utils/src/retry.rs @@ -0,0 +1,179 @@ +//use tokio_stream::Stream; +use std::future::Future; +use std::time::{Duration, Instant}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +// MaxRetry is the maximum number of retries before stopping. +pub const MAX_RETRY: i64 = 10; + +// MaxJitter will randomize over the full exponential backoff time +pub const MAX_JITTER: f64 = 1.0; + +// NoJitter disables the use of jitter for randomizing the exponential backoff time +pub const NO_JITTER: f64 = 0.0; + +// DefaultRetryUnit - default unit multiplicative per retry. +// defaults to 200 * time.Millisecond +//const DefaultRetryUnit = 200 * time.Millisecond; + +// DefaultRetryCap - Each retry attempt never waits no longer than +// this maximum time duration. +//const DefaultRetryCap = time.Second; + +/* +struct Delay { + when: Instant, +} + +impl Future for Delay { + type Output = &'static str; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) + -> Poll<&'static str> + { + if Instant::now() >= self.when { + println!("Hello world"); + Poll::Ready("done") + } else { + // Ignore this line for now. + cx.waker().wake_by_ref(); + Poll::Pending + } + } +} + +struct RetryTimer { + rem: usize, + delay: Delay, +} + +impl RetryTimer { + fn new() -> Self { + Self { + rem: 3, + delay: Delay { when: Instant::now() } + } + } +} + +impl Stream for RetryTimer { + type Item = (); + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) + -> Poll> + { + if self.rem == 0 { + // No more delays + return Poll::Ready(None); + } + + match Pin::new(&mut self.delay).poll(cx) { + Poll::Ready(_) => { + let when = self.delay.when + Duration::from_millis(10); + self.delay = Delay { when }; + self.rem -= 1; + Poll::Ready(Some(())) + } + Poll::Pending => Poll::Pending, + } + } +}*/ + +pub fn new_retry_timer(max_retry: i32, base_sleep: Duration, max_sleep: Duration, jitter: f64) -> Vec { + /*attemptCh := make(chan int) + + exponentialBackoffWait := func(attempt int) time.Duration { + // normalize jitter to the range [0, 1.0] + if jitter < NoJitter { + jitter = NoJitter + } + if jitter > MaxJitter { + jitter = MaxJitter + } + + // sleep = random_between(0, min(maxSleep, base * 2 ** attempt)) + sleep := baseSleep * time.Duration(1< maxSleep { + sleep = maxSleep + } + if jitter != NoJitter { + sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) + } + return sleep + } + + go func() { + defer close(attemptCh) + for i := 0; i < maxRetry; i++ { + select { + case attemptCh <- i + 1: + case <-ctx.Done(): + return + } + + select { + case <-time.After(exponentialBackoffWait(i)): + case <-ctx.Done(): + return + } + } + }() + return attemptCh*/ + todo!(); +} + +/*var retryableS3Codes = map[string]struct{}{ + "RequestError": {}, + "RequestTimeout": {}, + "Throttling": {}, + "ThrottlingException": {}, + "RequestLimitExceeded": {}, + "RequestThrottled": {}, + "InternalError": {}, + "ExpiredToken": {}, + "ExpiredTokenException": {}, + "SlowDown": {}, +} + +fn isS3CodeRetryable(s3Code string) (ok bool) { + _, ok = retryableS3Codes[s3Code] + return ok +} + +var retryableHTTPStatusCodes = map[int]struct{}{ + http.StatusRequestTimeout: {}, + 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet + 499: {}, // client closed request, retry. A non-standard status code introduced by nginx. + http.StatusInternalServerError: {}, + http.StatusBadGateway: {}, + http.StatusServiceUnavailable: {}, + http.StatusGatewayTimeout: {}, + 520: {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected. + // Add more HTTP status codes here. +} + +fn isHTTPStatusRetryable(httpStatusCode int) (ok bool) { + _, ok = retryableHTTPStatusCodes[httpStatusCode] + return ok +} + +fn isRequestErrorRetryable(ctx context.Context, err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // Retry if internal timeout in the HTTP call. + return ctx.Err() == nil + } + if ue, ok := err.(*url.Error); ok { + e := ue.Unwrap() + switch e.(type) { + // x509: certificate signed by unknown authority + case x509.UnknownAuthorityError: + return false + } + switch e.Error() { + case "http: server gave HTTP response to HTTPS client": + return false + } + } + return true +}*/ diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 019751f7..0d24be11 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -34,9 +34,12 @@ serde.workspace = true time.workspace = true bytesize.workspace = true serde_json.workspace = true +serde-xml-rs.workspace = true +serde_urlencoded.workspace = true tracing-error.workspace = true s3s.workspace = true http.workspace = true +http-body-util = "0.1.1" highway = { workspace = true } url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } @@ -56,14 +59,25 @@ tokio-util = { workspace = true, features = ["io", "compat"] } crc32fast = { workspace = true } siphasher = { workspace = true } base64-simd = { workspace = true } -sha2 = { version = "0.11.0-pre.4" } +sha1 = { workspace = true } +sha2 = { workspace = true } hex-simd = { workspace = true } path-clean = { workspace = true } tempfile.workspace = true +hyper.workspace = true +hyper-util.workspace = true +hyper-rustls.workspace = true +rustls.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal"] } tokio-stream = { workspace = true } tonic.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } +tower.workspace = true +async-channel.workspace = true +num = "0.4.3" +enumset = "1.1.5" +hmac.workspace = true +std-next.workspace = true num_cpus = { workspace = true } rand.workspace = true pin-project-lite.workspace = true @@ -80,6 +94,7 @@ shadow-rs.workspace = true rustfs-filemeta.workspace = true rustfs-utils ={workspace = true, features=["full"]} rustfs-rio.workspace = true +reader = { workspace = true } [target.'cfg(not(windows))'.dependencies] nix = { workspace = true } diff --git a/ecstore/src/admin_server_info.rs b/ecstore/src/admin_server_info.rs index a2af8352..7304ae05 100644 --- a/ecstore/src/admin_server_info.rs +++ b/ecstore/src/admin_server_info.rs @@ -23,7 +23,7 @@ use protos::{ }; use std::{ collections::{HashMap, HashSet}, - time::SystemTime, + time::{SystemTime, UNIX_EPOCH}, }; use time::OffsetDateTime; use tonic::Request; diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs new file mode 100644 index 00000000..cc7ffc1f --- /dev/null +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs @@ -0,0 +1,32 @@ +use super::lifecycle; + +#[derive(Debug, Clone, Default)] +pub enum LcEventSrc { + #[default] + None, + Heal, + Scanner, + Decom, + Rebal, + S3HeadObject, + S3GetObject, + S3ListObjects, + S3PutObject, + S3CopyObject, + S3CompleteMultipartUpload, +} + +#[derive(Clone, Debug, Default)] +pub struct LcAuditEvent { + pub event: lifecycle::Event, + pub source: LcEventSrc, +} + +impl LcAuditEvent { + pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self { + Self { + event, + source, + } + } +} \ No newline at end of file diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs new file mode 100644 index 00000000..1f764023 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -0,0 +1,815 @@ +use std::any::{Any, TypeId}; +use std::env; +use std::io::{Cursor, Write}; +use std::pin::Pin; +use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; +use std::sync::{Arc, Mutex}; +use futures::Future; +use lazy_static::lazy_static; +use s3s::Body; +use std::collections::HashMap; +use tracing::{error, info, warn}; +use sha2::{Digest, Sha256}; +use xxhash_rust::xxh64; +use uuid::Uuid; +use http::HeaderMap; +use tokio::select; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::{mpsc, RwLock}; +use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender}; + +use s3s::dto::BucketLifecycleConfiguration; +use crate::error::Error; +use crate::event::name::EventName; +use crate::store::ECStore; +use crate::store_api::StorageAPI; +use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,}; +use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; +use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; +use crate::client::object_api_utils::{new_getobjectreader,}; +use crate::event_notification::{send_event, EventArgs}; +use crate::heal::{ + data_scanner_metric::ScannerMetrics, + data_scanner::{ + apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects, + }, + data_usage_cache::TierStats, +}; +use crate::global::GLOBAL_LocalNodeName; +use crate::bucket::{ + metadata_sys::get_lifecycle_config, + versioning_sys::BucketVersioningSys, +}; +use crate::tier::warm_backend::WarmBackendGetOpts; +use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; +use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats}; +use super::tier_sweeper::{delete_object_from_remote_tier, Jentry}; +use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent}; + +pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; +pub type TraceFn = Arc) -> Pin + Send>> + Send + Sync + 'static>; +pub type ExpiryOpType = Box; + +static XXHASH_SEED: u64 = 0; + +const DISABLED: &str = "Disabled"; + +//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class."; +pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier."; + +lazy_static! { + pub static ref GLOBAL_ExpiryState: Arc> = ExpiryState::new(); + pub static ref GLOBAL_TransitionState: Arc = TransitionState::new(); +} + +pub struct LifecycleSys; + +impl LifecycleSys { + pub fn new() -> Arc { + Arc::new(Self) + } + + pub async fn get(&self, bucket: &str) -> Option { + let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0; + Some(lc) + } + + pub fn trace(oi: &ObjectInfo) -> TraceFn + { + todo!(); + } +} + +struct ExpiryTask { + obj_info: ObjectInfo, + event: lifecycle::Event, + src: LcEventSrc, +} + +impl ExpiryOp for ExpiryTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes()); + let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +struct ExpiryStats { + missed_expiry_tasks: AtomicI64, + missed_freevers_tasks: AtomicI64, + missed_tier_journal_tasks: AtomicI64, + workers: AtomicI64, +} + +impl ExpiryStats { + pub fn missed_tasks(&self) -> i64 { + self.missed_expiry_tasks.load(Ordering::SeqCst) + } + + fn missed_free_vers_tasks(&self) -> i64 { + self.missed_freevers_tasks.load(Ordering::SeqCst) + } + + fn missed_tier_journal_tasks(&self) -> i64 { + self.missed_tier_journal_tasks.load(Ordering::SeqCst) + } + + fn num_workers(&self) -> i64 { + self.workers.load(Ordering::SeqCst) + } +} + +pub trait ExpiryOp: 'static { + fn op_hash(&self) -> u64; + fn as_any(&self) -> &dyn Any; +} + +#[derive(Debug, Default, Clone)] +pub struct TransitionedObject { + pub name: String, + pub version_id: String, + pub tier: String, + pub free_version: bool, + pub status: String, +} + +struct FreeVersionTask(ObjectInfo); + +impl ExpiryOp for FreeVersionTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes()); + let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +struct NewerNoncurrentTask { + bucket: String, + versions: Vec, + event: lifecycle::Event, +} + +impl ExpiryOp for NewerNoncurrentTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.bucket).as_bytes()); + let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub struct ExpiryState { + tasks_tx: Vec>>, + tasks_rx: Vec>>>>, + stats: Option, +} + + + +impl ExpiryState { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + tasks_tx: vec![], + tasks_rx: vec![], + stats: Some(ExpiryStats { + missed_expiry_tasks: AtomicI64::new(0), + missed_freevers_tasks: AtomicI64::new(0), + missed_tier_journal_tasks: AtomicI64::new(0), + workers: AtomicI64::new(0), + }), + })) + } + + pub async fn pending_tasks(&self) -> usize { + let rxs = &self.tasks_rx; + if rxs.len() == 0 { + return 0; + } + let mut tasks=0; + for rx in rxs.iter() { + tasks += rx.lock().await.len(); + } + tasks + } + + pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> { + let wrkr = self.get_worker_ch(je.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + } + let wrkr = wrkr.expect("err"); + select! { + //_ -> GlobalContext.Done() => () + _ = wrkr.send(Some(Box::new(je.clone()))) => (), + else => { + *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + } + } + return Ok(()); + } + + pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) { + let task = FreeVersionTask(oi); + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + } + } + } + + pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { + let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()}; + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + } + } + } + + pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec, lc_event: lifecycle::Event) { + if versions.len() == 0 { + return; + } + + let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event}; + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + } + } + } + + pub fn get_worker_ch(&self, h: u64) -> Option>> { + if self.tasks_tx.len() == 0 { + return None; + } + Some(self.tasks_tx[h as usize %self.tasks_tx.len()].clone()) + } + + pub async fn resize_workers(n: usize, api: Arc) { + if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 { + return; + } + + let mut state = GLOBAL_ExpiryState.write().await; + + while state.tasks_tx.len() < n { + let (tx, mut rx) = mpsc::channel(10000); + let api = api.clone(); + let rx = Arc::new(tokio::sync::Mutex::new(rx)); + state.tasks_tx.push(tx); + state.tasks_rx.push(rx.clone()); + *state.stats.as_mut().expect("err").workers.get_mut() += 1; + tokio::spawn(async move { + let mut rx = rx.lock().await; + //let mut expiry_state = GLOBAL_ExpiryState.read().await; + ExpiryState::worker(&mut *rx, api).await; + }); + } + + let mut l = state.tasks_tx.len(); + while l > n { + let worker = state.tasks_tx[l-1].clone(); + worker.send(None).await.unwrap_or(()); + state.tasks_tx.remove(l-1); + state.tasks_rx.remove(l-1); + *state.stats.as_mut().expect("err").workers.get_mut() -= 1; + l -= 1; + } + } + + pub async fn worker(rx: &mut Receiver>, api: Arc) { + loop { + select! { + _ = tokio::signal::ctrl_c() => { + info!("got ctrl+c, exits"); + break; + } + v = rx.recv() => { + if v.is_none() { + break; + } + let v = v.expect("err!"); + if v.is_none() { + //rx.close(); + //drop(rx); + let _ = rx; + return; + } + let v = v.expect("err!"); + if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + if v.obj_info.transitioned_object.status != "" { + apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await; + } else { + apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await; + } + } + else if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + //delete_object_versions(api, &v.bucket, &v.versions, v.event).await; + } + else if v.as_any().is::() { + //transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName)) + } + else if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + let oi = v.0.clone(); + + } + else { + //info!("Invalid work type - {:?}", v); + todo!(); + } + } + } + } + } +} + +struct TransitionTask { + obj_info: ObjectInfo, + src: LcEventSrc, + event: lifecycle::Event, +} + +impl ExpiryOp for TransitionTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes()); + //let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub struct TransitionState { + transition_tx: A_Sender>, + transition_rx: A_Receiver>, + pub num_workers: AtomicI64, + kill_tx: A_Sender<()>, + kill_rx: A_Receiver<()>, + active_tasks: AtomicI64, + missed_immediate_tasks: AtomicI64, + last_day_stats: Arc>>, +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; +//type RetKill = impl Future> + Send + 'static; +//type RetTransitionTask = impl Future>> + Send + 'static; + +impl TransitionState { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc { + let (tx1, rx1) = bounded(100000); + let (tx2, rx2) = bounded(1); + Arc::new(Self { + transition_tx: tx1, + transition_rx: rx1, + num_workers: AtomicI64::new(0), + kill_tx: tx2, + kill_rx: rx2, + active_tasks: AtomicI64::new(0), + missed_immediate_tasks: AtomicI64::new(0), + last_day_stats: Arc::new(Mutex::new(HashMap::new())), + }) + } + + pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { + let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()}; + select! { + //_ -> t.ctx.Done() => (), + _ = self.transition_tx.send(Some(task)) => (), + else => { + match src { + LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => { + self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst); + } + _ => () + } + }, + } + } + + pub async fn init(api: Arc) { + let mut n = 10;//globalAPIConfig.getTransitionWorkers(); + let tw = 10;//globalILMConfig.getTransitionWorkers(); + if tw > 0 { + n = tw; + } + + //let mut transition_state = GLOBAL_TransitionState.write().await; + //self.objAPI = objAPI + Self::update_workers(api, n).await; + } + + pub fn pending_tasks(&self) -> usize { + //let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap(); + let transition_rx = &GLOBAL_TransitionState.transition_rx; + transition_rx.len() + } + + pub fn active_tasks(&self) -> i64 { + self.active_tasks.load(Ordering::SeqCst) + } + + pub fn missed_immediate_tasks(&self) -> i64 { + self.missed_immediate_tasks.load(Ordering::SeqCst) + } + + pub async fn worker(api: Arc) { + loop { + select! { + _ = GLOBAL_TransitionState.kill_rx.recv() => { + return; + } + task = GLOBAL_TransitionState.transition_rx.recv() => { + if task.is_err() { + break; + } + let task = task.expect("err!"); + if task.is_none() { + //self.transition_rx.close(); + //drop(self.transition_rx); + return; + } + let task = task.expect("err!"); + if task.as_any().is::() { + let task = task.as_any().downcast_ref::().expect("err!"); + + GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst); + if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await { + if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) { + if !err.to_string().contains("use of closed network connection") { + error!("Transition to {} failed for {}/{} version:{} with {}", + task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string()); + } + } + } else { + let mut ts = TierStats { + total_size: task.obj_info.size as u64, + num_versions: 1, + ..Default::default() + }; + if task.obj_info.is_latest { + ts.num_objects = 1; + } + GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts); + } + GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst); + } + } + else => () + } + } + } + + pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { + let mut tier_stats = self.last_day_stats.lock().unwrap(); + tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts)) + .or_insert(LastDayTierStats::default()); + } + + pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { + let tier_stats = self.last_day_stats.lock().unwrap(); + let mut res = DailyAllTierStats::with_capacity(tier_stats.len()); + for (tier, st) in tier_stats.iter() { + res.insert(tier.clone(), st.clone()); + } + res + } + + pub async fn update_workers(api: Arc, n: i64) { + Self::update_workers_inner(api, n).await; + } + + pub async fn update_workers_inner(api: Arc, n: i64) { + let mut n = n; + if n == 0 { + n = 100; + } + + let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); + while num_workers < n { + let clone_api = api.clone(); + tokio::spawn(async move { + TransitionState::worker(clone_api).await; + }); + num_workers = num_workers + 1; + GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst); + } + + let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); + while num_workers > n { + let worker = GLOBAL_TransitionState.kill_tx.clone(); + worker.send(()).await; + num_workers = num_workers - 1; + GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst); + } + } +} + +struct AuditTierOp { + tier: String, + time_to_responsens: i64, + output_bytes: i64, + error: String, +} + +impl AuditTierOp { + #[allow(clippy::new_ret_no_self)] + pub async fn new() -> Result { + Ok(Self { + tier: String::from("tier"), + time_to_responsens: 0, + output_bytes: 0, + error: String::from(""), + }) + } + + pub fn string(&self) -> String { + format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error) + } +} + +pub async fn init_background_expiry(api: Arc) { + let mut workers = num_cpus::get() / 2; + //globalILMConfig.getExpirationWorkers() + if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") { + if let Ok(num_expirations) = env_expiration_workers.parse::() { + workers = num_expirations; + } + } + + if workers == 0 { + workers = 100; + } + + //let expiry_state = GLOBAL_ExpiryStSate.write().await; + ExpiryState::resize_workers(workers, api).await; +} + +pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> { + for rule in &lc.rules { + if let Some(transitions) = &rule.transitions { + for transition in transitions { + if let Some(storage_class) = &transition.storage_class { + if storage_class.as_str() != "" { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); + } + } + } + } + } + if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions { + for noncurrent_version_transition in noncurrent_version_transitions { + if let Some(storage_class) = &noncurrent_version_transition.storage_class { + if storage_class.as_str() != "" { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); + } + } + } + } + } + } + Ok(()) +} + +pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { + let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await; + if !lc.is_none() { + let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await; + match event.action { + lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + if oi.delete_marker || oi.is_dir { + return; + } + GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await; + } + _ => () + } + } +} + +pub async fn expire_transitioned_object(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result { + //let traceFn = GLOBAL_LifecycleSys.trace(oi); + let mut opts = ObjectOptions { + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + expiration: ExpirationOptions {expire: true}, + ..Default::default() + }; + if lc_event.action == IlmAction::DeleteVersionAction { + opts.version_id = oi.version_id.map(|id| id.to_string()); + } + //let tags = LcAuditEvent::new(src, lcEvent).Tags(); + if lc_event.action == IlmAction::DeleteRestoredAction { + opts.transition.expire_restored = true; + match api.delete_object(&oi.bucket, &oi.name, opts).await { + Ok(dobj) => { + //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn); + return Ok(dobj); + } + Err(err) => return Err(std::io::Error::other(err)), + } + } + + let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await; + if ret.is_ok() { + opts.skip_decommissioned = true; + } else { + //transitionLogIf(ctx, err); + } + + let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?; + + //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) + + let mut event_name = EventName::ObjectRemovedDelete; + if oi.delete_marker { + event_name = EventName::ObjectRemovedDeleteMarkerCreated; + } + let obj_info = ObjectInfo { + name: oi.name.clone(), + version_id: oi.version_id, + delete_marker: oi.delete_marker, + ..Default::default() + }; + send_event(EventArgs { + event_name: event_name.as_ref().to_string(), + bucket_name: obj_info.bucket.clone(), + object: obj_info, + user_agent: "Internal: [ILM-Expiry]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + + Ok(dobj) +} + +pub fn gen_transition_objname(bucket: &str) -> Result { + let us = Uuid::new_v4().to_string(); + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes()); + hasher.flush(); + let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice()); + let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us); + Ok(obj) +} + +pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> { + let time_ilm = ScannerMetrics::time_ilm(lae.event.action); + + let opts = ObjectOptions { + transition: TransitionOptions { + status: lifecycle::TRANSITION_PENDING.to_string(), + tier: lae.event.storage_class, + etag: oi.etag.clone().expect("err").to_string(), + ..Default::default() + }, + //lifecycle_audit_event: lae, + version_id: Some(oi.version_id.expect("err").to_string()), + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, + mod_time: oi.mod_time, + ..Default::default() + }; + time_ilm(1); + api.transition_object(&oi.bucket, &oi.name, &opts).await +} + +pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn { + todo!(); +} + +pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result { + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { + Ok(d) => d, + Err(err) => return Err(std::io::Error::other(err)), + }; + + let ret = new_getobjectreader(rs, &oi, opts, &h); + if let Err(err) = ret { + return Err(error_resp_to_object_err(err, vec![bucket, object])); + } + let (get_fn, off, length) = ret.expect("err"); + let mut gopts = WarmBackendGetOpts::default(); + + if off >= 0 && length >= 0 { + gopts.start_offset = off; + gopts.length = length; + } + + //return Ok(HttpFileReader::new(rs, &oi, opts, &h)); + //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length) + let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?; + Ok(get_fn(reader, h)) +} + +pub fn post_restore_opts(r: http::Request, bucket: &str, object: &str) -> Result { + todo!(); +} + +pub fn put_restore_opts(bucket: &str, object: &str, rreq: &RestoreObjectRequest, oi: &ObjectInfo) -> ObjectOptions { + todo!(); +} + +pub trait LifecycleOps { + fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts; +} + +impl LifecycleOps for ObjectInfo { + fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts { + lifecycle::ObjectOpts { + name: self.name.clone(), + user_tags: self.user_tags.clone(), + version_id: self.version_id.expect("err").to_string(), + mod_time: self.mod_time, + size: self.size, + is_latest: self.is_latest, + num_versions: self.num_versions, + delete_marker: self.delete_marker, + successor_mod_time: self.successor_mod_time, + //restore_ongoing: self.restore_ongoing, + //restore_expires: self.restore_expires, + transition_status: self.transitioned_object.status.clone(), + ..Default::default() + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct S3Location { + pub bucketname: String, + //pub encryption: Encryption, + pub prefix: String, + pub storage_class: String, + //pub tagging: Tags, + pub user_metadata: HashMap, +} + +#[derive(Debug, Default, Clone)] +pub struct OutputLocation(pub S3Location); + +#[derive(Debug, Default, Clone)] +pub struct RestoreObjectRequest { + pub days: i64, + pub ror_type: String, + pub tier: String, + pub description: String, + //pub select_parameters: SelectParameters, + pub output_location: OutputLocation, +} + +const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; + diff --git a/ecstore/src/bucket/lifecycle/lifecycle.rs b/ecstore/src/bucket/lifecycle/lifecycle.rs new file mode 100644 index 00000000..759fde7a --- /dev/null +++ b/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -0,0 +1,702 @@ +use std::cmp::Ordering; +use std::env; +use std::fmt::Display; +use s3s::dto::{ + BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration, + ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition, +}; +use time::macros::{datetime, offset}; +use time::{self, OffsetDateTime, Duration}; + +use crate::bucket::lifecycle::rule::TransitionOps; + +use super::bucket_lifecycle_ops::RestoreObjectRequest; + +pub const TRANSITION_COMPLETE: &str = "complete"; +pub const TRANSITION_PENDING: &str = "pending"; + +const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules"; +const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; +const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; +const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; +const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IlmAction { + NoneAction = 0, + DeleteAction, + DeleteVersionAction, + TransitionAction, + TransitionVersionAction, + DeleteRestoredAction, + DeleteRestoredVersionAction, + DeleteAllVersionsAction, + DelMarkerDeleteAllVersionsAction, + ActionCount, +} + +impl IlmAction { + pub fn delete_restored(&self) -> bool { + *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_versioned(&self) -> bool { + *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_all(&self) -> bool { + *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction + } + + pub fn delete(&self) -> bool { + if self.delete_restored() { + return true; + } + *self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction + } +} + +impl Display for IlmAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +#[async_trait::async_trait] +pub trait RuleValidate { + fn validate(&self) -> Result<(), std::io::Error>; +} + +#[async_trait::async_trait] +impl RuleValidate for LifecycleRule { + /*fn validate_id(&self) -> Result<()> { + if self.id.len() > 255 { + return errInvalidRuleID; + } + Ok(()) + } + + fn validate_status(&self) -> Result<()> { + if self.Status.len() == 0 { + return errEmptyRuleStatus; + } + + if self.Status != Enabled && self.Status != Disabled { + return errInvalidRuleStatus; + } + Ok(()) + } + + fn validate_expiration(&self) -> Result<()> { + self.Expiration.Validate(); + } + + fn validate_noncurrent_expiration(&self) -> Result<()> { + self.NoncurrentVersionExpiration.Validate() + } + + fn validate_prefix_and_filter(&self) -> Result<()> { + if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() { + return errXMLNotWellFormed; + } + if !self.Prefix.set { + return self.Filter.Validate(); + } + Ok(()) + } + + fn validate_transition(&self) -> Result<()> { + self.Transition.Validate() + } + + fn validate_noncurrent_transition(&self) -> Result<()> { + self.NoncurrentVersionTransition.Validate() + } + + fn get_prefix(&self) -> String { + if p := self.Prefix.String(); p != "" { + return p + } + if p := self.Filter.Prefix.String(); p != "" { + return p + } + if p := self.Filter.And.Prefix.String(); p != "" { + return p + } + "".to_string() + }*/ + + fn validate(&self) -> Result<(), std::io::Error> { + /*self.validate_id()?; + self.validate_status()?; + self.validate_expiration()?; + self.validate_noncurrent_expiration()?; + self.validate_prefix_and_filter()?; + self.validate_transition()?; + self.validate_noncurrent_transition()?; + if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() { + return errInvalidRuleDelMarkerExpiration + } + if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() { + return errXMLNotWellFormed + }*/ + Ok(()) + } +} + +#[async_trait::async_trait] +pub trait Lifecycle { + async fn has_transition(&self) -> bool; + fn has_expiry(&self) -> bool; + async fn has_active_rules(&self, prefix: &str) -> bool; + async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>; + async fn filter_rules(&self, obj: &ObjectOpts) -> Option>; + async fn eval(&self, obj: &ObjectOpts) -> Event; + async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event; + //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts); + async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event; +} + +#[async_trait::async_trait] +impl Lifecycle for BucketLifecycleConfiguration { + async fn has_transition(&self) -> bool { + for rule in self.rules.iter() { + if !rule.transitions.is_none() { + return true; + } + } + false + } + + fn has_expiry(&self) -> bool { + for rule in self.rules.iter() { + if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() { + return true; + } + } + false + } + + async fn has_active_rules(&self, prefix: &str) -> bool { + if self.rules.len() == 0 { + return false; + } + for rule in self.rules.iter() { + if rule.status.as_str() == ExpirationStatus::DISABLED { + continue; + } + + let rule_prefix = rule.prefix.as_ref().expect("err!"); + if prefix.len() > 0 && rule_prefix.len() > 0 { + if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix) { + continue; + } + } + + let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!"); + if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 { + return true; + } + if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 { + return true; + } + if !rule.noncurrent_version_transitions.is_none() { + return true; + } + let rule_expiration = rule.expiration.as_ref().expect("err!"); + if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { + return true; + } + if !rule_expiration.date.is_none() { + return true; + } + if rule_expiration.expired_object_delete_marker.expect("err!") { + return true; + } + let rule_transitions: &[Transition]= &rule.transitions.as_ref().expect("err!"); + let rule_transitions_0 = rule_transitions[0].clone(); + if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { + return true; + } + if !rule.transitions.is_none() { + return true; + } + } + false + } + + async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error> { + if self.rules.len() > 1000 { + return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES)); + } + if self.rules.len() == 0 { + return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE)); + } + + for r in &self.rules { + r.validate()?; + if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() { + if let Some(expiration) = r.expiration.as_ref() { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (!expired_object_delete_marker) { + return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); + } + } /*else { + if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED { + return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED)); + } + }*/ + } + } + } + for (i,_) in self.rules.iter().enumerate() { + if i == self.rules.len()-1 { + break; + } + let other_rules = &self.rules[i+1..]; + for other_rule in other_rules { + if self.rules[i].id == other_rule.id { + return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); + } + } + } + Ok(()) + } + + async fn filter_rules(&self, obj: &ObjectOpts) -> Option> { + if obj.name == "" { + return None; + } + let mut rules = Vec::::new(); + for rule in self.rules.iter() { + if rule.status.as_str() == ExpirationStatus::DISABLED { + continue; + } + if let Some(prefix) = rule.prefix.clone() { + if !obj.name.starts_with(prefix.as_str()) { + continue; + } + } + /*if !rule.filter.test_tags(obj.user_tags) { + continue; + }*/ + //if !obj.delete_marker && !rule.filter.BySize(obj.size) { + if !obj.delete_marker && false{ + continue; + } + rules.push(rule.clone()); + } + Some(rules) + } + + async fn eval(&self, obj: &ObjectOpts) -> Event { + self.eval_inner(obj, OffsetDateTime::now_utc()).await + } + + async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event { + let mut events = Vec::::new(); + if obj.mod_time.expect("err").unix_timestamp() == 0 { + return Event::default(); + } + + if let Some(restore_expires) = obj.restore_expires { + if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() { + let mut action = IlmAction::DeleteRestoredAction; + if !obj.is_latest { + action = IlmAction::DeleteRestoredVersionAction; + } + + events.push(Event{ + action: action, + due: Some(now), + rule_id: "".into(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + + if let Some(ref lc_rules) = self.filter_rules(obj).await { + for rule in lc_rules.iter() { + if obj.expired_object_deletemarker() { + if let Some(expiration) = rule.expiration.as_ref() { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + events.push(Event{ + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(now), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + break; + } + } + + if let Some(expiration) = rule.expiration.as_ref() { + if let Some(days) = expiration.days { + let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days/*, date*/); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + events.push(Event{ + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + break; + } + } + } + } + + if obj.is_latest { + if let Some(ref expiration) = rule.expiration { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + if obj.delete_marker && expired_object_delete_marker { + let due = expiration.next_due(obj); + if let Some(due) = due { + if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() { + events.push(Event{ + action: IlmAction::DelMarkerDeleteAllVersionsAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(due), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + continue; + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { + if newer_noncurrent_versions > 0 { + continue; + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days { + if noncurrent_days != 0 { + if let Some(successor_mod_time) = obj.successor_mod_time { + let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + events.push(Event{ + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions { + if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class { + if storage_class.as_str() != "" { + if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE { + let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); + if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) { + events.push(Event { + action: IlmAction::TransitionVersionAction, + rule_id: rule.id.clone().expect("err!"), + due, + storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(), + ..Default::default() + }); + } + } + } + } + } + } + + if obj.is_latest && !obj.delete_marker { + if let Some(ref expiration) = rule.expiration { + if let Some(ref date) = expiration.date { + let date0 = OffsetDateTime::from(date.clone()); + if date0.unix_timestamp() != 0 { + if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() { + events.push(Event{ + action: IlmAction::DeleteAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(date0), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + } else if let Some(days) = expiration.days { + if days != 0 { + let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + let mut event = Event{ + action: IlmAction::DeleteAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }; + /*if rule.expiration.expect("err!").delete_all.val { + event.action = IlmAction::DeleteAllVersionsAction + }*/ + events.push(event); + } + } + } + } + + if obj.transition_status != TRANSITION_COMPLETE { + if let Some(ref transitions) = rule.transitions { + let due = transitions[0].next_due(obj); + if let Some(due) = due { + if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) { + events.push(Event{ + action: IlmAction::TransitionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(due), + storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + }); + } + } + } + } + } + } + } + + if events.len() > 0 { + events.sort_by(|a, b| { + if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() { + match a.action { + IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { + return Ordering::Less; + } + _ => () + } + match b.action { + IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { + return Ordering::Greater; + } + _ => () + } + return Ordering::Less; + } + + if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() { + return Ordering::Less; + } + return Ordering::Greater; + }); + return events[0].clone(); + } + + Event::default() + } + + async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event { + if let Some(filter_rules) = self.filter_rules(obj).await { + for rule in filter_rules.iter() { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { + if newer_noncurrent_versions == 0 { + continue; + } + return Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + newer_noncurrent_versions: newer_noncurrent_versions as usize, + due: Some(OffsetDateTime::UNIX_EPOCH), + storage_class: "".into(), + }; + } else { + return Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + newer_noncurrent_versions: 0, + due: Some(OffsetDateTime::UNIX_EPOCH), + storage_class: "".into(), + }; + } + } + } + } + Event::default() + } +} + +#[async_trait::async_trait] +pub trait LifecycleCalculate { + fn next_due(&self, obj: &ObjectOpts) -> Option; +} + +#[async_trait::async_trait] +impl LifecycleCalculate for LifecycleExpiration { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if !obj.is_latest || !obj.delete_marker { + return None; + } + + Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap())) + } +} + +#[async_trait::async_trait] +impl LifecycleCalculate for NoncurrentVersionTransition { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if obj.is_latest || self.storage_class.is_none() { + return None; + } + if self.noncurrent_days.is_none() { + return obj.successor_mod_time; + } + Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap())) + } +} + +#[async_trait::async_trait] +impl LifecycleCalculate for Transition { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if !obj.is_latest || self.days.is_none() { + return None; + } + + if let Some(date) = self.date.clone() { + return Some(date.into()); + } + + if self.days.is_none() { + return obj.mod_time; + } + Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap())) + } +} + +pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime { + if days == 0 { + return mod_time; + } + let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug + let mut hour = 3600; + if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") { + if let Ok(num_hour) = env_ilm_hour.parse::() { + hour = num_hour; + } + } + //t.Truncate(24 * hour) + t +} + +#[derive(Default)] +pub struct ObjectOpts { + pub name: String, + pub user_tags: String, + pub mod_time: Option, + pub size: usize, + pub version_id: String, + pub is_latest: bool, + pub delete_marker: bool, + pub num_versions: usize, + pub successor_mod_time: Option, + pub transition_status: String, + pub restore_ongoing: bool, + pub restore_expires: Option, + pub versioned: bool, + pub version_suspended: bool, +} + +impl ObjectOpts { + pub fn expired_object_deletemarker(&self) -> bool { + self.delete_marker && self.num_versions == 1 + } +} + +#[derive(Debug, Clone)] +pub struct Event { + pub action: IlmAction, + pub rule_id: String, + pub due: Option, + pub noncurrent_days: u32, + pub newer_noncurrent_versions: usize, + pub storage_class: String, +} + +impl Default for Event { + fn default() -> Self { + Self { + action: IlmAction::NoneAction, + rule_id: "".into(), + due: Some(OffsetDateTime::UNIX_EPOCH), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct ExpirationOptions { + pub expire: bool +} + +impl ExpirationOptions { + fn marshal_msg(&self, b: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + fn unmarshal_msg(&self, bts: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + fn msg_size(&self) -> i64 { + 1 + 7 + 10 + } +} + +#[derive(Debug, Clone)] +pub struct TransitionOptions { + pub status: String, + pub tier: String, + pub etag: String, + pub restore_request: RestoreObjectRequest, + pub restore_expiry: OffsetDateTime, + pub expire_restored: bool, +} + +impl Default for TransitionOptions { + fn default() -> Self { + Self { + status: Default::default(), + tier: Default::default(), + etag: Default::default(), + restore_request: Default::default(), + restore_expiry: OffsetDateTime::now_utc(), + expire_restored: Default::default(), + } + } +} diff --git a/ecstore/src/bucket/lifecycle/mod.rs b/ecstore/src/bucket/lifecycle/mod.rs new file mode 100644 index 00000000..85715016 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/mod.rs @@ -0,0 +1,6 @@ +pub mod rule; +pub mod lifecycle; +pub mod tier_sweeper; +pub mod tier_last_day_stats; +pub mod bucket_lifecycle_ops; +pub mod bucket_lifecycle_audit; \ No newline at end of file diff --git a/ecstore/src/bucket/lifecycle/rule.rs b/ecstore/src/bucket/lifecycle/rule.rs new file mode 100644 index 00000000..d31b35cf --- /dev/null +++ b/ecstore/src/bucket/lifecycle/rule.rs @@ -0,0 +1,51 @@ +use s3s::dto::{ + LifecycleRuleFilter, Transition, +}; + +const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; +const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; +const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; +const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT"; + +pub trait Filter { + fn test_tags(&self, user_tags: &str) -> bool; + fn by_size(&self, sz: i64) -> bool; +} + +impl Filter for LifecycleRuleFilter { + fn test_tags(&self, user_tags: &str) -> bool { + true + } + + fn by_size(&self, sz: i64) -> bool { + true + } +} + +pub trait TransitionOps { + fn validate(&self) -> Result<(), std::io::Error>; +} + +impl TransitionOps for Transition { + fn validate(&self) -> Result<(), std::io::Error> { + if !self.date.is_none() && self.days.expect("err!") > 0 { + return Err(std::io::Error::other(ERR_TRANSITION_INVALID)); + } + + if self.storage_class.is_none() { + return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED")); + } + Ok(()) + } +} + + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn test_rule() { + //assert!(skip_access_checks(p.to_str().unwrap())); + } +} diff --git a/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs new file mode 100644 index 00000000..5582db69 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -0,0 +1,86 @@ +use sha2::Sha256; + +use std::collections::HashMap; +use time::OffsetDateTime; +use tracing::{error, warn}; +use std::ops::Sub; + +use crate::heal::data_usage_cache::TierStats; + +pub type DailyAllTierStats = HashMap; + +#[derive(Clone)] +pub struct LastDayTierStats { + bins: [TierStats; 24], + updated_at: OffsetDateTime, +} + +impl Default for LastDayTierStats { + fn default() -> Self { + Self { + bins: Default::default(), + updated_at: OffsetDateTime::now_utc(), + } + } +} + +impl LastDayTierStats { + pub fn add_stats(&mut self, ts: TierStats) { + let mut now = OffsetDateTime::now_utc(); + self.forward_to(&mut now); + + let now_idx = now.hour() as usize; + self.bins[now_idx] = self.bins[now_idx].add(&ts); + } + + fn forward_to(&mut self, t: &mut OffsetDateTime) { + if t.unix_timestamp() == 0 { + *t = OffsetDateTime::now_utc(); + } + + let since = t.sub(self.updated_at).whole_hours(); + if since < 1 { + return; + } + + let (idx, mut last_idx) = (t.hour(), self.updated_at.hour()); + + self.updated_at = *t; + + if since >= 24 { + self.bins = [TierStats::default(); 24]; + return; + } + + while last_idx != idx { + last_idx = (last_idx + 1) % 24; + self.bins[last_idx as usize] = TierStats::default(); + } + } + + fn merge(&self, m: LastDayTierStats) -> LastDayTierStats { + let mut cl = self.clone(); + let mut cm = m.clone(); + let mut merged = LastDayTierStats::default(); + + if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() { + cm.forward_to(&mut cl.updated_at); + merged.updated_at = cl.updated_at; + } else { + cl.forward_to(&mut cm.updated_at); + merged.updated_at = cm.updated_at; + } + + for (i, _) in cl.bins.iter().enumerate() { + merged.bins[i] = cl.bins[i].add(&cm.bins[i]); + } + + merged + } +} + + +#[cfg(test)] +mod test { + +} diff --git a/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/ecstore/src/bucket/lifecycle/tier_sweeper.rs new file mode 100644 index 00000000..17237bb1 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -0,0 +1,130 @@ +use sha2::{Digest, Sha256}; +use xxhash_rust::xxh64; +use std::any::Any; +use std::io::{Cursor, Write}; + +use crate::global::GLOBAL_TierConfigMgr; +use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject}; +use super::lifecycle::{self, ObjectOpts}; + +static XXHASH_SEED: u64 = 0; + +#[derive(Default)] +struct ObjSweeper { + object: String, + bucket: String, + version_id: String, + versioned: bool, + suspended: bool, + transition_status: String, + transition_tier: String, + transition_version_id: String, + remote_object: String, +} + +impl ObjSweeper { + #[allow(clippy::new_ret_no_self)] + pub async fn new(bucket: &str, object: &str) -> Result { + Ok(Self { + object: object.into(), + bucket: bucket.into(), + ..Default::default() + }) + } + + pub fn with_version(&mut self, vid: String) -> &Self { + self.version_id = vid; + self + } + + pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self { + self.versioned = versioned; + self.suspended = suspended; + self + } + + pub fn get_opts(&self) -> lifecycle::ObjectOpts { + let mut opts = ObjectOpts{ + version_id: self.version_id.clone(), + versioned: self.versioned, + version_suspended: self.suspended, + ..Default::default() + }; + if self.suspended && self.version_id == "" { + opts.version_id = String::from(""); + } + opts + } + + pub fn set_transition_state(&mut self, info: TransitionedObject) { + self.transition_tier = info.tier; + self.transition_status = info.status; + self.remote_object = info.name; + self.transition_version_id = info.version_id; + } + + pub fn should_remove_remote_object(&self) -> Option { + if self.transition_status != lifecycle::TRANSITION_COMPLETE { + return None; + } + + let mut del_tier = false; + if !self.versioned || self.suspended { // 1, 2.a, 2.b + del_tier = true; + } else if self.versioned && self.version_id != "" { // 3.a + del_tier = true; + } + if del_tier { + return Some(Jentry { + obj_name: self.remote_object.clone(), + version_id: self.transition_version_id.clone(), + tier_name: self.transition_tier.clone(), + }); + } + None + } + + pub async fn sweep(&self) { + let je = self.should_remove_remote_object(); + if !je.is_none() { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_tier_journal_entry(&je.expect("err!")); + } + } +} + +#[derive(Debug, Clone)] +pub struct Jentry { + obj_name: String, + version_id: String, + tier_name: String, +} + +impl ExpiryOp for Jentry { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.tier_name).as_bytes()); + let _ = hasher.write(format!("{}", self.obj_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { + let mut config_mgr = GLOBAL_TierConfigMgr.write().await; + let w = match config_mgr.get_driver(tier_name).await { + Ok(w) => w, + Err(e) => return Err(std::io::Error::other(e)), + }; + w.remove(obj_name, rv_id).await +} + + +#[cfg(test)] +mod test { + +} diff --git a/ecstore/src/bucket/mod.rs b/ecstore/src/bucket/mod.rs index a4e79c93..e9ce0b7c 100644 --- a/ecstore/src/bucket/mod.rs +++ b/ecstore/src/bucket/mod.rs @@ -10,3 +10,4 @@ pub mod target; pub mod utils; pub mod versioning; pub mod versioning_sys; +pub mod lifecycle; \ No newline at end of file diff --git a/ecstore/src/bucket/object_lock/mod.rs b/ecstore/src/bucket/object_lock/mod.rs index 38eb4886..64b834c9 100644 --- a/ecstore/src/bucket/object_lock/mod.rs +++ b/ecstore/src/bucket/object_lock/mod.rs @@ -1,3 +1,6 @@ +pub mod objectlock; +pub mod objectlock_sys; + use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; pub trait ObjectLockApi { diff --git a/ecstore/src/bucket/object_lock/objectlock.rs b/ecstore/src/bucket/object_lock/objectlock.rs new file mode 100644 index 00000000..0771d54e --- /dev/null +++ b/ecstore/src/bucket/object_lock/objectlock.rs @@ -0,0 +1,95 @@ +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use time::{OffsetDateTime, format_description}; +use tracing::{error, warn}; + +use s3s::dto::{ + ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold, + Date, +}; +use s3s::header::{ + X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, +}; + +//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; +//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; +//const AMZ_OBJECTLOCK_MODE: &str = "X-Amz-Object-Lock-Mode"; +//const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold"; + +const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config"; +const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; +const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; +const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; +const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; +const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; +const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; + +pub fn utc_now_ntp() -> OffsetDateTime { + return OffsetDateTime::now_utc(); +} + +pub fn get_object_retention_meta(meta: HashMap) -> ObjectLockRetention { + let mode: ObjectLockRetentionMode; + let mut retain_until_date: Date = Date::from(OffsetDateTime::UNIX_EPOCH); + + let mut mode_str = meta.get(X_AMZ_OBJECT_LOCK_MODE.as_str().to_lowercase().as_str()); + if mode_str.is_none() { + mode_str = Some(&meta[X_AMZ_OBJECT_LOCK_MODE.as_str()]); + } + if let Some(mode_str) = mode_str { + mode = parse_ret_mode(mode_str.as_str()); + } else { + return ObjectLockRetention {mode: None, retain_until_date: None}; + } + + let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str()); + if till_str.is_none() { + till_str = Some(&meta[X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()]); + } + if let Some(till_str) = till_str { + let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT); + if t.is_err() { + retain_until_date = Date::from(t.expect("err")); //TODO: utc + } + } + ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)} +} + +pub fn get_object_legalhold_meta(meta: HashMap) -> ObjectLockLegalHold { + let mut hold_str = meta.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_lowercase().as_str()); + if hold_str.is_none() { + hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]); + } + if let Some(hold_str) = hold_str { + return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))}; + } + ObjectLockLegalHold {status: None} +} + +pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode { + let mut mode; + match mode_str.to_uppercase().as_str() { + "GOVERNANCE" => { + mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE); + } + "COMPLIANCE" => { + mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE); + } + _ => unreachable!() + } + mode +} + +pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus { + let mut st; + match hold_str { + "ON" => { + st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON); + } + "OFF" => { + st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF); + } + _ => unreachable!() + } + st +} diff --git a/ecstore/src/bucket/object_lock/objectlock_sys.rs b/ecstore/src/bucket/object_lock/objectlock_sys.rs new file mode 100644 index 00000000..59eda77b --- /dev/null +++ b/ecstore/src/bucket/object_lock/objectlock_sys.rs @@ -0,0 +1,55 @@ +use std::any::{Any, TypeId}; +use std::sync::Arc; +use time::OffsetDateTime; +use tracing::{error, warn}; + +use s3s::dto::{ + DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus, +}; + +use crate::store_api::ObjectInfo; +use crate::bucket::metadata_sys::get_object_lock_config; + +use super::objectlock; + +pub struct BucketObjectLockSys {} + +impl BucketObjectLockSys { + #[allow(clippy::new_ret_no_self)] + pub async fn new() -> Arc { + Arc::new(Self {}) + } + + pub async fn get(bucket: &str) -> Option { + if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule { + return object_lock_rule.default_retention; + } + None + } +} + +pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool { + if obj_info.delete_marker { + return false; + } + + let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err")); + match lhold.status { + Some(st) if st.as_str()==ObjectLockLegalHoldStatus::ON => { + return true; + } + _ => () + } + + let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err")); + match ret.mode { + Some(r) if (r.as_str() == ObjectLockRetentionMode::COMPLIANCE || r.as_str() == ObjectLockRetentionMode::GOVERNANCE) => { + let t = objectlock::utc_now_ntp(); + if OffsetDateTime::from(ret.retain_until_date.expect("err!")).unix_timestamp() > t.unix_timestamp() { + return true; + } + } + _ => () + } + false +} diff --git a/ecstore/src/checksum.rs b/ecstore/src/checksum.rs new file mode 100644 index 00000000..dad39390 --- /dev/null +++ b/ecstore/src/checksum.rs @@ -0,0 +1,310 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; +use std::ops::{BitAnd, BitOr}; +use lazy_static::lazy_static; + +use reader::hasher::{Hasher, Sha256}; +use s3s::header::{ + X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, +}; +use crate::client::{ + api_put_object::PutObjectOptions, + api_s3_datatypes::ObjectPart, +}; +use rustfs_utils::crypto::{base64_decode, base64_encode}; +use crate::{ + disk::DiskAPI, + store_api::GetObjectReader, +}; + +use enumset::{enum_set, EnumSet, EnumSetType}; + +#[derive(Debug, EnumSetType, Default)] +#[enumset(repr = "u8")] +pub enum ChecksumMode { + #[default] + ChecksumNone, + ChecksumSHA256, + ChecksumSHA1, + ChecksumCRC32, + ChecksumCRC32C, + ChecksumCRC64NVME, + ChecksumFullObject, +} + +lazy_static! { + static ref C_ChecksumMask: EnumSet = { + let mut s = EnumSet::all(); + s.remove(ChecksumMode::ChecksumFullObject); + s + }; + static ref C_ChecksumFullObjectCRC32: EnumSet = enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); + static ref C_ChecksumFullObjectCRC32C: EnumSet = enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); +} +const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme"; + +impl ChecksumMode { + //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659; + + pub fn base(&self) -> ChecksumMode { + let s = EnumSet::from(*self).intersection(*C_ChecksumMask); + match s.as_u8() { + 1_u8 => { + ChecksumMode::ChecksumNone + } + 2_u8 => { + ChecksumMode::ChecksumSHA256 + } + 4_u8 => { + ChecksumMode::ChecksumSHA1 + } + 8_u8 => { + ChecksumMode::ChecksumCRC32 + } + 16_u8 => { + ChecksumMode::ChecksumCRC32C + } + 32_u8 => { + ChecksumMode::ChecksumCRC64NVME + } + _ => panic!("enum err."), + } + } + + pub fn is(&self, t: ChecksumMode) -> bool { + *self & t == t + } + + pub fn key(&self) -> String { + //match c & checksumMask { + match self { + ChecksumMode::ChecksumCRC32 => { + return X_AMZ_CHECKSUM_CRC32.to_string(); + } + ChecksumMode::ChecksumCRC32C => { + return X_AMZ_CHECKSUM_CRC32C.to_string(); + } + ChecksumMode::ChecksumSHA1 => { + return X_AMZ_CHECKSUM_SHA1.to_string(); + } + ChecksumMode::ChecksumSHA256 => { + return X_AMZ_CHECKSUM_SHA256.to_string(); + } + ChecksumMode::ChecksumCRC64NVME => { + return AMZ_CHECKSUM_CRC64NVME.to_string(); + } + _ => { + return "".to_string(); + } + } + } + + pub fn can_composite(&self) -> bool { + todo!(); + } + + pub fn can_merge_crc(&self) -> bool { + todo!(); + } + + pub fn full_object_requested(&self) -> bool { + todo!(); + } + + pub fn key_capitalized(&self) -> String { + self.key() + } + + pub fn raw_byte_len(&self) -> usize { + let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8(); + if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 { + 4 + } + else if u == ChecksumMode::ChecksumSHA1 as u8 { + 4//sha1.size + } + else if u == ChecksumMode::ChecksumSHA256 as u8 { + 4//sha256.size + } + else if u == ChecksumMode::ChecksumCRC64NVME as u8 { + 4//crc64.size + } + else { + 0 + } + } + + pub fn hasher(&self) -> Result, std::io::Error> { + match /*C_ChecksumMask & **/self { + /*ChecksumMode::ChecksumCRC32 => { + return Ok(Box::new(crc32fast::Hasher::new())); + }*/ + /*ChecksumMode::ChecksumCRC32C => { + return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli)))); + } + ChecksumMode::ChecksumSHA1 => { + return Ok(Box::new(sha1::new())); + }*/ + ChecksumMode::ChecksumSHA256 => { + return Ok(Box::new(Sha256::new())); + } + /*ChecksumMode::ChecksumCRC64NVME => { + return Ok(Box::new(crc64nvme.New()); + }*/ + _ => return Err(std::io::Error::other("unsupported checksum type")), + } + } + + pub fn is_set(&self) -> bool { + let s = EnumSet::from(*self).intersection(*C_ChecksumMask); + s.len() == 1 + } + + pub fn set_default(&mut self, t: ChecksumMode) { + if !self.is_set() { + *self = t; + } + } + + pub fn encode_to_string(&self, b: &[u8]) -> Result { + if !self.is_set() { + return Ok("".to_string()); + } + let mut h = self.hasher()?; + h.write(b); + Ok(base64_encode(h.sum().as_bytes())) + } + + pub fn to_string(&self) -> String { + //match c & checksumMask { + match self { + ChecksumMode::ChecksumCRC32 => { + return "CRC32".to_string(); + } + ChecksumMode::ChecksumCRC32C => { + return "CRC32C".to_string(); + } + ChecksumMode::ChecksumSHA1 => { + return "SHA1".to_string(); + } + ChecksumMode::ChecksumSHA256 => { + return "SHA256".to_string(); + } + ChecksumMode::ChecksumNone => { + return "".to_string(); + } + ChecksumMode::ChecksumCRC64NVME => { + return "CRC64NVME".to_string(); + } + _=> { + return "".to_string(); + } + } + } + + pub fn check_sum_reader(&self, r: GetObjectReader) -> Result { + let mut h = self.hasher()?; + Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + } + + pub fn check_sum_bytes(&self, b: &[u8]) -> Result { + let mut h = self.hasher()?; + Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + } + + pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result { + if !self.can_composite() { + return Err(std::io::Error::other("cannot do composite checksum")); + } + p.sort_by(|i, j| { + if i.part_num < j.part_num { + std::cmp::Ordering::Less + } else if i.part_num > j.part_num { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }); + let c = self.base(); + let mut crc_bytes = Vec::::with_capacity(p.len()*self.raw_byte_len() as usize); + let mut h = self.hasher()?; + h.write(&crc_bytes); + Ok(Checksum {checksum_type: self.clone(), r: h.sum().as_bytes().to_vec()}) + } + + pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result { + todo!(); + } +} + +#[derive(Default)] +struct Checksum { + checksum_type: ChecksumMode, + r: Vec, +} + +impl Checksum { + fn new(t: ChecksumMode, b: &[u8]) -> Checksum { + if t.is_set() && b.len() == t.raw_byte_len() { + return Checksum {checksum_type: t, r: b.to_vec()}; + } + Checksum::default() + } + + fn new_checksum_string(t: ChecksumMode, s: &str) -> Result { + let b = match base64_decode(s.as_bytes()) { + Ok(b) => b, + Err(err) => return Err(std::io::Error::other(err.to_string())), + }; + if t.is_set() && b.len() == t.raw_byte_len() { + return Ok(Checksum {checksum_type: t, r: b}); + } + Ok(Checksum::default()) + } + + fn is_set(&self) -> bool { + self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len() + } + + fn encoded(&self) -> String { + if !self.is_set() { + return "".to_string(); + } + base64_encode(&self.r) + } + + fn raw(&self) -> Option> { + if !self.is_set() { + return None; + } + Some(self.r.clone()) + } +} + +pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) { + opts.user_metadata.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); + if opts.auto_checksum.full_object_requested() { + opts.user_metadata.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + } +} + +pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> { + if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) { + let crc = opts.auto_checksum.composite_checksum(all_parts)?; + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key(), crc.encoded()); + hm + } + } else if opts.auto_checksum.can_merge_crc() { + let crc = opts.auto_checksum.full_object_checksum(all_parts)?; + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded()); + hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + hm + } + } + + Ok(()) +} diff --git a/ecstore/src/client/admin_handler_utils.rs b/ecstore/src/client/admin_handler_utils.rs new file mode 100644 index 00000000..63c2e583 --- /dev/null +++ b/ecstore/src/client/admin_handler_utils.rs @@ -0,0 +1,33 @@ +use http::status::StatusCode; +use std::fmt::{self, Display, Formatter}; + +#[derive(Default, thiserror::Error, Debug, PartialEq)] +pub struct AdminError { + pub code: &'static str, + pub message: &'static str, + pub status_code: StatusCode, +} + +impl Display for AdminError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl AdminError { + pub fn new(code: &'static str, message: &'static str, status_code: StatusCode) -> Self { + Self { + code, + message, + status_code, + } + } + + pub fn msg(message: &'static str) -> Self { + Self { + code: "InternalError", + message, + status_code: StatusCode::INTERNAL_SERVER_ERROR, + } + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_bucket_policy.rs b/ecstore/src/client/api_bucket_policy.rs new file mode 100644 index 00000000..65b4abf5 --- /dev/null +++ b/ecstore/src/client/api_bucket_policy.rs @@ -0,0 +1,113 @@ +#![allow(clippy::map_entry)] +use std::collections::HashMap; +use http::{HeaderMap, StatusCode}; +use bytes::Bytes; + +use crate::client::{ + api_error_response::http_resp_to_error_response, + transition_api::{RequestMetadata, TransitionClient, ReaderImpl} +}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +impl TransitionClient { + pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> { + if policy == "" { + return self.remove_bucket_policy(bucket_name).await; + } + + self.put_bucket_policy(bucket_name, policy).await + } + + pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())), + content_length: policy.len() as i64, + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_md5_base64: "".to_string(), + content_sha256_hex: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + //defer closeResponse(resp) + //if resp != nil { + if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + //} + Ok(()) + } + + pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + //defer closeResponse(resp) + + if resp.status() != StatusCode::NO_CONTENT { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + + Ok(()) + } + + pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result { + let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?; + Ok(bucket_policy) + } + + pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); + Ok(policy) + } +} diff --git a/ecstore/src/client/api_error_response.rs b/ecstore/src/client/api_error_response.rs new file mode 100644 index 00000000..1c9d74a3 --- /dev/null +++ b/ecstore/src/client/api_error_response.rs @@ -0,0 +1,252 @@ +#![allow(clippy::map_entry)] +use std::fmt::Display; +use http::StatusCode; +use serde::{Serialize, Deserialize}; +use serde::{ser::Serializer, de::Deserializer}; + +use s3s::S3ErrorCode; +use s3s::Body; + +const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues."; + +#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] +#[serde(default, rename_all = "PascalCase")] +pub struct ErrorResponse { + #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")] + pub code: S3ErrorCode, + pub message: String, + pub bucket_name: String, + pub key: String, + pub resource: String, + pub request_id: String, + pub host_id: String, + pub region: String, + pub server: String, + #[serde(skip)] + pub status_code: StatusCode, +} + +fn serialize_code(data: &S3ErrorCode, s: S) -> Result +where + S: Serializer +{ + s.serialize_str("") +} + +fn deserialize_code<'de, D>(d: D) -> Result +where + D: Deserializer<'de> +{ + Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) +} + +impl Default for ErrorResponse { + fn default() -> Self { + ErrorResponse { + code: S3ErrorCode::Custom("".into()), + message: Default::default(), + bucket_name: Default::default(), + key: Default::default(), + resource: Default::default(), + request_id: Default::default(), + host_id: Default::default(), + region: Default::default(), + server: Default::default(), + status_code: Default::default(), + } + } +} + +impl Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +pub fn to_error_response(err: &std::io::Error) -> ErrorResponse { + if let Some(err) = err.get_ref() { + if err.is::() { + err.downcast_ref::().expect("err!").clone() + } else { + ErrorResponse::default() + } + } else { + ErrorResponse::default() + } +} + +pub fn http_resp_to_error_response(resp: http::Response, b: Vec, bucket_name: &str, object_name: &str) -> ErrorResponse { + let err_body = String::from_utf8(b).unwrap(); + //let err_body = xml_decode_and_body(resp.body, &err_resp); + let err_resp_ = serde_xml_rs::from_str::(&err_body); + let mut err_resp = ErrorResponse::default(); + if err_resp_.is_err() { + match resp.status() { + StatusCode::NOT_FOUND => { + if object_name == "" { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::NoSuchBucket, + message: "The specified bucket does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } else { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::NoSuchKey, + message: "The specified key does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + } + StatusCode::FORBIDDEN => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::AccessDenied, + message: "Access Denied.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + StatusCode::CONFLICT => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::BucketNotEmpty, + message: "Bucket not empty.".to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } + StatusCode::PRECONDITION_FAILED => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::PreconditionFailed, + message: "Pre condition failed.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + _ => { + let mut msg = resp.status().to_string(); + if err_body.len() > 0 { + msg = err_body; + } + err_resp = ErrorResponse{ + status_code: resp.status(), + code: S3ErrorCode::Custom(resp.status().to_string().into()), + message: msg, + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } + } + } else { + err_resp = err_resp_.unwrap(); + } + err_resp.status_code = resp.status(); + if let Some(server_name) = resp.headers().get("Server") { + err_resp.server = server_name.to_str().expect("err").to_string(); + } + + let code = resp.headers().get("x-minio-error-code"); + if code.is_some() { + err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into()); + } + let desc = resp.headers().get("x-minio-error-desc"); + if desc.is_some() { + err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string(); + } + + if err_resp.request_id == "" { + if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") { + err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string(); + } + } + if err_resp.host_id == "" { + if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") { + err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string(); + } + } + if err_resp.region == "" { + if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") { + err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string(); + } + } + if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" { + err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region); + } + + err_resp +} + +pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::InvalidArgument, + message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.".to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + } +} + +pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!("Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", total_size, max_object_size); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::EntityTooLarge, + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!("Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", total_size); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::EntityTooSmall, + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!("Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", total_read, total_size); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::Custom("UnexpectedEOF".into()), + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_invalid_argument(message: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::InvalidArgument, + message: message.to_string(), + request_id: "rustfs".to_string(), + ..Default::default() + } +} + +pub fn err_api_not_supported(message: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::NOT_IMPLEMENTED, + code: S3ErrorCode::Custom("APINotSupported".into()), + message: message.to_string(), + request_id: "rustfs".to_string(), + ..Default::default() + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_get_object.rs b/ecstore/src/client/api_get_object.rs new file mode 100644 index 00000000..d443866c --- /dev/null +++ b/ecstore/src/client/api_get_object.rs @@ -0,0 +1,198 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::HeaderMap; +use tokio::io::BufReader; +use std::io::Cursor; + +use crate::client::{ + transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, + api_error_response::err_invalid_argument, + api_get_options::GetObjectOptions, +}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +impl TransitionClient { + pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result { + todo!(); + } + + pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: opts.to_query_values(), + custom_header: opts.header(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + let resp = &resp; + let object_stat = to_object_info(bucket_name, object_name, resp.headers())?; + + let b = resp.body().bytes().expect("err").to_vec(); + Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b)))) + } +} + +#[derive(Default)] +struct GetRequest { + pub buffer: Vec, + pub offset: i64, + pub did_offset_change: bool, + pub been_read: bool, + pub is_read_at: bool, + pub is_read_op: bool, + pub is_first_req: bool, + pub setting_object_info: bool, +} + +struct GetResponse { + pub size: i64, + //pub error: error, + pub did_read: bool, + pub object_info: ObjectInfo, +} + +#[derive(Default)] +struct Object { + //pub reqch: chan<- getRequest, + //pub resch: <-chan getResponse, + //pub cancel: context.CancelFunc, + pub curr_offset: i64, + pub object_info: ObjectInfo, + pub seek_data: bool, + pub is_closed: bool, + pub is_started: bool, + //pub prev_err: error, + pub been_read: bool, + pub object_info_set: bool, +} + +impl Object { + pub fn new() -> Object { + Self { + ..Default::default() + } + } + + fn do_get_request(&self, request: &GetRequest) -> Result { + todo!() + } + + fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> { + self.curr_offset += bytes_read; + + Ok(()) + } + + fn read(&mut self, b: &[u8]) -> Result { + let mut read_req = GetRequest { + is_read_op: true, + been_read: self.been_read, + buffer: b.to_vec(), + ..Default::default() + }; + + if !self.is_started { + read_req.is_first_req = true; + } + + read_req.did_offset_change = self.seek_data; + read_req.offset = self.curr_offset; + + let response = self.do_get_request(&read_req)?; + + let bytes_read = response.size; + + let oerr = self.set_offset(bytes_read); + + Ok(response.size) + } + + fn stat(&self) -> Result { + if !self.is_started || !self.object_info_set { + let _ = self.do_get_request(&GetRequest { + is_first_req: !self.is_started, + setting_object_info: !self.object_info_set, + ..Default::default() + })?; + } + + Ok(self.object_info.clone()) + } + + fn read_at(&mut self, b: &[u8], offset: i64) -> Result { + self.curr_offset = offset; + + let mut read_at_req = GetRequest { + is_read_op: true, + is_read_at: true, + did_offset_change: true, + been_read: self.been_read, + offset, + buffer: b.to_vec(), + ..Default::default() + }; + + if !self.is_started { + read_at_req.is_first_req = true; + } + + let response = self.do_get_request(&read_at_req)?; + let bytes_read = response.size; + if !self.object_info_set { + self.curr_offset += bytes_read; + } else { + let oerr = self.set_offset(bytes_read); + } + Ok(response.size) + } + + fn seek(&mut self, offset: i64, whence: i64) -> Result { + if !self.is_started || !self.object_info_set { + let seek_req = GetRequest { + is_read_op: false, + offset: offset, + is_first_req: true, + ..Default::default() + }; + let _ = self.do_get_request(&seek_req); + } + + let mut new_offset = self.curr_offset; + + match whence { + 0 => { + new_offset = offset; + } + 1 => { + new_offset += offset; + } + 2 => { + new_offset = self.object_info.size as i64 + offset as i64; + } + _ => { + return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence)))); + } + } + + self.seek_data = (new_offset != self.curr_offset) || self.seek_data; + self.curr_offset = new_offset; + + Ok(self.curr_offset) + } + + fn close(&mut self) -> Result<(), std::io::Error> { + self.is_closed = true; + Ok(()) + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_get_options.rs b/ecstore/src/client/api_get_options.rs new file mode 100644 index 00000000..842a02e0 --- /dev/null +++ b/ecstore/src/client/api_get_options.rs @@ -0,0 +1,127 @@ +#![allow(clippy::map_entry)] +use std::collections::HashMap; +use http::{HeaderMap, HeaderName, HeaderValue}; +use time::OffsetDateTime; +use tracing::warn; + +use crate::client::api_error_response::err_invalid_argument; + +#[derive(Default)] +pub struct AdvancedGetOptions { + replication_deletemarker: bool, + is_replication_ready_for_deletemarker: bool, + replication_proxy_request: String, +} + +pub struct GetObjectOptions { + pub headers: HashMap, + pub req_params: HashMap, + //pub server_side_encryption: encrypt.ServerSide, + pub version_id: String, + pub part_number: i64, + pub checksum: bool, + pub internal: AdvancedGetOptions, +} + +type StatObjectOptions = GetObjectOptions; + +impl Default for GetObjectOptions { + fn default() -> Self { + Self { + headers: HashMap::new(), + req_params: HashMap::new(), + //server_side_encryption: encrypt.ServerSide::default(), + version_id: "".to_string(), + part_number: 0, + checksum: false, + internal: AdvancedGetOptions::default(), + } + } +} + +impl GetObjectOptions { + pub fn header(&self) -> HeaderMap { + let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len()); + for (k, v) in &self.headers { + if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { + headers.insert(header_name, v.parse().expect("err")); + } else { + warn!("Invalid header name: {}", k); + } + } + if self.checksum { + headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err")); + } + headers + } + + pub fn set(&self, key: &str, value: &str) { + //self.headers[http.CanonicalHeaderKey(key)] = value; + } + + pub fn set_req_param(&mut self, key: &str, value: &str) { + self.req_params.insert(key.to_string(), value.to_string()); + } + + pub fn add_req_param(&mut self, key: &str, value: &str) { + self.req_params.insert(key.to_string(), value.to_string()); + } + + pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> { + self.set("If-Match", &format!("\"{etag}\"")); + Ok(()) + } + + pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> { + self.set("If-None-Match", &format!("\"{etag}\"")); + Ok(()) + } + + pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> { + if mod_time.unix_timestamp() == 0 { + return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty."))); + } + self.set("If-Unmodified-Since", &mod_time.to_string()); + Ok(()) + } + + pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> { + if mod_time.unix_timestamp() == 0 { + return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty."))); + } + self.set("If-Modified-Since", &mod_time.to_string()); + Ok(()) + } + + pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> { + if start == 0 && end < 0 { + self.set("Range", &format!("bytes={}", end)); + } + else if 0 < start && end == 0 { + self.set("Range", &format!("bytes={}-", start)); + } + else if 0 <= start && start <= end { + self.set("Range", &format!("bytes={}-{}", start, end)); + } + else { + return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end)))); + } + Ok(()) + } + + pub fn to_query_values(&self) -> HashMap { + let mut url_values = HashMap::new(); + if self.version_id != "" { + url_values.insert("versionId".to_string(), self.version_id.clone()); + } + if self.part_number > 0 { + url_values.insert("partNumber".to_string(), self.part_number.to_string()); + } + + for (key, value) in self.req_params.iter() { + url_values.insert(key.to_string(), value.to_string()); + } + + url_values + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_list.rs b/ecstore/src/client/api_list.rs new file mode 100644 index 00000000..2c81bf14 --- /dev/null +++ b/ecstore/src/client/api_list.rs @@ -0,0 +1,229 @@ +#![allow(clippy::map_entry)] +use std::collections::HashMap; +use bytes::Bytes; +use http::{HeaderMap, StatusCode}; + +use crate::client::{ + api_error_response::http_resp_to_error_response, + credentials, + api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart}, + transition_api::{ReaderImpl, TransitionClient, RequestMetadata,}, +}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use crate::store_api::BucketInfo; + +impl TransitionClient { + pub fn list_buckets(&self) -> Result, std::io::Error> { + todo!(); + } + + pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result { + let mut url_values = HashMap::new(); + + url_values.insert("list-type".to_string(), "2".to_string()); + if metadata { + url_values.insert("metadata".to_string(), "true".to_string()); + } + if start_after != "" { + url_values.insert("start-after".to_string(), start_after.to_string()); + } + url_values.insert("encoding-type".to_string(), "url".to_string()); + url_values.insert("prefix".to_string(), object_prefix.to_string()); + url_values.insert("delimiter".to_string(), delimiter.to_string()); + + if continuation_token != "" { + url_values.insert("continuation-token".to_string(), continuation_token.to_string()); + } + + if fetch_owner { + url_values.insert("fetch-owner".to_string(), "true".to_string()); + } + + if max_keys > 0 { + url_values.insert("max-keys".to_string(), max_keys.to_string()); + } + + let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: "".to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + + //let mut list_bucket_result = ListBucketV2Result::default(); + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let mut list_bucket_result = match serde_xml_rs::from_str::(&String::from_utf8(b).unwrap()) { + Ok(result) => result, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + //println!("list_bucket_result: {:?}", list_bucket_result); + + if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" { + return Err(std::io::Error::other(credentials::ErrorResponse { + sts_error: credentials::STSError { + r#type: "".to_string(), + code: "NotImplemented".to_string(), + message: "Truncated response should have continuation token set".to_string(), + }, + request_id: "".to_string(), + })); + } + + for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() { + obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?; + //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond); + } + + for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() { + obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?; + } + + Ok(list_bucket_result) + } + + pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result { + /*if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListVersionsResult{}, err + } + if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { + return ListVersionsResult{}, err + } + urlValues := make(url.Values) + + urlValues.Set("versions", "") + + urlValues.Set("prefix", opts.Prefix) + + urlValues.Set("delimiter", delimiter) + + if keyMarker != "" { + urlValues.Set("key-marker", keyMarker) + } + + if opts.max_keys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys)) + } + + if versionIDMarker != "" { + urlValues.Set("version-id-marker", versionIDMarker) + } + + if opts.WithMetadata { + urlValues.Set("metadata", "true") + } + + urlValues.Set("encoding-type", "url") + + let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + customHeader: opts.headers, + }).await?; + defer closeResponse(resp) + if err != nil { + return ListVersionsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + listObjectVersionsOutput := ListVersionsResult{} + err = xml_decoder(resp.Body, &listObjectVersionsOutput) + if err != nil { + return ListVersionsResult{}, err + } + + for i, obj := range listObjectVersionsOutput.Versions { + listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + for i, obj := range listObjectVersionsOutput.CommonPrefixes { + listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + if listObjectVersionsOutput.NextKeyMarker != "" { + listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + Ok(listObjectVersionsOutput)*/ + todo!(); + } + + pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result { + todo!(); + } + + pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result { + todo!(); + } + + pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result, std::io::Error> { + todo!(); + } + + pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result, std::io::Error> { + todo!(); + } + + pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result { + todo!(); + } +} + +pub struct ListObjectsOptions { + reverse_versions: bool, + with_versions: bool, + with_metadata: bool, + prefix: String, + recursive: bool, + max_keys: i64, + start_after: String, + use_v1: bool, + headers: HeaderMap, +} + +impl ListObjectsOptions { + pub fn set(&mut self, key: &str, value: &str) { + todo!(); + } +} + +fn decode_s3_name(name: &str, encoding_type: &str) -> Result { + match encoding_type { + "url" => { + //return url::QueryUnescape(name); + return Ok(name.to_string()); + } + _ => { + return Ok(name.to_string()); + } + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_put_object.rs b/ecstore/src/client/api_put_object.rs new file mode 100644 index 00000000..e1209cda --- /dev/null +++ b/ecstore/src/client/api_put_object.rs @@ -0,0 +1,382 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; +use bytes::Bytes; +use time::{OffsetDateTime, macros::format_description, Duration}; +use http::{HeaderMap, HeaderName, HeaderValue}; +use tracing::{error, info, warn}; + +use s3s::dto::{ + ObjectLockRetentionMode, ObjectLockLegalHoldStatus, + ReplicationStatus, +}; +use s3s::header::{ + X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, + X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, X_AMZ_REPLICATION_STATUS, +}; +use reader::hasher::Hasher; +//use crate::disk::{BufferReader, Reader}; +use rustfs_utils::{ + crypto::base64_encode, + net::{is_amz_header, is_standard_header, is_storageclass_header, is_rustfs_header, is_minio_header}, +}; +use crate::checksum::ChecksumMode; +use crate::client::{ + api_s3_datatypes::{CompletePart, ObjectPart, CompleteMultipartUpload}, + api_put_object_common::optimal_part_info, + transition_api::{TransitionClient, UploadInfo, ReaderImpl}, + api_error_response::{err_invalid_argument, err_entity_too_large}, + api_put_object_multipart::UploadPartParams, + credentials::SignatureType, + constants::{MAX_MULTIPART_PUT_OBJECT_SIZE, TOTAL_WORKERS, MIN_PART_SIZE, ISO8601_DATEFORMAT,}, +}; + +#[derive(Debug, Clone)] +pub struct AdvancedPutOptions { + pub source_version_id: String, + pub source_etag: String, + pub replication_status: ReplicationStatus, + pub source_mtime: OffsetDateTime, + pub replication_request: bool, + pub retention_timestamp: OffsetDateTime, + pub tagging_timestamp: OffsetDateTime, + pub legalhold_timestamp: OffsetDateTime, + pub replication_validity_check: bool, +} + +impl Default for AdvancedPutOptions { + fn default() -> Self { + Self { + source_version_id: "".to_string(), + source_etag: "".to_string(), + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + source_mtime: OffsetDateTime::now_utc(), + replication_request: false, + retention_timestamp: OffsetDateTime::now_utc(), + tagging_timestamp: OffsetDateTime::now_utc(), + legalhold_timestamp: OffsetDateTime::now_utc(), + replication_validity_check: false, + } + } +} + +#[derive(Clone)] +pub struct PutObjectOptions { + pub user_metadata: HashMap, + pub user_tags: HashMap, + //pub progress: ReaderImpl, + pub content_type: String, + pub content_encoding: String, + pub content_disposition: String, + pub content_language: String, + pub cache_control: String, + pub expires: OffsetDateTime, + pub mode: ObjectLockRetentionMode, + pub retain_until_date: OffsetDateTime, + //pub server_side_encryption: encrypt.ServerSide, + pub num_threads: u64, + pub storage_class: String, + pub website_redirect_location: String, + pub part_size: u64, + pub legalhold: ObjectLockLegalHoldStatus, + pub send_content_md5: bool, + pub disable_content_sha256: bool, + pub disable_multipart: bool, + pub auto_checksum: ChecksumMode, + pub checksum: ChecksumMode, + pub concurrent_stream_parts: bool, + pub internal: AdvancedPutOptions, + pub custom_header: HeaderMap, +} + +impl Default for PutObjectOptions { + fn default() -> Self { + Self { + user_metadata: HashMap::new(), + user_tags: HashMap::new(), + //progress: ReaderImpl::Body(Bytes::new()), + content_type: "".to_string(), + content_encoding: "".to_string(), + content_disposition: "".to_string(), + content_language: "".to_string(), + cache_control: "".to_string(), + expires: OffsetDateTime::UNIX_EPOCH, + mode: ObjectLockRetentionMode::from_static(""), + retain_until_date: OffsetDateTime::UNIX_EPOCH, + //server_side_encryption: encrypt.ServerSide::default(), + num_threads: 0, + storage_class: "".to_string(), + website_redirect_location: "".to_string(), + part_size: 0, + legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF), + send_content_md5: false, + disable_content_sha256: false, + disable_multipart: false, + auto_checksum: ChecksumMode::ChecksumNone, + checksum: ChecksumMode::ChecksumNone, + concurrent_stream_parts: false, + internal: AdvancedPutOptions::default(), + custom_header: HeaderMap::new(), + } + } +} + +impl PutObjectOptions { + fn set_matche_tag(&mut self, etag: &str) { + if etag == "*" { + self.custom_header.insert("If-Match", HeaderValue::from_str("*").expect("err")); + } else { + self.custom_header.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err")); + } + } + + fn set_matche_tag_except(&mut self, etag: &str) { + if etag == "*" { + self.custom_header.insert("If-None-Match", HeaderValue::from_str("*").expect("err")); + } else { + self.custom_header.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err")); + } + } + + pub fn header(&self) -> HeaderMap { + let mut header = HeaderMap::new(); + + let mut content_type = self.content_type.clone(); + if content_type == "" { + content_type = "application/octet-stream".to_string(); + } + header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err")); + + if self.content_encoding != "" { + header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err")); + } + if self.content_disposition != "" { + header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err")); + } + if self.content_language != "" { + header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err")); + } + if self.cache_control != "" { + header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err")); + } + + if self.expires.unix_timestamp() != 0 { + header.insert("Expires", HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); //rustfs invalid heade + } + + if self.mode.as_str() != "" { + header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err")); + } + + if self.retain_until_date.unix_timestamp() != 0 { + header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); + } + + if self.legalhold.as_str() != "" { + header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err")); + } + + if self.storage_class != "" { + header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err")); + } + + if self.website_redirect_location != "" { + header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, HeaderValue::from_str(&self.website_redirect_location).expect("err")); + } + + if !self.internal.replication_status.as_str().is_empty() { + header.insert(X_AMZ_REPLICATION_STATUS, HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err")); + } + + for (k, v) in &self.user_metadata { + if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) { + if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { + header.insert(header_name, HeaderValue::from_str(&v).unwrap()); + } + } else { + if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) { + header.insert(header_name, HeaderValue::from_str(&v).unwrap()); + } + } + } + + for (k, v) in self.custom_header.iter() { + header.insert(k.clone(), v.clone()); + } + + header + } + + fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { + //if self.checksum.is_set() { + /*if !self.trailing_header_support { + return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled"))); + }*/ + /*else if self.override_signer_type == SignatureType::SignatureV2 { + return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures"))); + }*/ + //} + + Ok(()) + } +} + +impl TransitionClient { + pub async fn put_object(self: Arc, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64, + opts: &PutObjectOptions + ) -> Result { + if object_size < 0 && opts.disable_multipart { + return Err(std::io::Error::other("object size must be provided with disable multipart upload")); + } + + self.put_object_common(bucket_name, object_name, reader, object_size, opts).await + } + + pub async fn put_object_common(self: Arc, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result { + if size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name))); + } + let mut opts = opts.clone(); + opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); + + let mut part_size = opts.part_size as i64; + if opts.part_size == 0 { + part_size = MIN_PART_SIZE; + } + + if SignatureType::SignatureV2 == self.override_signer_type { + if size >= 0 && size < part_size || opts.disable_multipart { + return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; + } + return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await; + } + + if size < 0 { + if opts.disable_multipart { + return Err(std::io::Error::other("no length provided and multipart disabled")); + } + if opts.concurrent_stream_parts && opts.num_threads > 1 { + return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await; + } + return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await; + } + + if size <= part_size || opts.disable_multipart { + return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; + } + + self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await + } + + pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result { + let mut total_uploaded_size: i64 = 0; + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?; + + let mut opts = opts.clone(); + + if opts.checksum.is_set() { + opts.send_content_md5 = false; + opts.auto_checksum = opts.checksum.clone(); + } + if !opts.send_content_md5 { + //add_auto_checksum_headers(&mut opts); + } + + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut part_number = 1; + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + + let mut custom_header = HeaderMap::new(); + + while part_number <= total_parts_count { + buf = match &mut reader { + ReaderImpl::Body(content_body) => { + content_body.to_vec() + } + ReaderImpl::ObjectBody(content_body) => { + content_body.read_all().await? + } + }; + let length = buf.len(); + + let mut md5_base64: String = "".to_string(); + if opts.send_content_md5 { + let mut md5_hasher = self.md5_hasher.lock().unwrap(); + let mut hash = md5_hasher.as_mut().expect("err"); + hash.write(&buf[..length]); + md5_base64 = base64_encode(hash.sum().as_bytes()); + } else { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap()); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress); + let rd = ReaderImpl::Body(Bytes::from(buf)); + + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: rd, + part_number, + md5_base64, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + sha256_hex: Default::default(), + trailer: Default::default(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.entry(part_number).or_insert(obj_part); + total_uploaded_size += length as i64; + part_number += 1; + } + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + for i in 1..part_number { + let part = parts_info[&i].clone(); + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + //apply_auto_checksum(&mut opts, all_parts); + + let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_put_object_common.rs b/ecstore/src/client/api_put_object_common.rs new file mode 100644 index 00000000..7f5c8774 --- /dev/null +++ b/ecstore/src/client/api_put_object_common.rs @@ -0,0 +1,76 @@ +#![allow(clippy::map_entry)] +use crate::client::{ + api_put_object::PutObjectOptions, + constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE}, + transition_api::TransitionClient, + transition_api::ReaderImpl, + api_error_response::{err_entity_too_large, err_invalid_argument}, +}; + +const NULL_VERSION_ID: &str = "null"; + +pub fn is_object(reader: &ReaderImpl) -> bool { + todo!(); +} + +pub fn is_read_at(reader: ReaderImpl) -> bool { + todo!(); +} + +pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> { + let unknown_size; + let mut object_size = object_size; + if object_size == -1 { + unknown_size = true; + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } else { + unknown_size = false; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", ""))); + } + + let mut part_size_flt: f64; + if configured_part_size > 0 { + if configured_part_size as i64 > object_size { + return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", ""))); + } + + if !unknown_size { + if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) { + return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize."))); + } + } + + if (configured_part_size as i64) < ABS_MIN_PART_SIZE { + return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB."))); + } + + if configured_part_size as i64 > MAX_PART_SIZE { + return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB."))); + } + + part_size_flt = configured_part_size as f64; + if unknown_size { + object_size = configured_part_size as i64 * MAX_PARTS_COUNT; + } + } else { + let mut configured_part_size = configured_part_size; + configured_part_size = MIN_PART_SIZE as u64; + part_size_flt = (object_size / MAX_PARTS_COUNT) as f64; + part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64; + } + + let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64; + let part_size = part_size_flt.ceil() as i64; + let last_part_size = object_size - (total_parts_count-1) * part_size; + Ok((total_parts_count, part_size, last_part_size)) +} + +impl TransitionClient { + pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result { + let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?; + Ok(init_multipart_upload_result.upload_id) + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_put_object_multipart.rs b/ecstore/src/client/api_put_object_multipart.rs new file mode 100644 index 00000000..2c7547cd --- /dev/null +++ b/ecstore/src/client/api_put_object_multipart.rs @@ -0,0 +1,348 @@ +#![allow(clippy::map_entry)] +use std::io::Read; +use std::{collections::HashMap, sync::Arc}; +use bytes::Bytes; +use s3s::S3ErrorCode; +use time::{format_description, OffsetDateTime}; +use uuid::Uuid; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use url::form_urlencoded::Serializer; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use tracing::{error, info}; + +use s3s::{dto::StreamingBlob, Body}; +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; +use reader::hasher::Hasher; +//use crate::disk::{Reader, BufferReader}; +use crate::client::{ + transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,}, + api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response}, + api_put_object::PutObjectOptions, + api_put_object_common::optimal_part_info, + api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart}, + constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, }, +}; +use rustfs_utils::{ + path::trim_etag, + crypto::base64_encode, +}; +use crate::{ + disk::DiskAPI, + store_api::{ + GetObjectReader, StorageAPI, + }, + checksum::ChecksumMode, +}; + +impl TransitionClient { + pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, + opts: &PutObjectOptions + ) -> Result { + let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await; + if let Err(err) = &info { + let err_resp = to_error_response(err); + if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") { + if size > MAX_SINGLE_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name))); + } + return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await; + } + } + Ok(info?) + } + + pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result { + let mut total_uploaded_size: i64 = 0; + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let ret = optimal_part_info(-1, opts.part_size)?; + let (total_parts_count, part_size, _) = ret; + + let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256); + let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?; + let mut opts = opts.clone(); + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut part_number = 1; + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + let mut custom_header = HeaderMap::new(); + while part_number <= total_parts_count { + match reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + for (k, v) in hash_algos.iter_mut() { + v.write(&buf[..length]); + hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap()); + } + + //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress); + let rd = Bytes::from(buf.clone()); + + let mut md5_base64: String; + let mut sha256_hex: String; + + //if hash_sums["md5"] != nil { + md5_base64 = base64_encode(&hash_sums["md5"]); + //} + //if hash_sums["sha256"] != nil { + sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); + //} + if hash_sums.len() == 0 { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err")); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: ReaderImpl::Body(rd), + part_number, + md5_base64, + sha256_hex, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + trailer: HeaderMap::new(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.insert(part_number, obj_part); + total_uploaded_size += length as i64; + part_number += 1; + } + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + for i in 1..part_number { + let part = parts_info[&i].clone(); + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + //apply_auto_checksum(&mut opts, all_parts); + + let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("uploads".to_string(), "".to_string()); + + if opts.internal.source_version_id != "" { + if !opts.internal.source_version_id.is_empty() { + if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) { + return Err(std::io::Error::other(err_invalid_argument(&err.to_string()))); + } + } + url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone()); + } + + let mut custom_header = opts.header(); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + custom_header, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + content_sha256_hex: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; + //if resp.is_none() { + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + //} + let initiate_multipart_upload_result = InitiateMultipartUploadResult::default(); + Ok(initiate_multipart_upload_result) + } + + pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result { + if p.size > MAX_PART_SIZE { + return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name))); + } + if p.size <= -1 { + return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name))); + } + if p.part_number <= 0 { + return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero."))); + } + if p.upload_id == "" { + return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty."))); + } + + let mut url_values = HashMap::new(); + url_values.insert("partNumber".to_string(), p.part_number.to_string()); + url_values.insert("uploadId".to_string(), p.upload_id.clone()); + + let buf = match &mut p.reader { + ReaderImpl::Body(content_body) => { + content_body.to_vec() + } + ReaderImpl::ObjectBody(content_body) => { + content_body.read_all().await? + } + }; + let mut req_metadata = RequestMetadata { + bucket_name: p.bucket_name.clone(), + object_name: p.object_name.clone(), + query_values: url_values, + custom_header: p.custom_header.clone(), + content_body: ReaderImpl::Body(Bytes::from(buf)), + content_length: p.size, + content_md5_base64: p.md5_base64.clone(), + content_sha256_hex: p.sha256_hex.clone(), + stream_sha256: p.stream_sha256, + trailer: p.trailer.clone(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + //defer closeResponse(resp) + //if resp.is_none() { + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name))); + } + //} + let h = resp.headers(); + let mut obj_part = ObjectPart { + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, + ..Default::default() + }; + obj_part.size = p.size; + obj_part.part_num = p.part_number; + obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() }; + Ok(obj_part) + } + + pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str, + complete: CompleteMultipartUpload, opts: &PutObjectOptions + ) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("uploadId".to_string(), upload_id.to_string()); + let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec(); + + let mut headers = opts.header(); + + let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes); + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + content_body: ReaderImpl::Body(complete_multipart_upload_buffer), + content_length: 100,//complete_multipart_upload_bytes.len(), + content_sha256_hex: "".to_string(),//hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower), + custom_header: headers, + stream_sha256: Default::default(), + trailer: Default::default(), + content_md5_base64: "".to_string(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; + + let b = resp.body().bytes().expect("err").to_vec(); + let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default(); + + let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { + ( + OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), + "".to_string() + ) + } else { + (OffsetDateTime::now_utc(), "".to_string()) + }; + + let h = resp.headers(); + Ok(UploadInfo { + bucket: complete_multipart_upload_result.bucket, + key: complete_multipart_upload_result.key, + etag: trim_etag(&complete_multipart_upload_result.etag), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, + location: complete_multipart_upload_result.location, + expiration: exp_time, + expiration_rule_id: rule_id, + checksum_sha256: complete_multipart_upload_result.checksum_sha256, + checksum_sha1: complete_multipart_upload_result.checksum_sha1, + checksum_crc32: complete_multipart_upload_result.checksum_crc32, + checksum_crc32c: complete_multipart_upload_result.checksum_crc32c, + checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme, + ..Default::default() + }) + } +} + +pub struct UploadPartParams { + pub bucket_name: String, + pub object_name: String, + pub upload_id: String, + pub reader: ReaderImpl, + pub part_number: i64, + pub md5_base64: String, + pub sha256_hex: String, + pub size: i64, + //pub sse: encrypt.ServerSide, + pub stream_sha256: bool, + pub custom_header: HeaderMap, + pub trailer: HeaderMap, +} diff --git a/ecstore/src/client/api_put_object_streaming.rs b/ecstore/src/client/api_put_object_streaming.rs new file mode 100644 index 00000000..dd12f9bb --- /dev/null +++ b/ecstore/src/client/api_put_object_streaming.rs @@ -0,0 +1,447 @@ +#![allow(clippy::map_entry)] +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc}; +use bytes::Bytes; +use futures::future::join_all; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use time::{format_description, OffsetDateTime}; +use tokio::{select, sync::mpsc}; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use uuid::Uuid; + +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; +use reader::hasher::Hasher; +use crate::client::{ + constants::ISO8601_DATEFORMAT, + api_put_object::PutObjectOptions, + api_put_object_multipart::UploadPartParams, + api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, + transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl}, + api_put_object_common::{is_object, optimal_part_info,}, + api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof}, +}; +use rustfs_utils::{crypto::base64_encode, path::trim_etag}; +use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode}; + +pub struct UploadedPartRes { + pub error: std::io::Error, + pub part_num: i64, + pub size: i64, + pub part: ObjectPart, +} + +pub struct UploadPartReq { + pub part_num: i64, + pub part: ObjectPart, +} + +impl TransitionClient { + pub async fn put_object_multipart_stream(self: Arc, bucket_name: &str, object_name: &str, + mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + ) -> Result { + let info: UploadInfo; + if opts.concurrent_stream_parts && opts.num_threads > 1 { + info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?; + } else if !is_object(&reader) && !opts.send_content_md5 { + info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?; + } else { + info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?; + } + + Ok(info) + } + + pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str, + mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + ) -> Result { + let ret = optimal_part_info(size, opts.part_size)?; + let (total_parts_count, part_size, lastpart_size) = ret; + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.auto_checksum = opts.checksum.clone(); + } + let with_checksum = self.trailing_header_support; + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + todo!(); + } + + pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str, + mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions + ) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.auto_checksum = opts.checksum.clone(); + opts.send_content_md5 = false; + } + + if !opts.send_content_md5 { + add_auto_checksum_headers(&mut opts); + } + + let ret = optimal_part_info(size, opts.part_size)?; + let (total_parts_count, mut part_size, lastpart_size) = ret; + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut custom_header = opts.header().clone(); + + let mut total_uploaded_size: i64 = 0; + + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + + let mut md5_base64: String = "".to_string(); + for part_number in 1..=total_parts_count { + if part_number == total_parts_count { + part_size = lastpart_size; + } + + match &mut reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + if opts.send_content_md5 { + let mut md5_hasher = self.md5_hasher.lock().unwrap(); + let mut md5_hash = md5_hasher.as_mut().expect("err"); + md5_hash.reset(); + md5_hash.write(&buf[..length]); + md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + } else { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) { + custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err")); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized()); + } + } + + let hooked = ReaderImpl::Body(Bytes::from(buf));//newHook(BufferReader::new(buf), opts.progress); + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: hooked, + part_number, + md5_base64: md5_base64.clone(), + size: part_size, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + sha256_hex: "".to_string(), + trailer: HeaderMap::new(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.entry(part_number).or_insert(obj_part); + + total_uploaded_size += part_size as i64; + } + + if size > 0 { + if total_uploaded_size != size { + return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name))); + } + } + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + let part_number = total_parts_count; + for i in 1..part_number { + let part = parts_info[&i].clone(); + + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + apply_auto_checksum(&mut opts, &mut all_parts); + let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn put_object_multipart_stream_parallel(self: Arc, bucket_name: &str, object_name: &str, + mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions + ) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.send_content_md5 = false; + opts.auto_checksum = opts.checksum.clone(); + } + if !opts.send_content_md5 { + add_auto_checksum_headers(&mut opts); + } + + let ret = optimal_part_info(-1, opts.part_size)?; + let (total_parts_count, part_size, _) = ret; + + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut total_uploaded_size: i64 = 0; + let mut parts_info = Arc::new(RwLock::new(HashMap::::new())); + + let n_buffers = opts.num_threads; + let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize); + //let all = Vec::::with_capacity(n_buffers as usize * part_size as usize); + for i in 0..n_buffers { + //bufs_tx.send(&all[i * part_size..i * part_size + part_size]); + bufs_tx.send(Vec::::with_capacity(part_size as usize)); + } + + let mut futures = Vec::with_capacity(total_parts_count as usize); + let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize); + let cancel_token = CancellationToken::new(); + + //reader = newHook(reader, opts.progress); + + for part_number in 1..=total_parts_count { + let mut buf = Vec::::new(); + select! { + buf = bufs_rx.recv() => {} + err = err_rx.recv() => { + //cancel_token.cancel(); + //wg.Wait() + return Err(err.expect("err")); + } + else => (), + } + + if buf.len() != part_size as usize { + return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size))); + } + + match &mut reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + let mut custom_header = HeaderMap::new(); + if !opts.send_content_md5 { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) { + custom_header.insert(header_name, header_value); + } + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + let clone_bufs_tx = bufs_tx.clone(); + let clone_parts_info = parts_info.clone(); + let clone_upload_id = upload_id.clone(); + let clone_self = self.clone(); + futures.push(async move { + let mut md5_base64: String = "".to_string(); + + if opts.send_content_md5 { + let mut md5_hasher = clone_self.md5_hasher.lock().unwrap(); + let mut md5_hash = md5_hasher.as_mut().expect("err"); + md5_hash.write(&buf[..length]); + md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + } + + //defer wg.Done() + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: clone_upload_id, + reader: ReaderImpl::Body(Bytes::from(buf.clone())), + part_number, + md5_base64, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header, + sha256_hex: "".to_string(), + trailer: HeaderMap::new(), + }; + let obj_part = clone_self.upload_part(&mut p).await.expect("err"); + + let mut clone_parts_info = clone_parts_info.write().unwrap(); + clone_parts_info.entry(part_number).or_insert(obj_part); + + clone_bufs_tx.send(buf); + }); + + total_uploaded_size += length as i64; + } + + let results = join_all(futures).await; + + select! { + err = err_rx.recv() => { + return Err(err.expect("err")); + } + else => (), + } + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let mut part_number: i64 = total_parts_count; + let mut all_parts = Vec::::with_capacity(parts_info.read().unwrap().len()); + for i in 1..part_number { + let part = parts_info.read().unwrap()[&i].clone(); + + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + apply_auto_checksum(&mut opts, &mut all_parts); + + let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.send_content_md5 = false; + } + + let mut md5_base64: String = "".to_string(); + let progress_reader = reader;//newHook(reader, opts.progress); + + self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await + } + + pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result { + let custom_header = opts.header(); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + custom_header, + content_body: reader, + content_length: size, + content_md5_base64: md5_base64.to_string(), + content_sha256_hex: sha256_hex.to_string(), + stream_sha256: !opts.disable_content_sha256, + add_crc: Default::default(), + bucket_location: Default::default(), + pre_sign_url: Default::default(), + query_values: Default::default(), + extra_pre_sign_header: Default::default(), + expires: Default::default(), + trailer: Default::default(), + }; + let mut add_crc = false;//self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure); + let mut opts = opts.clone(); + if opts.checksum.is_set() { + req_metadata.add_crc = opts.checksum; + } else if add_crc { + for (k, _) in opts.user_metadata { + if k.to_lowercase().starts_with("x-amz-checksum-") { + add_crc = false; + } + } + if add_crc { + opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); + req_metadata.add_crc = opts.auto_checksum; + } + } + + if opts.internal.source_version_id != "" { + if !opts.internal.source_version_id.is_empty() { + if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) { + return Err(std::io::Error::other(err_invalid_argument(&err.to_string()))); + } + } + let mut url_values = HashMap::new(); + url_values.insert("versionId".to_string(), opts.internal.source_version_id); + req_metadata.query_values = url_values; + } + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + + let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { + ( + OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), + "".to_string() + ) + } else { + (OffsetDateTime::now_utc(), "".to_string()) + }; + let h = resp.headers(); + Ok(UploadInfo { + bucket: bucket_name.to_string(), + key: object_name.to_string(), + etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, + size: size, + expiration: exp_time, + expiration_rule_id: rule_id, + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, + ..Default::default() + }) + } +} \ No newline at end of file diff --git a/ecstore/src/client/api_remove.rs b/ecstore/src/client/api_remove.rs new file mode 100644 index 00000000..3f37ee8f --- /dev/null +++ b/ecstore/src/client/api_remove.rs @@ -0,0 +1,387 @@ +#![allow(clippy::map_entry)] +use std::fmt::Display; +use std::{collections::HashMap, sync::Arc}; +use bytes::Bytes; +use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION; +use s3s::S3ErrorCode; +use time::OffsetDateTime; +use s3s::dto::ReplicationStatus; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use http::{HeaderMap, HeaderValue, Method, StatusCode}; + +use reader::hasher::{sum_sha256_hex, sum_md5_base64}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use crate::{ + disk::DiskAPI, + store_api::{ + GetObjectReader, ObjectInfo, StorageAPI, + }, +}; +use crate::client::{ + transition_api::{TransitionClient, RequestMetadata, ReaderImpl}, + api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,}, +}; + +struct RemoveBucketOptions { + forced_elete: bool, +} + +#[derive(Debug)] +pub struct AdvancedRemoveOptions { + replication_delete_marker: bool, + replication_status: ReplicationStatus, + replication_mtime: OffsetDateTime, + replication_request: bool, + replication_validity_check: bool, +} + +impl Default for AdvancedRemoveOptions { + fn default() -> Self { + Self { + replication_delete_marker: false, + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + replication_mtime: OffsetDateTime::now_utc(), + replication_request: false, + replication_validity_check: false, + } + } +} + +#[derive(Debug, Default)] +pub struct RemoveObjectOptions { + pub force_delete: bool, + pub governance_bypass: bool, + pub version_id: String, + pub internal: AdvancedRemoveOptions, +} + +impl TransitionClient { + pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> { + let mut headers = HeaderMap::new(); + /*if opts.force_delete { + headers.insert(rustFSForceDelete, "true"); + }*/ + + let resp = self.execute_method(Method::DELETE, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.delete(bucket_name); + } + Ok(()) + } + + pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { + let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: Default::default(), + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.delete(bucket_name); + } + + Ok(()) + } + + pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option { + let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err"); + res.err + } + + pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result { + let mut url_values = HashMap::new(); + + if opts.version_id != "" { + url_values.insert("versionId".to_string(), opts.version_id.clone()); + } + + let mut headers = HeaderMap::new(); + + if opts.governance_bypass { + headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));//amzBypassGovernance + } + + let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + query_values: url_values, + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + Ok(RemoveObjectResult { + object_name: object_name.to_string(), + object_version_id: opts.version_id, + delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", + delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(), + ..Default::default() + }) + } + + pub async fn remove_objects_with_result(self: Arc, bucket_name: &str, objects_rx: Receiver, opts: RemoveObjectsOptions) -> Receiver { + let (result_tx, mut result_rx) = mpsc::channel(1); + + let self_clone = Arc::clone(&self); + let bucket_name_owned = bucket_name.to_string(); + + tokio::spawn(async move { + self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; + }); + result_rx + } + + pub async fn remove_objects(self: Arc, bucket_name: &str, objects_rx: Receiver, opts: RemoveObjectsOptions) -> Receiver { + let (error_tx, mut error_rx) = mpsc::channel(1); + + let self_clone = Arc::clone(&self); + let bucket_name_owned = bucket_name.to_string(); + + let (result_tx, mut result_rx) = mpsc::channel(1); + tokio::spawn(async move { + self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; + }); + tokio::spawn(async move { + while let Some(res) = result_rx.recv().await { + if res.err.is_none() { + continue; + } + error_tx.send(RemoveObjectError { + object_name: res.object_name, + version_id: res.object_version_id, + err: res.err, + ..Default::default() + }).await; + } + }); + + error_rx + } + + pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver, result_tx: &Sender, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> { + let max_entries = 1000; + let mut finish = false; + let mut url_values = HashMap::new(); + url_values.insert("delete".to_string(), "".to_string()); + + loop { + if finish { + break; + } + let mut count = 0; + let mut batch = Vec::::new(); + + while let Some(object) = objects_rx.recv().await { + if has_invalid_xml_char(&object.name) { + let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions { + version_id: object.version_id.expect("err").to_string(), + governance_bypass: opts.governance_bypass, + ..Default::default() + }).await?; + let remove_result_clone = remove_result.clone(); + if !remove_result.err.is_none() { + match to_error_response(&remove_result.err.expect("err")).code { + S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => { + continue; + } + _ => (), + } + result_tx.send(remove_result_clone.clone()).await; + } + + result_tx.send(remove_result_clone).await; + continue; + } + + batch.push(object); + count += 1; + if count >= max_entries { + break; + } + } + if count == 0 { + break; + } + if count < max_entries { + finish = true; + } + + let mut headers = HeaderMap::new(); + if opts.governance_bypass { + headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); + } + + let remove_bytes = generate_remove_multi_objects_request(&batch); + let resp = self.execute_method(http::Method::POST, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values.clone(), + content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), + content_length: remove_bytes.len() as i64, + content_md5_base64: sum_md5_base64(&remove_bytes), + content_sha256_hex: sum_sha256_hex(&remove_bytes), + custom_header: headers, + object_name: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + + let body_bytes: Vec = resp.body().bytes().expect("err").to_vec(); + process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone()); + } + Ok(()) + } + + pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> { + let upload_ids = self.find_upload_ids(bucket_name, object_name)?; + for upload_id in upload_ids { + self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?; + } + + Ok(()) + } + + pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("uploadId".to_string(), upload_id.to_string()); + + let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }).await?; + //if resp.is_some() { + if resp.status() != StatusCode::NO_CONTENT { + let error_response: ErrorResponse; + match resp.status() { + StatusCode::NOT_FOUND => { + error_response = ErrorResponse { + code: S3ErrorCode::NoSuchUpload, + message: "The specified multipart upload does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + request_id: resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(), + host_id: resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(), + region: resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(), + ..Default::default() + }; + } + _ => { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + } + return Err(std::io::Error::other(error_response)); + } + //} + Ok(()) + } +} + +#[derive(Debug, Default)] +struct RemoveObjectError { + object_name: String, + version_id: String, + err: Option, +} + +impl Display for RemoveObjectError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.err.is_none() { + return write!(f, "unexpected remove object error result"); + } + write!(f, "{}", self.err.as_ref().expect("err").to_string()) + } +} + +#[derive(Debug, Default)] +pub struct RemoveObjectResult { + pub object_name: String, + pub object_version_id: String, + pub delete_marker: bool, + pub delete_marker_version_id: String, + pub err: Option, +} + +impl Clone for RemoveObjectResult { + fn clone(&self) -> Self { + Self { + object_name: self.object_name.clone(), + object_version_id: self.object_version_id.clone(), + delete_marker: self.delete_marker, + delete_marker_version_id: self.delete_marker_version_id.clone(), + err: None, //err + } + } +} + +pub struct RemoveObjectsOptions { + pub governance_bypass: bool, +} + +pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec { + todo!(); +} + +pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender) { + todo!(); +} + +fn has_invalid_xml_char(str: &str) -> bool { + false +} diff --git a/ecstore/src/client/api_s3_datatypes.rs b/ecstore/src/client/api_s3_datatypes.rs new file mode 100644 index 00000000..592f3695 --- /dev/null +++ b/ecstore/src/client/api_s3_datatypes.rs @@ -0,0 +1,336 @@ +#![allow(clippy::map_entry)] +use std::collections::HashMap; +use s3s::dto::Owner; +use serde::{Serialize, Deserialize}; +use time::OffsetDateTime; + +use rustfs_utils::crypto::base64_decode; +use crate::checksum::ChecksumMode; +use crate::client::transition_api::ObjectMultipartInfo; + +use super::transition_api; + +pub struct ListAllMyBucketsResult { + pub owner: Owner, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CommonPrefix { + pub prefix: String, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListBucketV2Result { + pub common_prefixes: Vec, + pub contents: Vec, + pub delimiter: String, + pub encoding_type: String, + pub is_truncated: bool, + pub max_keys: i64, + pub name: String, + pub next_continuation_token: String, + pub continuation_token: String, + pub prefix: String, + pub fetch_owner: String, + pub start_after: String, +} + +pub struct Version { + etag: String, + is_latest: bool, + key: String, + last_modified: OffsetDateTime, + owner: Owner, + size: i64, + storage_class: String, + version_id: String, + user_metadata: HashMap, + user_tags: HashMap, + is_delete_marker: bool, +} + +pub struct ListVersionsResult { + versions: Vec, + common_prefixes: Vec, + name: String, + prefix: String, + delimiter: String, + max_keys: i64, + encoding_type: String, + is_truncated: bool, + key_marker: String, + version_id_marker: String, + next_key_marker: String, + next_version_id_marker: String, +} + +impl ListVersionsResult { + fn unmarshal_xml() -> Result<(), std::io::Error> { + todo!(); + } +} + +pub struct ListBucketResult { + common_prefixes: Vec, + contents: Vec, + delimiter: String, + encoding_type: String, + is_truncated: bool, + marker: String, + max_keys: i64, + name: String, + next_marker: String, + prefix: String, +} + +pub struct ListMultipartUploadsResult { + bucket: String, + key_marker: String, + upload_id_marker: String, + next_key_marker: String, + next_upload_id_marker: String, + encoding_type: String, + max_uploads: i64, + is_truncated: bool, + uploads: Vec, + prefix: String, + delimiter: String, + common_prefixes: Vec, +} + +pub struct Initiator { + id: String, + display_name: String, +} + +pub struct CopyObjectResult { + pub etag: String, + pub last_modified: OffsetDateTime, +} + +#[derive(Debug, Clone)] +pub struct ObjectPart { + pub etag: String, + pub part_num: i64, + pub last_modified: OffsetDateTime, + pub size: i64, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +impl Default for ObjectPart { + fn default() -> Self { + ObjectPart { + etag: Default::default(), + part_num: 0, + last_modified: OffsetDateTime::now_utc(), + size: 0, + checksum_crc32: Default::default(), + checksum_crc32c: Default::default(), + checksum_sha1: Default::default(), + checksum_sha256: Default::default(), + checksum_crc64nvme: Default::default(), + } + } + +} + +impl ObjectPart { + fn checksum(&self, t: &ChecksumMode) -> String { + match t { + ChecksumMode::ChecksumCRC32C => { + return self.checksum_crc32c.clone(); + } + ChecksumMode::ChecksumCRC32 => { + return self.checksum_crc32.clone(); + } + ChecksumMode::ChecksumSHA1 => { + return self.checksum_sha1.clone(); + } + ChecksumMode::ChecksumSHA256 => { + return self.checksum_sha256.clone(); + } + ChecksumMode::ChecksumCRC64NVME => { + return self.checksum_crc64nvme.clone(); + } + _ => { + return "".to_string(); + } + } + } + + pub fn checksum_raw(&self, t: &ChecksumMode) -> Result, std::io::Error> { + let b = self.checksum(t); + if b == "" { + return Err(std::io::Error::other("no checksum set")); + } + let decoded = match base64_decode(b.as_bytes()) { + Ok(b) => b, + Err(e) => return Err(std::io::Error::other(e)), + }; + if decoded.len() != t.raw_byte_len() as usize { + return Err(std::io::Error::other("checksum length mismatch")); + } + Ok(decoded) + } +} + +pub struct ListObjectPartsResult { + pub bucket: String, + pub key: String, + pub upload_id: String, + pub initiator: Initiator, + pub owner: Owner, + pub storage_class: String, + pub part_number_marker: i32, + pub next_part_number_marker: i32, + pub max_parts: i32, + pub checksum_algorithm: String, + pub checksum_type: String, + pub is_truncated: bool, + pub object_parts: Vec, + pub encoding_type: String, +} + +#[derive(Debug, Default)] +pub struct InitiateMultipartUploadResult { + pub bucket: String, + pub key: String, + pub upload_id: String, +} + +#[derive(Debug, Default)] +pub struct CompleteMultipartUploadResult { + pub location: String, + pub bucket: String, + pub key: String, + pub etag: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +#[derive(serde::Serialize)] +pub struct CompletePart { //api has + pub etag: String, + pub part_num: i64, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +impl CompletePart { + fn checksum(&self, t: &ChecksumMode) -> String { + match t { + ChecksumMode::ChecksumCRC32C => { + return self.checksum_crc32c.clone(); + } + ChecksumMode::ChecksumCRC32 => { + return self.checksum_crc32.clone(); + } + ChecksumMode::ChecksumSHA1 => { + return self.checksum_sha1.clone(); + } + ChecksumMode::ChecksumSHA256 => { + return self.checksum_sha256.clone(); + } + ChecksumMode::ChecksumCRC64NVME => { + return self.checksum_crc64nvme.clone(); + } + _ => { + return "".to_string(); + } + } + } +} + +pub struct CopyObjectPartResult { + pub etag: String, + pub last_modified: OffsetDateTime, +} + +#[derive(Debug, Default)] +#[derive(serde::Serialize)] +pub struct CompleteMultipartUpload { + pub parts: Vec, +} + +impl CompleteMultipartUpload { + pub fn marshal_msg(&self) -> Result { + //let buf = serde_json::to_string(self)?; + let buf = match serde_xml_rs::to_string(self) { + Ok(buf) => buf, + Err(e) => { + return Err(std::io::Error::other(e)); + } + }; + + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result { + todo!(); + } +} + +pub struct CreateBucketConfiguration { + pub location: String, +} + +#[derive(serde::Serialize)] +pub struct DeleteObject { //api has + pub key: String, + pub version_id: String, +} + +pub struct DeletedObject { //s3s has + pub key: String, + pub version_id: String, + pub deletemarker: bool, + pub deletemarker_version_id: String, +} + +pub struct NonDeletedObject { + pub key: String, + pub code: String, + pub message: String, + pub version_id: String, +} + +#[derive(serde::Serialize)] +pub struct DeleteMultiObjects { + pub quiet: bool, + pub objects: Vec, +} + +impl DeleteMultiObjects { + pub fn marshal_msg(&self) -> Result { + //let buf = serde_json::to_string(self)?; + let buf = match serde_xml_rs::to_string(self) { + Ok(buf) => buf, + Err(e) => { + return Err(std::io::Error::other(e)); + } + }; + + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result { + todo!(); + } +} + +pub struct DeleteMultiObjectsResult { + pub deleted_objects: Vec, + pub undeleted_objects: Vec, +} diff --git a/ecstore/src/client/bucket_cache.rs b/ecstore/src/client/bucket_cache.rs new file mode 100644 index 00000000..00f8c22d --- /dev/null +++ b/ecstore/src/client/bucket_cache.rs @@ -0,0 +1,224 @@ +#![allow(clippy::map_entry)] +use http::Request; +use hyper::body::Incoming; +use std::{collections::HashMap, sync::Arc}; +use tracing::warn; +use tracing::{error, info, debug}; +use hyper::StatusCode; + +use reader::hasher::{Hasher, Sha256}; +use s3s::S3ErrorCode; +use s3s::Body; +use crate::client::{ + api_error_response::{http_resp_to_error_response, to_error_response}, + transition_api::{TransitionClient, Document}, +}; +use crate::signer; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +use super::constants::UNSIGNED_PAYLOAD; +use super::credentials::SignatureType; + +pub struct BucketLocationCache { + items: HashMap, +} + +impl BucketLocationCache { + pub fn new() -> BucketLocationCache { + BucketLocationCache{ + items: HashMap::new(), + } + } + + pub fn get(&self, bucket_name: &str) -> Option { + self.items.get(bucket_name).map(|s| s.clone()) + } + + pub fn set(&mut self, bucket_name: &str, location: &str) { + self.items.insert(bucket_name.to_string(), location.to_string()); + } + + pub fn delete(&mut self, bucket_name: &str) { + self.items.remove(bucket_name); + } +} + +impl TransitionClient { + pub async fn get_bucket_location(&self, bucket_name: &str) -> Result { + Ok(self.get_bucket_location_inner(bucket_name).await?) + } + + async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result { + if self.region != "" { + return Ok(self.region.clone()) + } + + let mut location; + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + let ret = bucket_loc_cache.get(bucket_name); + if let Some(location) = ret { + return Ok(location); + } + //location = ret?; + } + + let req = self.get_bucket_location_request(bucket_name)?; + + let mut resp = self.doit(req).await?; + location = process_bucket_location_response(resp, bucket_name).await?; + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.set(bucket_name, &location); + } + Ok(location) + } + + fn get_bucket_location_request(&self, bucket_name: &str) -> Result, std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("location".to_string(), "".to_string()); + + let mut target_url = self.endpoint_url.clone(); + let scheme = self.endpoint_url.scheme(); + let h = target_url.host().expect("host is none."); + let default_port = if scheme == "https" { + 443 + } else { + 80 + }; + let p = target_url.port().unwrap_or(default_port); + + let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name); + + let mut url_str: String = "".to_string(); + + if is_virtual_style { + url_str = scheme.to_string(); + url_str.push_str("://"); + url_str.push_str(bucket_name); + url_str.push_str("."); + url_str.push_str(target_url.host_str().expect("err")); + url_str.push_str("/?location"); + } else { + let mut path = bucket_name.to_string(); + path.push_str("/"); + target_url.set_path(&path); + { + let mut q = target_url.query_pairs_mut(); + for (k, v) in url_values { + q.append_pair(&k, &urlencoding::encode(&v)); + } + } + url_str = target_url.to_string(); + } + + let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str); + + self.set_user_agent(&mut req_builder); + + let value; + { + let mut creds_provider = self.creds_provider.lock().unwrap(); + value = match creds_provider.get_with_context(Some(self.cred_context())) { + Ok(v) => v, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + let mut signer_type = value.signer_type.clone(); + let mut access_key_id = value.access_key_id; + let mut secret_access_key = value.secret_access_key; + let mut session_token = value.session_token; + + if self.override_signer_type != SignatureType::SignatureDefault { + signer_type = self.override_signer_type.clone(); + } + + if value.signer_type == SignatureType::SignatureAnonymous { + signer_type = SignatureType::SignatureAnonymous + } + + if signer_type == SignatureType::SignatureAnonymous { + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + if signer_type == SignatureType::SignatureV2 { + let req_builder = signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style); + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string(); + if self.secure { + content_sha256 = UNSIGNED_PAYLOAD.to_string(); + } + + req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); + let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } +} + +async fn process_bucket_location_response(mut resp: http::Response, bucket_name: &str) -> Result { + //if resp != nil { + if resp.status() != StatusCode::OK { + let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, ""); + match err_resp.code { + S3ErrorCode::NotImplemented => { + match err_resp.server.as_str() { + "AmazonSnowball" => { + return Ok("snowball".to_string()); + } + "cloudflare" => { + return Ok("us-east-1".to_string()); + } + _ => { + return Err(std::io::Error::other(err_resp)); + } + } + } + S3ErrorCode::AuthorizationHeaderMalformed | + //S3ErrorCode::InvalidRegion | + S3ErrorCode::AccessDenied => { + if err_resp.region == "" { + return Ok("us-east-1".to_string()); + } + return Ok(err_resp.region); + } + _ => { + return Err(std::io::Error::other(err_resp)); + } + } + } + //} + + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let Document(location_constraint) = serde_xml_rs::from_str::(&String::from_utf8(b).unwrap()).unwrap(); + + let mut location = location_constraint; + if location == "" { + location = "us-east-1".to_string(); + } + + if location == "EU" { + location = "eu-west-1".to_string(); + } + + Ok(location) +} \ No newline at end of file diff --git a/ecstore/src/client/constants.rs b/ecstore/src/client/constants.rs new file mode 100644 index 00000000..505041e1 --- /dev/null +++ b/ecstore/src/client/constants.rs @@ -0,0 +1,24 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; +use time::{macros::format_description, format_description::FormatItem}; +use lazy_static::lazy_static; + +pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5; +pub const MAX_PARTS_COUNT: i64 = 10000; +pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16; + +pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5; +pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; + +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; + +pub const TOTAL_WORKERS: i64 = 4; + +pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); + +const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts"; +const GetObjectAttributesMaxParts: i64 = 1000; +const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFs-Source-Mtime"; diff --git a/ecstore/src/client/credentials.rs b/ecstore/src/client/credentials.rs new file mode 100644 index 00000000..8ccf8e01 --- /dev/null +++ b/ecstore/src/client/credentials.rs @@ -0,0 +1,166 @@ +use std::fmt::{Display, Formatter}; + +use time::OffsetDateTime; + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub enum SignatureType { + #[default] + SignatureDefault, + SignatureV4, + SignatureV2, + SignatureV4Streaming, + SignatureAnonymous, +} + +#[derive(Debug, Clone, Default)] +pub struct Credentials { + creds: Value, + force_refresh: bool, + provider: P, +} + +impl Credentials

+{ + pub fn new(provider: P) -> Self { + Self { + provider: provider, + force_refresh: true, + ..Default::default() + } + } + + pub fn get(&mut self) -> Result { + self.get_with_context(None) + } + + pub fn get_with_context(&mut self, mut cc: Option) -> Result { + if self.is_expired() { + let creds = self.provider.retrieve_with_cred_context(cc.expect("err")); + self.creds = creds; + self.force_refresh = false; + } + + Ok(self.creds.clone()) + } + + fn expire(&mut self) { + self.force_refresh = true; + } + + pub fn is_expired(&self) -> bool { + self.force_refresh || self.provider.is_expired() + } +} + +#[derive(Debug, Clone)] +pub struct Value { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: OffsetDateTime, + pub signer_type: SignatureType, +} + +impl Default for Value { + fn default() -> Self { + Self { + access_key_id: "".to_string(), + secret_access_key: "".to_string(), + session_token: "".to_string(), + expiration: OffsetDateTime::now_utc(), + signer_type: SignatureType::SignatureDefault, + } + } +} + +pub struct CredContext { + //pub client: SendRequest, + pub endpoint: String, +} + +trait Provider { + fn retrieve(&self) -> Value; + fn retrieve_with_cred_context(&self, _: CredContext) -> Value; + fn is_expired(&self) -> bool; +} + +#[derive(Debug, Clone, Default)] +pub struct Static(pub Value); + +impl Provider for Static { + fn retrieve(&self) -> Value { + if self.0.access_key_id == "" || self.0.secret_access_key == "" { + return Value { + signer_type: SignatureType::SignatureAnonymous, + ..Default::default() + }; + } + self.0.clone() + } + + fn retrieve_with_cred_context(&self, _: CredContext) -> Value { + self.retrieve() + } + + fn is_expired(&self) -> bool { + false + } +} + +#[derive(Debug, Clone, Default)] +pub struct STSError { + pub r#type: String, + pub code: String, + pub message: String, +} + +#[derive(Debug, Clone, thiserror::Error)] +pub struct ErrorResponse { + pub sts_error: STSError, + pub request_id: String, +} + +impl Display for ErrorResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error()) + } +} + +impl ErrorResponse { + fn error(&self) -> String { + if self.sts_error.message == "" { + return format!("Error response code {}.", self.sts_error.code); + } + return self.sts_error.message.clone(); + } +} + +struct Error { + code: String, + message: String, + bucket_name: String, + key: String, + resource: String, + request_id: String, + host_id: String, + region: String, + server: String, + status_code: i64, +} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.message == "" { + return write!(f, "{}", format!("Error response code {}.", self.code)); + } + write!(f, "{}", self.message) + } +} + +pub fn xml_decoder(body: &[u8]) -> Result { + todo!(); +} + +pub fn xml_decode_and_body(body_reader: &[u8]) -> Result<(Vec, T), std::io::Error> { + todo!(); +} diff --git a/ecstore/src/client/hook_reader.rs b/ecstore/src/client/hook_reader.rs new file mode 100644 index 00000000..c8351e4d --- /dev/null +++ b/ecstore/src/client/hook_reader.rs @@ -0,0 +1,45 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; + +use crate::{ + disk::{ + error::{is_unformatted_disk, DiskError}, + format::{DistributionAlgoVersion, FormatV3}, + new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore, + }, + store_api::{ + BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, + ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, + ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + }, + credentials::{Credentials, SignatureType,}, + api_put_object_multipart::UploadPartParams, +}; + +use http::HeaderMap; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use tracing::{error, info}; +use url::Url; + +struct HookReader { + source: GetObjectReader, + hook: GetObjectReader, +} + +impl HookReader { + pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader { + HookReader { + source, + hook, + } + } + + fn seek(&self, offset: i64, whence: i64) -> Result { + todo!(); + } + + fn read(&self, b: &[u8]) -> Result { + todo!(); + } +} \ No newline at end of file diff --git a/ecstore/src/client/mod.rs b/ecstore/src/client/mod.rs new file mode 100644 index 00000000..2a060af8 --- /dev/null +++ b/ecstore/src/client/mod.rs @@ -0,0 +1,18 @@ +pub mod constants; +pub mod transition_api; +pub mod api_list; +pub mod api_error_response; +pub mod api_s3_datatypes; +pub mod api_bucket_policy; +pub mod api_put_object_common; +pub mod api_get_options; +pub mod api_get_object; +pub mod api_put_object; +pub mod api_put_object_streaming; +pub mod api_put_object_multipart; +pub mod api_remove; +pub mod object_api_utils; +pub mod object_handlers_common; +pub mod admin_handler_utils; +pub mod credentials; +pub mod bucket_cache; \ No newline at end of file diff --git a/ecstore/src/client/object_api_utils.rs b/ecstore/src/client/object_api_utils.rs new file mode 100644 index 00000000..494969aa --- /dev/null +++ b/ecstore/src/client/object_api_utils.rs @@ -0,0 +1,130 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; +use http::HeaderMap; +use tokio::io::BufReader; +use std::io::Cursor; + +use s3s::S3ErrorCode; +use crate::store_api::{ + GetObjectReader, HTTPRangeSpec, + ObjectInfo, ObjectOptions, +}; +use rustfs_filemeta::fileinfo::ObjectPartInfo; +use rustfs_rio::HashReader; +use crate::error::ErrorResponse; + +//#[derive(Clone)] +pub struct PutObjReader { + pub reader: HashReader, + pub raw_reader: HashReader, + //pub sealMD5Fn: SealMD5CurrFn, +} + +impl PutObjReader { + pub fn new(raw_reader: HashReader) -> Self { + todo!(); + } + + fn size(&self) -> usize { + //self.reader.size() + todo!(); + } + + fn md5_current_hex_string(&self) -> String { + todo!(); + } + + fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> { + self.reader = enc_reader; + + Ok(()) + } +} + +pub type ObjReaderFn = Arc>>, HeaderMap) -> GetObjectReader + 'static>; + +fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option { + if oi.size == 0 || oi.parts.len() == 0 { + return None; + } + + let mut start: i64 = 0; + let mut end: i64 = -1; + let mut i = 0; + while i < oi.parts.len() && i < part_number { + start = end + 1; + end = start + oi.parts[i].actual_size as i64 - 1; + i += 1; + } + + Some(HTTPRangeSpec {start: start as usize, end: Some(end as usize), is_suffix_length: false}) +} + +fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) { + let mut skip_length: i64 = 0; + let mut cumulative_actual_size: i64 = 0; + let mut first_part_idx: i64 = 0; + let mut compressed_offset: i64 = 0; + let mut part_skip: i64 = 0; + let mut decrypt_skip: i64 = 0; + let mut seq_num: u64 = 0; + for (i, part) in oi.parts.iter().enumerate() { + cumulative_actual_size += part.actual_size as i64; + if cumulative_actual_size <= offset { + compressed_offset += part.size as i64; + } else { + first_part_idx = i as i64; + skip_length = cumulative_actual_size - part.actual_size as i64; + break; + } + } + skip_length = offset - skip_length; + + let parts: &[ObjectPartInfo] = &oi.parts; + if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 { + todo!(); + } + + (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) +} + +pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { + //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?; + let mut is_encrypted = false; + let is_compressed = false;//oi.is_compressed_ok(); + + let mut get_fn: ObjReaderFn; + + let (off, length) = match rs.get_offset_length(oi.size) { + Ok(x) => x, + Err(err) => return Err(ErrorResponse { + code: S3ErrorCode::InvalidRange, + message: err.to_string(), + key: None, + bucket_name: None, + region: None, + request_id: None, + host_id: "".to_string(), + }), + }; + get_fn = Arc::new(move |input_reader: BufReader>>, _: HeaderMap| { + //Box::pin({ + /*let r = GetObjectReader { + object_info: oi.clone(), + stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)), + }; + r*/ + todo!(); + //}) + }); + + Ok((get_fn, off as i64, length as i64)) +} + +pub fn extract_etag(metadata: &HashMap) -> String { + if let Some(etag) = metadata.get("etag") { + etag.clone() + } else { + metadata["md5Sum"].clone() + } +} \ No newline at end of file diff --git a/ecstore/src/client/object_handlers_common.rs b/ecstore/src/client/object_handlers_common.rs new file mode 100644 index 00000000..0e0e7a7d --- /dev/null +++ b/ecstore/src/client/object_handlers_common.rs @@ -0,0 +1,27 @@ +use lock::local_locker::MAX_DELETE_LIST; +use crate::bucket::versioning::VersioningApi; +use crate::bucket::versioning_sys::BucketVersioningSys; +use crate::store::ECStore; +use crate::store_api::{ObjectOptions, ObjectToDelete,}; +use crate::StorageAPI; +use crate::bucket::lifecycle::lifecycle; + +pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) { + let mut remaining = to_del; + loop { + if remaining.len() <= 0 {break}; + let mut to_del = remaining; + if to_del.len() > MAX_DELETE_LIST { + remaining = &to_del[MAX_DELETE_LIST..]; + to_del = &to_del[..MAX_DELETE_LIST]; + } else { + remaining = &[]; + } + let vc = BucketVersioningSys::get(bucket).await.expect("err!"); + let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions { + //prefix_enabled_fn: vc.prefix_enabled(""), + version_suspended: vc.suspended(), + ..Default::default() + }); + } +} diff --git a/ecstore/src/client/transition_api.rs b/ecstore/src/client/transition_api.rs new file mode 100644 index 00000000..e48a9627 --- /dev/null +++ b/ecstore/src/client/transition_api.rs @@ -0,0 +1,888 @@ +#![allow(clippy::map_entry)] +use std::pin::Pin; +use bytes::Bytes; +use futures::Future; +use http::{HeaderMap, HeaderName}; +use serde::{Serialize, Deserialize}; +use uuid::Uuid; +use rand::Rng; +use std::{collections::HashMap, sync::{Arc, Mutex}}; +use std::sync::atomic::{AtomicI32, Ordering}; +use std::task::{Context, Poll}; +use time::Duration; +use time::OffsetDateTime; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; +use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector}; +use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response}; +use tracing::{error, debug}; +use url::{form_urlencoded, Url}; +use tokio::io::BufReader; +use std::io::Cursor; + +use s3s::S3ErrorCode; +use s3s::{dto::Owner, Body}; +use s3s::dto::ReplicationStatus; +use crate::client::bucket_cache::BucketLocationCache; +use reader::hasher::{Sha256, MD5,}; +use rustfs_rio::HashReader; +use rustfs_utils::{ + net::get_endpoint_url, + retry::{new_retry_timer, MAX_RETRY}, +}; +use crate::{ + store_api::GetObjectReader, + checksum::ChecksumMode, +}; +use crate::signer; +use crate::client::{ + constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER}, + credentials::{Credentials, SignatureType, CredContext, Static,}, + api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument}, + api_put_object_multipart::UploadPartParams, + api_put_object::PutObjectOptions, + api_get_options::GetObjectOptions, + api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart}, +}; + +const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)"; +const C_USER_AGENT: &str = "RustFS (linux; x86)"; + +const SUCCESS_STATUS: [StatusCode; 3] = [ + StatusCode::OK, + StatusCode::NO_CONTENT, + StatusCode::PARTIAL_CONTENT, +]; + +const C_UNKNOWN: i32 = -1; +const C_OFFLINE: i32 = 0; +const C_ONLINE: i32 = 1; + +//pub type ReaderImpl = Box; +pub enum ReaderImpl { + Body(Bytes), + ObjectBody(GetObjectReader), +} + +pub type ReadCloser = BufReader>>; + +pub struct TransitionClient { + pub endpoint_url: Url, + pub creds_provider: Arc>>, + pub override_signer_type: SignatureType, + /*app_info: TODO*/ + pub secure: bool, + pub http_client: Client, Body>, + //pub http_trace: Httptrace.ClientTrace, + pub bucket_loc_cache: Arc>, + pub is_trace_enabled: Arc>, + pub trace_errors_only: Arc>, + //pub trace_output: io.Writer, + pub s3_accelerate_endpoint: Arc>, + pub s3_dual_stack_enabled: Arc>, + pub region: String, + pub random: u64, + pub lookup: BucketLookupType, + //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType, + pub md5_hasher: Arc>>, + pub sha256_hasher: Option, + pub health_status: AtomicI32, + pub trailing_header_support: bool, + pub max_retries: i64, +} + +#[derive(Debug, Default)] +pub struct Options { + pub creds: Credentials, + pub secure: bool, + //pub transport: http.RoundTripper, + //pub trace: *httptrace.ClientTrace, + pub region: String, + pub bucket_lookup: BucketLookupType, + //pub custom_region_via_url: func(u url.URL) string, + //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType, + pub trailing_headers: bool, + pub custom_md5: Option, + pub custom_sha256: Option, + pub max_retries: i64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +enum BucketLookupType { + #[default] + BucketLookupAuto, + BucketLookupDNS, + BucketLookupPath, +} + +impl TransitionClient { + pub async fn new(endpoint: &str, opts: Options) -> Result { + let clnt = Self::private_new(endpoint, opts).await?; + + Ok(clnt) + } + + async fn private_new(endpoint: &str, opts: Options) -> Result { + let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; + + //let jar = cookiejar.New(cookiejar.Options{PublicSuffixList: publicsuffix.List})?; + + //#[cfg(feature = "ring")] + //let _ = rustls::crypto::ring::default_provider().install_default(); + //#[cfg(feature = "aws-lc-rs")] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let scheme = endpoint_url.scheme(); + let client; + //if scheme == "https" { + // client = Client::builder(TokioExecutor::new()).build_http(); + //} else { + let tls = rustls::ClientConfig::builder() + .with_native_roots()? + .with_no_client_auth(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(tls) + .https_or_http() + .enable_http1() + .build(); + client = Client::builder(TokioExecutor::new()).build(https); + //} + + let mut clnt = TransitionClient { + endpoint_url, + creds_provider: Arc::new(Mutex::new(opts.creds)), + override_signer_type: SignatureType::SignatureDefault, + secure: opts.secure, + http_client: client, + bucket_loc_cache: Arc::new(Mutex::new(BucketLocationCache::new())), + is_trace_enabled: Arc::new(Mutex::new(false)), + trace_errors_only: Arc::new(Mutex::new(false)), + s3_accelerate_endpoint: Arc::new(Mutex::new("".to_string())), + s3_dual_stack_enabled: Arc::new(Mutex::new(false)), + region: opts.region, + random: rand::rng().random_range(10..=50), + lookup: opts.bucket_lookup, + md5_hasher: Arc::new(Mutex::new(opts.custom_md5)), + sha256_hasher: opts.custom_sha256, + health_status: AtomicI32::new(C_UNKNOWN), + trailing_header_support: opts.trailing_headers, + max_retries: opts.max_retries, + }; + + { + let mut md5_hasher = clnt.md5_hasher.lock().unwrap(); + if md5_hasher.is_none() { + *md5_hasher = Some(MD5::new()); + } + } + if clnt.sha256_hasher.is_none() { + clnt.sha256_hasher = Some(Sha256::new()); + } + + clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4; + + if opts.max_retries > 0 { + clnt.max_retries = opts.max_retries; + } + + Ok(clnt) + } + + fn endpoint_url(&self) -> Url { + self.endpoint_url.clone() + } + + fn set_appinfo(&self, app_name: &str, app_version: &str) { + /*if app_name != "" && app_version != "" { + self.appInfo.app_name = app_name + self.appInfo.app_version = app_version + }*/ + } + + fn trace_errors_only_off(&self) { + let mut trace_errors_only = self.trace_errors_only.lock().unwrap(); + *trace_errors_only = false; + } + + fn trace_off(&self) { + let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap(); + *is_trace_enabled = false; + let mut trace_errors_only = self.trace_errors_only.lock().unwrap(); + *trace_errors_only = false; + } + + fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) { + todo!(); + } + + fn set_s3_enable_dual_stack(&self, enabled: bool) { + todo!(); + } + + pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap, HashMap>) { + todo!(); + } + + fn is_online(&self) -> bool { + !self.is_offline() + } + + fn mark_offline(&self) { + self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); + } + + fn is_offline(&self) -> bool { + self.health_status.load(Ordering::SeqCst) == C_OFFLINE + } + + fn health_check(hc_duration: Duration) { + todo!(); + } + + fn dump_http(&self, req: &http::Request, resp: &http::Response) -> Result<(), std::io::Error> { + let mut resp_trace: Vec; + + //info!("{}{}", self.trace_output, "---------END-HTTP---------"); + + Ok(()) + } + + pub async fn doit(&self, req: http::Request) -> Result, std::io::Error> { + let req_method; + let req_uri; + let req_headers; + let resp; + let http_client = self.http_client.clone(); + { + //let mut http_client = http_client.lock().unwrap(); + req_method = req.method().clone(); + req_uri = req.uri().clone(); + req_headers = req.headers().clone(); + + debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); + resp = http_client.request(req); + } + let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from)); + debug!("http_client url: {} {}", req_method, req_uri); + debug!("http_client headers: {:?}", req_headers); + if let Err(err) = resp { + error!("http_client call error: {:?}", err); + return Err(std::io::Error::other(err)); + } + + let mut resp = resp.unwrap(); + debug!("http_resp: {:?}", resp); + + //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) { + if resp.status() != StatusCode::OK { + //self.dump_http(&cloned_req, &resp)?; + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + debug!("err_body: {}", String::from_utf8(b).unwrap()); + } + + Ok(resp) + } + + pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result, std::io::Error> { + if self.is_offline() { + let mut s = self.endpoint_url.to_string(); + s.push_str(" is offline."); + return Err(std::io::Error::other(s)); + } + + let mut retryable: bool; + //let mut body_seeker: BufferReader; + let mut req_retry = self.max_retries; + let mut resp: http::Response; + + //if metadata.content_body != nil { + //body_seeker = BufferReader::new(metadata.content_body.read_all().await?); + retryable = true; + if !retryable { + req_retry = 1; + } + //} + + //let mut retry_timer = RetryTimer::new(); + //while let Some(v) = retry_timer.next().await { + for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ { + let req = self.new_request(method, metadata).await?; + + resp = self.doit(req).await?; + + for http_status in SUCCESS_STATUS { + if http_status == resp.status() { + return Ok(resp); + } + } + + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let err_response = http_resp_to_error_response(resp, b.clone(), &metadata.bucket_name, &metadata.object_name); + + if self.region == "" { + match err_response.code { + S3ErrorCode::AuthorizationHeaderMalformed | S3ErrorCode::InvalidArgument /*S3ErrorCode::InvalidRegion*/ => { + //break; + return Err(std::io::Error::other(err_response)); + } + S3ErrorCode::AccessDenied => { + if err_response.region == "" { + return Err(std::io::Error::other(err_response)); + } + if metadata.bucket_name != "" { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + let location = bucket_loc_cache.get(&metadata.bucket_name); + if location.is_some() && location.unwrap() != err_response.region { + bucket_loc_cache.set(&metadata.bucket_name, &err_response.region); + //continue; + } + } else { + if err_response.region != metadata.bucket_location { + metadata.bucket_location = err_response.region.clone(); + //continue; + } + } + return Err(std::io::Error::other(err_response)); + } + _ => { + return Err(std::io::Error::other(err_response)); + } + } + } + + break; + } + + Err(std::io::Error::other("resp err")) + } + + async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result, std::io::Error> { + let location = metadata.bucket_location.clone(); + if location == "" { + if metadata.bucket_name != "" { + let location = self.get_bucket_location(&metadata.bucket_name).await?; + } + } + + let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0; + let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket; + + let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location, + is_virtual_host, &metadata.query_values)?; + + let mut req_builder = Request::builder().method(method).uri(target_url.to_string()); + + let value; + { + let mut creds_provider = self.creds_provider.lock().unwrap(); + value = creds_provider.get_with_context(Some(self.cred_context()))?; + } + + let mut signer_type = value.signer_type.clone(); + let access_key_id = value.access_key_id; + let secret_access_key = value.secret_access_key; + let session_token = value.session_token; + + if self.override_signer_type != SignatureType::SignatureDefault { + signer_type = self.override_signer_type.clone(); + } + + if value.signer_type == SignatureType::SignatureAnonymous { + signer_type = SignatureType::SignatureAnonymous; + } + + if metadata.expires != 0 && metadata.pre_sign_url { + if signer_type == SignatureType::SignatureAnonymous { + return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials."))); + } + if metadata.extra_pre_sign_header.is_some() { + if signer_type == SignatureType::SignatureV2 { + return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported."))); + } + for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() { + req_builder = req_builder.header(k, v); + } + } + if signer_type == SignatureType::SignatureV2 { + req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); + } else if signer_type == SignatureType::SignatureV4 { + req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc()); + } + let req = match req_builder.body(Body::empty()) { + Ok(req) => req, + Err(err) => { return Err(std::io::Error::other(err)); } + }; + return Ok(req); + } + + self.set_user_agent(&mut req_builder); + + for (k, v) in metadata.custom_header.clone() { + req_builder.headers_mut().expect("err").insert(k.expect("err"), v); + } + + //req.content_length = metadata.content_length; + if metadata.content_length <= -1 { + let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); + req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value); + } + + if metadata.content_md5_base64.len() > 0 { + let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err"); + req_builder.headers_mut().expect("err").insert("Content-Md5", md5_value); + } + + if signer_type == SignatureType::SignatureAnonymous { + let req = match req_builder.body(Body::empty()) { + Ok(req) => req, + Err(err) => { return Err(std::io::Error::other(err)); } + }; + return Ok(req); + } + + if signer_type == SignatureType::SignatureV2 { + req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); + } + else if metadata.stream_sha256 && !self.secure { + if metadata.trailer.len() > 0 { + //req.Trailer = metadata.trailer; + for (_, v) in &metadata.trailer { + req_builder = req_builder.header(http::header::TRAILER, v.clone()); + } + } + //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id, + // &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher()); + } + else { + let mut sha_header = UNSIGNED_PAYLOAD.to_string(); + if metadata.content_sha256_hex != "" { + sha_header = metadata.content_sha256_hex.clone(); + if metadata.trailer.len() > 0 { + return Err(std::io::Error::other("internal error: content_sha256_hex with trailer not supported")); + } + } else if metadata.trailer.len() > 0 { + sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); + } + req_builder = req_builder.header::("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); + + req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone()); + } + + let req; + if metadata.content_length == 0 { + req = req_builder.body(Body::empty()); + } else { + match &mut metadata.content_body { + ReaderImpl::Body(content_body) => { + req = req_builder.body(Body::from(content_body.clone())); + } + ReaderImpl::ObjectBody(content_body) => { + req = req_builder.body(Body::from(content_body.read_all().await?)); + } + } + //req = req_builder.body(s3s::Body::from(metadata.content_body.read_all().await?)); + } + + match req { + Ok(req) => Ok(req), + Err(err) => { + Err(std::io::Error::other(err)) + } + } + } + + pub fn set_user_agent(&self, req: &mut Builder) { + let mut headers = req.headers_mut().expect("err"); + headers.insert("User-Agent", C_USER_AGENT.parse().expect("err")); + /*if self.app_info.app_name != "" && self.app_info.app_version != "" { + headers.insert("User-Agent", C_USER_AGENT+" "+self.app_info.app_name+"/"+self.app_info.app_version); + }*/ + } + + fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap) -> Result { + let scheme = self.endpoint_url.scheme(); + let host = self.endpoint_url.host().unwrap(); + let default_port = if scheme == "https" { + 443 + } else { + 80 + }; + let port = self.endpoint_url.port().unwrap_or(default_port); + + let mut url_str = format!("{scheme}://{host}:{port}/"); + + if bucket_name != "" { + if is_virtual_host_style { + url_str = format!("{scheme}://{bucket_name}.{host}:{port}/"); + if object_name != "" { + url_str.push_str(object_name); + } + } else { + url_str.push_str(bucket_name); + url_str.push_str("/"); + if object_name != "" { + url_str.push_str(object_name); + } + } + } + + if query_values.len() > 0 { + let mut encoded = form_urlencoded::Serializer::new(String::new()); + for (k, v) in query_values { + encoded.append_pair(&k, &v); + } + url_str.push_str("?"); + url_str.push_str(&encoded.finish()); + } + + Url::parse(&url_str).map_err(|e| std::io::Error::other(e.to_string())) + } + + pub fn is_virtual_host_style_request(&self, url: &Url, bucket_name: &str) -> bool { + if bucket_name == "" { + return false; + } + + if self.lookup == BucketLookupType::BucketLookupDNS { + return true; + } + + if self.lookup == BucketLookupType::BucketLookupPath { + return false; + } + + false + } + + pub fn cred_context(&self) -> CredContext { + CredContext { + //client: http_client, + endpoint: self.endpoint_url.to_string(), + } + } +} + +struct LockedRandSource { + src: u64,//rand.Source, +} + +impl LockedRandSource { + fn int63(&self) -> i64 { + /*let n = self.src.int63(); + n*/ + todo!(); + } + + fn seed(&self, seed: i64) { + //self.src.seed(seed); + todo!(); + } +} + +pub struct RequestMetadata { + pub pre_sign_url: bool, + pub bucket_name: String, + pub object_name: String, + pub query_values: HashMap, + pub custom_header: HeaderMap, + pub extra_pre_sign_header: Option, + pub expires: i64, + pub bucket_location: String, + pub content_body: ReaderImpl, + pub content_length: i64, + pub content_md5_base64: String, + pub content_sha256_hex: String, + pub stream_sha256: bool, + pub add_crc: ChecksumMode, + pub trailer: HeaderMap, +} + +pub struct TransitionCore(pub Arc); + +impl TransitionCore { + pub async fn new(endpoint: &str, opts: Options) -> Result { + let client = TransitionClient::new(endpoint, opts).await?; + Ok(Self(Arc::new(client))) + } + + pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result { + let client = self.0.clone(); + client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new()) + } + + pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result { + let client = self.0.clone(); + client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await + } + + /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result { + self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts) + }*/ + + pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str, + part_id: i32, start_offset: i32, length: i64, metadata: HashMap, + ) -> Result { + //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id, + // part_id, start_offset, length, metadata) + todo!(); + } + + pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result { + let hook_reader = data;//newHook(data, opts.progress); + let client = self.0.clone(); + client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await + } + + pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result { + let client = self.0.clone(); + let result = client.initiate_multipart_upload(bucket, object, &opts).await?; + Ok(result.upload_id) + } + + pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result { + let client = self.0.clone(); + client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads) + } + + pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64, + data: ReaderImpl, size: i64, opts: PutObjectPartOptions + ) -> Result { + let mut p = UploadPartParams { + bucket_name: bucket.to_string(), + object_name: object.to_string(), + upload_id: upload_id.to_string(), + reader: data, + part_number: part_id, + md5_base64: opts.md5_base64, + sha256_hex: opts.sha256_hex, + size: size, + //sse: opts.sse, + stream_sha256: !opts.disable_content_sha256, + custom_header: opts.custom_header, + trailer: opts.trailer, + }; + let client = self.0.clone(); + client.upload_part(&mut p).await + } + + pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result { + let client = self.0.clone(); + client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await + } + + pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result { + let client = self.0.clone(); + let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { + parts: parts.to_vec(), + }, &opts).await?; + Ok(res) + } + + pub async fn abort_multipart_upload(&self, bucket_name: &str, object: &str, upload_id: &str) -> Result<(), std::io::Error> { + let client = self.0.clone(); + client.abort_multipart_upload(bucket_name, object, upload_id).await + } + + pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result { + let client = self.0.clone(); + client.get_bucket_policy(bucket_name).await + } + + pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> { + let client = self.0.clone(); + client.put_bucket_policy(bucket_name, bucket_policy).await + } + + pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let client = self.0.clone(); + client.get_object_inner(bucket_name, object_name, opts).await + } +} + +pub struct PutObjectPartOptions { + pub md5_base64: String, + pub sha256_hex: String, + //pub sse: encrypt.ServerSide, + pub custom_header: HeaderMap, + pub trailer: HeaderMap, + pub disable_content_sha256: bool, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ObjectInfo { + pub etag: String, + pub name: String, + pub mod_time: OffsetDateTime, + pub size: usize, + pub content_type: Option, + #[serde(skip)] + pub metadata: HeaderMap, + pub user_metadata: HashMap, + pub user_tags: String, + pub user_tag_count: i64, + #[serde(skip)] + pub owner: Owner, + //pub grant: Vec, + pub storage_class: String, + pub is_latest: bool, + pub is_delete_marker: bool, + pub version_id: Uuid, + + #[serde(skip, default = "replication_status_default")] + pub replication_status: ReplicationStatus, + pub replication_ready: bool, + pub expiration: OffsetDateTime, + pub expiration_rule_id: String, + pub num_versions: usize, + + pub restore: RestoreInfo, + + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, + pub checksum_mode: String, +} + +fn replication_status_default() -> ReplicationStatus { + ReplicationStatus::from_static(ReplicationStatus::PENDING) +} + +impl Default for ObjectInfo { + fn default() -> Self { + Self { + etag: "".to_string(), + name: "".to_string(), + mod_time: OffsetDateTime::now_utc(), + size: 0, + content_type: None, + metadata: HeaderMap::new(), + user_metadata: HashMap::new(), + user_tags: "".to_string(), + user_tag_count: 0, + owner: Owner::default(), + storage_class: "".to_string(), + is_latest: false, + is_delete_marker: false, + version_id: Uuid::nil(), + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + replication_ready: false, + expiration: OffsetDateTime::now_utc(), + expiration_rule_id: "".to_string(), + num_versions: 0, + restore: RestoreInfo::default(), + checksum_crc32: "".to_string(), + checksum_crc32c: "".to_string(), + checksum_sha1: "".to_string(), + checksum_sha256: "".to_string(), + checksum_crc64nvme: "".to_string(), + checksum_mode: "".to_string(), + } + } +} + +#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone)] +pub struct RestoreInfo { + ongoing_restore: bool, + expiry_time: OffsetDateTime, +} + +impl Default for RestoreInfo { + fn default() -> Self { + Self { + ongoing_restore: false, + expiry_time: OffsetDateTime::now_utc(), + } + } +} + +pub struct ObjectMultipartInfo { + pub initiated: OffsetDateTime, + //pub initiator: initiator, + //pub owner: owner, + pub storage_class: String, + pub key: String, + pub size: i64, + pub upload_id: String, + //pub err error, +} + +pub struct UploadInfo { + pub bucket: String, + pub key: String, + pub etag: String, + pub size: i64, + pub last_modified: OffsetDateTime, + pub location: String, + pub version_id: String, + pub expiration: OffsetDateTime, + pub expiration_rule_id: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, + pub checksum_mode: String, +} + +impl Default for UploadInfo { + fn default() -> Self { + Self { + bucket: "".to_string(), + key: "".to_string(), + etag: "".to_string(), + size: 0, + last_modified: OffsetDateTime::now_utc(), + location: "".to_string(), + version_id: "".to_string(), + expiration: OffsetDateTime::now_utc(), + expiration_rule_id: "".to_string(), + checksum_crc32: "".to_string(), + checksum_crc32c: "".to_string(), + checksum_sha1: "".to_string(), + checksum_sha256: "".to_string(), + checksum_crc64nvme: "".to_string(), + checksum_mode: "".to_string(), + } + } +} + +pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Result { + todo!() +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +//#[derive(Clone)] +pub struct SendRequest { + inner: hyper::client::conn::http1::SendRequest, +} + +impl From> for SendRequest { + fn from(inner: hyper::client::conn::http1::SendRequest) -> Self { + Self { inner } + } +} + +impl tower::Service> for SendRequest { + type Response = Response; + type Error = std::io::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(std::io::Error::other) + } + + fn call(&mut self, req: Request) -> Self::Future { + //let req = hyper::Request::builder().uri("/").body(http_body_util::Empty::::new()).unwrap(); + //let req = hyper::Request::builder().uri("/").body(Body::empty()).unwrap(); + + let fut = self.inner.send_request(req); + + Box::pin(async move { fut.await.map_err(std::io::Error::other).map(|res| res.map(Body::from)) }) + } +} + +#[derive(Serialize, Deserialize)] +pub struct Document(pub String); \ No newline at end of file diff --git a/ecstore/src/cmd/bucket_replication.rs b/ecstore/src/cmd/bucket_replication.rs index 65b89a7d..42c46a00 100644 --- a/ecstore/src/cmd/bucket_replication.rs +++ b/ecstore/src/cmd/bucket_replication.rs @@ -56,6 +56,7 @@ use tracing::{debug, error, info, warn}; use uuid::Uuid; use xxhash_rust::xxh3::xxh3_64; // use bucket_targets::{self, GLOBAL_Bucket_Target_Sys}; +use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject; #[derive(Serialize, Deserialize, Debug)] struct MRFReplicateEntry { @@ -2026,6 +2027,7 @@ impl ReplicateObjectInfo { data_blocks: 0, version_id: Uuid::try_parse(&self.version_id).ok(), delete_marker: self.delete_marker, + transitioned_object: TransitionedObject::default(), user_tags: self.user_tags.clone(), parts: Vec::new(), is_latest: true, diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 8d83006c..3c8cae26 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -2125,7 +2125,7 @@ impl DiskAPI for LocalDisk { // Check if the current bucket has a configured lifecycle policy if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await { if lc_has_active_rules(&lc, "") { - cache.info.life_cycle = Some(lc); + cache.info.lifecycle = Some(lc); } } @@ -2234,9 +2234,9 @@ impl DiskAPI for LocalDisk { } } - for frer_version in fivs.free_versions.iter() { + for free_version in fivs.free_versions.iter() { let _obj_info = ObjectInfo::from_file_info( - frer_version, + free_version, &item.bucket, &item.object_path().to_string_lossy(), versioned, diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index 4fb5c7ee..d1979ff9 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -1,3 +1,5 @@ +use s3s::{S3ErrorCode, S3Error}; + use rustfs_utils::path::decode_dir_object; use crate::disk::error::DiskError; @@ -724,6 +726,248 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error { } } +pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool { + err.contains("Connection closed by foreign host") || + err.contains("TLS handshake timeout") || + err.contains("i/o timeout") || + err.contains("connection timed out") || + err.contains("connection reset by peer") || + err.contains("broken pipe") || + err.to_lowercase().contains("503 service unavailable") || + err.contains("use of closed network connection") || + err.contains("An existing connection was forcibly closed by the remote host") || + err.contains("client error (Connect)") +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct GenericError { + pub bucket: String, + pub object: String, + pub version_id: String, + //pub err: Error, +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ObjectApiError { + #[error("Operation timed out")] + OperationTimedOut, + + #[error("etag of the object has changed")] + InvalidETag, + + #[error("BackendDown")] + BackendDown(String), + + #[error("Unsupported headers in Metadata")] + UnsupportedMetadata, + + #[error("Method not allowed: {}/{}", .0.bucket, .0.object)] + MethodNotAllowed(GenericError), + + #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)] + InvalidObjectState(GenericError), +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[error("{}", .message)] +pub struct ErrorResponse { + pub code: S3ErrorCode, + pub message: String, + pub key: Option, + pub bucket_name: Option, + pub region: Option, + pub request_id: Option, + pub host_id: String, +} + +pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error { + let mut bucket = ""; + let mut object = ""; + let mut version_id = ""; + if params.len() >= 1 { + bucket = params[0]; + } + if params.len() >= 2 { + object = params[1]; + } + if params.len() >= 3 { + version_id = params[2]; + } + + if is_network_or_host_down(&err.to_string(), false) { + return std::io::Error::other(ObjectApiError::BackendDown(format!("{}", err))); + } + + let mut err_ = std::io::Error::other(err.to_string()); + let r_err = err; + let mut err = err_; + let bucket = bucket.to_string(); + let object = object.to_string(); + let version_id = version_id.to_string(); + + match r_err.code { + /*S3ErrorCode::SlowDownWrite => { + err = StorageError::InsufficientWriteQuorum;//{bucket: bucket, object: object}; + }*/ + /*S3ErrorCode::SlowDown/* SlowDownRead */ => { + err = std::io::Error::other(StorageError::InsufficientReadQuorum.to_string()); + }*/ + /*S3ErrorCode::PreconditionFailed => { + err = Error::from(StorageError::PreConditionFailed); + }*/ + /*S3ErrorCode::InvalidRange => { + err = Error::from(StorageError::InvalidRange); + }*/ + /*S3ErrorCode::BucketAlreadyOwnedByYou => { + err = Error::from(StorageError::BucketAlreadyOwnedByYou); + }*/ + S3ErrorCode::BucketNotEmpty => { + err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string()); + } + /*S3ErrorCode::NoSuchBucketPolicy => { + err = Error::from(StorageError::BucketPolicyNotFound); + }*/ + /*S3ErrorCode::NoSuchLifecycleConfiguration => { + err = Error::from(StorageError::BucketLifecycleNotFound); + }*/ + S3ErrorCode::InvalidBucketName => { + err = std::io::Error::other(StorageError::BucketNameInvalid(bucket)); + } + S3ErrorCode::InvalidPart => { + err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object/* , version_id */)); + } + S3ErrorCode::NoSuchBucket => { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + S3ErrorCode::NoSuchKey => { + if object != "" { + err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); + } else { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + } + S3ErrorCode::NoSuchVersion => { + if object != "" { + err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));//, version_id); + } else { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + } + /*S3ErrorCode::XRustFsInvalidObjectName => { + err = Error::from(StorageError::ObjectNameInvalid(bucket, object)); + }*/ + S3ErrorCode::AccessDenied => { + err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object)); + } + /*S3ErrorCode::XAmzContentSHA256Mismatch => { + err = hash.SHA256Mismatch{}; + }*/ + S3ErrorCode::NoSuchUpload => { + err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id)); + } + /*S3ErrorCode::EntityTooSmall => { + err = std::io::Error::other(StorageError::PartTooSmall); + }*/ + /*S3ErrorCode::ReplicationPermissionCheck => { + err = std::io::Error::other(StorageError::ReplicationPermissionCheck); + }*/ + _ => { + err = std::io::Error::other("err"); + } + } + + err +} + +pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { + let storage_err = &err; + let mut bucket: String = "".to_string(); + let mut object: String = "".to_string(); + if params.len() >= 1 { + bucket = params[0].to_string(); + } + if params.len() >= 2 { + object = decode_dir_object(params[1]); + } + return match storage_err { + /*StorageError::NotImplemented => s3_error!(NotImplemented), + StorageError::InvalidArgument(bucket, object, version_id) => { + s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) + }*/ + StorageError::MethodNotAllowed => { + S3Error::with_message(S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError {bucket: bucket, object: object, ..Default::default()}).to_string()) + } + /*StorageError::BucketNotFound(bucket) => { + s3_error!(NoSuchBucket, "bucket not found {}", bucket) + } + StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket), + StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket), + StorageError::ObjectNameInvalid(bucket, object) => { + s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object) + } + StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket), + StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."), + StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"), + StorageError::PrefixAccessDenied(bucket, object) => { + s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object) + } + StorageError::InvalidUploadIDKeyCombination(bucket, object) => { + s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object) + } + StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket), + StorageError::ObjectNameTooLong(bucket, object) => { + s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object) + } + StorageError::ObjectNamePrefixAsSlash(bucket, object) => { + s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object) + } + StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object), + StorageError::VersionNotFound(bucket, object, version_id) => { + s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id) + } + StorageError::InvalidUploadID(bucket, object, version_id) => { + s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id) + } + StorageError::InvalidVersionID(bucket, object, version_id) => { + s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id) + } + // extended + StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!( + InvalidArgument, + "invalid data movement operation, source and destination pool are the same for : {}/{}-{}", + bucket, + object, + version_id + ), + // extended + StorageError::ObjectExistsAsDirectory(bucket, object) => { + s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object) + } + StorageError::InsufficientReadQuorum => { + s3_error!(SlowDown, "Storage resources are insufficient for the read operation") + } + StorageError::InsufficientWriteQuorum => { + s3_error!(SlowDown, "Storage resources are insufficient for the write operation") + } + StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"), + + StorageError::VolumeNotFound(bucket) => { + s3_error!(NoSuchBucket, "bucket not found {}", bucket) + } + StorageError::InvalidPart(bucket, object, version_id) => { + s3_error!( + InvalidPart, + "Specified part could not be found. PartNumber {}, Expected {}, got {}", + bucket, + object, + version_id + ) + } + StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/ + _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()), + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/ecstore/src/event/mod.rs b/ecstore/src/event/mod.rs new file mode 100644 index 00000000..f3cb1623 --- /dev/null +++ b/ecstore/src/event/mod.rs @@ -0,0 +1,3 @@ +pub mod name; +pub mod targetid; +pub mod targetlist; \ No newline at end of file diff --git a/ecstore/src/event/name.rs b/ecstore/src/event/name.rs new file mode 100644 index 00000000..33fee80f --- /dev/null +++ b/ecstore/src/event/name.rs @@ -0,0 +1,225 @@ +#[derive(Default)] +pub enum EventName { + ObjectAccessedGet, + ObjectAccessedGetRetention, + ObjectAccessedGetLegalHold, + ObjectAccessedHead, + ObjectAccessedAttributes, + ObjectCreatedCompleteMultipartUpload, + ObjectCreatedCopy, + ObjectCreatedPost, + ObjectCreatedPut, + ObjectCreatedPutRetention, + ObjectCreatedPutLegalHold, + ObjectCreatedPutTagging, + ObjectCreatedDeleteTagging, + ObjectRemovedDelete, + ObjectRemovedDeleteMarkerCreated, + ObjectRemovedDeleteAllVersions, + ObjectRemovedNoOP, + BucketCreated, + BucketRemoved, + ObjectReplicationFailed, + ObjectReplicationComplete, + ObjectReplicationMissedThreshold, + ObjectReplicationReplicatedAfterThreshold, + ObjectReplicationNotTracked, + ObjectRestorePost, + ObjectRestoreCompleted, + ObjectTransitionFailed, + ObjectTransitionComplete, + ObjectManyVersions, + ObjectLargeVersions, + PrefixManyFolders, + ILMDelMarkerExpirationDelete, + objectSingleTypesEnd, + ObjectAccessedAll, + ObjectCreatedAll, + ObjectRemovedAll, + ObjectReplicationAll, + ObjectRestoreAll, + ObjectTransitionAll, + ObjectScannerAll, + #[default] + Everything, +} + +impl EventName { + fn expand(&self) -> Vec { + todo!(); + } + + fn mask(&self) -> u64 { + todo!(); + } +} + +impl AsRef for EventName { + fn as_ref(&self) -> &str { + match self { + EventName::BucketCreated => "s3:BucketCreated:*", + EventName::BucketRemoved => "s3:BucketRemoved:*", + EventName::ObjectAccessedAll => "s3:ObjectAccessed:*", + EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get", + EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention", + EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold", + EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head", + EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes", + EventName::ObjectCreatedAll => "s3:ObjectCreated:*", + EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload", + EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy", + EventName::ObjectCreatedPost => "s3:ObjectCreated:Post", + EventName::ObjectCreatedPut => "s3:ObjectCreated:Put", + EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging", + EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging", + EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention", + EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold", + EventName::ObjectRemovedAll => "s3:ObjectRemoved:*", + EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete", + EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated", + EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP", + EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions", + EventName::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete", + EventName::ObjectReplicationAll => "s3:Replication:*", + EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication", + EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication", + EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked", + EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold", + EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold", + EventName::ObjectRestoreAll => "s3:ObjectRestore:*", + EventName::ObjectRestorePost => "s3:ObjectRestore:Post", + EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed", + EventName::ObjectTransitionAll => "s3:ObjectTransition:*", + EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed", + EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete", + EventName::ObjectManyVersions => "s3:Scanner:ManyVersions", + EventName::ObjectLargeVersions => "s3:Scanner:LargeVersions", + EventName::PrefixManyFolders => "s3:Scanner:BigPrefix", + _ => "", + } + } +} + +impl From<&str> for EventName { + fn from(s: &str) -> Self { + match s { + "s3:BucketCreated:*" => { + EventName::BucketCreated + } + "s3:BucketRemoved:*" => { + EventName::BucketRemoved + } + "s3:ObjectAccessed:*" => { + EventName::ObjectAccessedAll + } + "s3:ObjectAccessed:Get" => { + EventName::ObjectAccessedGet + } + "s3:ObjectAccessed:GetRetention" => { + EventName::ObjectAccessedGetRetention + } + "s3:ObjectAccessed:GetLegalHold" => { + EventName::ObjectAccessedGetLegalHold + } + "s3:ObjectAccessed:Head" => { + EventName::ObjectAccessedHead + } + "s3:ObjectAccessed:Attributes" => { + EventName::ObjectAccessedAttributes + } + "s3:ObjectCreated:*" => { + EventName::ObjectCreatedAll + } + "s3:ObjectCreated:CompleteMultipartUpload" => { + EventName::ObjectCreatedCompleteMultipartUpload + } + "s3:ObjectCreated:Copy" => { + EventName::ObjectCreatedCopy + } + "s3:ObjectCreated:Post" => { + EventName::ObjectCreatedPost + } + "s3:ObjectCreated:Put" => { + EventName::ObjectCreatedPut + } + "s3:ObjectCreated:PutRetention" => { + EventName::ObjectCreatedPutRetention + } + "s3:ObjectCreated:PutLegalHold" => { + EventName::ObjectCreatedPutLegalHold + } + "s3:ObjectCreated:PutTagging" => { + EventName::ObjectCreatedPutTagging + } + "s3:ObjectCreated:DeleteTagging" => { + EventName::ObjectCreatedDeleteTagging + } + "s3:ObjectRemoved:*" => { + EventName::ObjectRemovedAll + } + "s3:ObjectRemoved:Delete" => { + EventName::ObjectRemovedDelete + } + "s3:ObjectRemoved:DeleteMarkerCreated" => { + EventName::ObjectRemovedDeleteMarkerCreated + } + "s3:ObjectRemoved:NoOP" => { + EventName::ObjectRemovedNoOP + } + "s3:ObjectRemoved:DeleteAllVersions" => { + EventName::ObjectRemovedDeleteAllVersions + } + "s3:LifecycleDelMarkerExpiration:Delete" => { + EventName::ILMDelMarkerExpirationDelete + } + "s3:Replication:*" => { + EventName::ObjectReplicationAll + } + "s3:Replication:OperationFailedReplication" => { + EventName::ObjectReplicationFailed + } + "s3:Replication:OperationCompletedReplication" => { + EventName::ObjectReplicationComplete + } + "s3:Replication:OperationMissedThreshold" => { + EventName::ObjectReplicationMissedThreshold + } + "s3:Replication:OperationReplicatedAfterThreshold" => { + EventName::ObjectReplicationReplicatedAfterThreshold + } + "s3:Replication:OperationNotTracked" => { + EventName::ObjectReplicationNotTracked + } + "s3:ObjectRestore:*" => { + EventName::ObjectRestoreAll + } + "s3:ObjectRestore:Post" => { + EventName::ObjectRestorePost + } + "s3:ObjectRestore:Completed" => { + EventName::ObjectRestoreCompleted + } + "s3:ObjectTransition:Failed" => { + EventName::ObjectTransitionFailed + } + "s3:ObjectTransition:Complete" => { + EventName::ObjectTransitionComplete + } + "s3:ObjectTransition:*" => { + EventName::ObjectTransitionAll + } + "s3:Scanner:ManyVersions" => { + EventName::ObjectManyVersions + } + "s3:Scanner:LargeVersions" => { + EventName::ObjectLargeVersions + } + "s3:Scanner:BigPrefix" => { + EventName::PrefixManyFolders + } + _ => { + EventName::Everything + } + } + } +} \ No newline at end of file diff --git a/ecstore/src/event/targetid.rs b/ecstore/src/event/targetid.rs new file mode 100644 index 00000000..c27e6eed --- /dev/null +++ b/ecstore/src/event/targetid.rs @@ -0,0 +1,10 @@ +pub struct TargetID { + id: String, + name: String, +} + +impl TargetID { + fn to_string(&self) -> String { + format!("{}:{}", self.id, self.name) + } +} diff --git a/ecstore/src/event/targetlist.rs b/ecstore/src/event/targetlist.rs new file mode 100644 index 00000000..67559df6 --- /dev/null +++ b/ecstore/src/event/targetlist.rs @@ -0,0 +1,31 @@ +use std::sync::atomic::AtomicI64; + +use super::targetid::TargetID; + +#[derive(Default)] +pub struct TargetList { + pub current_send_calls: AtomicI64, + pub total_events: AtomicI64, + pub events_skipped: AtomicI64, + pub events_errors_total: AtomicI64, + //pub targets: HashMap, + //pub queue: AsyncEvent, + //pub targetStats: HashMap, +} + +impl TargetList { + pub fn new() -> TargetList { + TargetList::default() + } +} + +struct TargetStat { + current_send_calls: i64, + total_events: i64, + failed_events: i64, +} + +struct TargetIDResult { + id: TargetID, + err: std::io::Error, +} diff --git a/ecstore/src/event_notification.rs b/ecstore/src/event_notification.rs new file mode 100644 index 00000000..4182fbc7 --- /dev/null +++ b/ecstore/src/event_notification.rs @@ -0,0 +1,61 @@ +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; + +use crate::store_api::ObjectInfo; +use crate::event::name::EventName; +use crate::event::targetlist::TargetList; +use crate::store::ECStore; +use crate::bucket::metadata::BucketMetadata; + +pub struct EventNotifier { + target_list: TargetList, + //bucket_rules_map: HashMap>, +} + +impl EventNotifier { + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + target_list: TargetList::new(), + //bucket_rules_map: HashMap::new(), + })) + } + + fn get_arn_list(&self) -> Vec { + todo!(); + } + + fn set(&self, bucket: &str, meta: BucketMetadata) { + todo!(); + } + + fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> { + /*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil { + return err + } + self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4) + nil*/ + todo!(); + } + + fn send(&self, args: EventArgs) { + todo!(); + } +} + +#[derive(Debug, Default)] +pub struct EventArgs { + pub event_name: String, + pub bucket_name: String, + pub object: ObjectInfo, + pub req_params: HashMap, + pub resp_elements: HashMap, + pub host: String, + pub user_agent: String, +} + +impl EventArgs { +} + +pub fn send_event(args: EventArgs) { +} diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index c60c6c59..d860718d 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -12,6 +12,9 @@ use crate::{ disk::DiskStore, endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, + bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, + tier::tier::TierConfigMgr, + event_notification::EventNotifier, store::ECStore, }; @@ -20,6 +23,8 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; +pub const DEFAULT_PORT: u16 = 9000; + lazy_static! { static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); @@ -33,11 +38,17 @@ lazy_static! { pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); + pub static ref GLOBAL_TierConfigMgr: Arc> = TierConfigMgr::new(); + pub static ref GLOBAL_LifecycleSys: Arc = LifecycleSys::new(); + pub static ref GLOBAL_EventNotifier: Arc> = EventNotifier::new(); + //pub static ref GLOBAL_RemoteTargetTransport pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); -} + pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); + pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes()); + pub static ref GLOBAL_NodeNamesHex: HashMap = HashMap::new();} /// Get the global rustfs port pub fn global_rustfs_port() -> u16 { diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index 3e26160f..3d69ba2a 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -6,19 +6,45 @@ use std::{ path::{Path, PathBuf}, pin::Pin, sync::{ - Arc, atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, + Arc, }, time::{Duration, SystemTime}, }; +use time::{self, OffsetDateTime}; +use common::defer; + +use rustfs_utils::path::encode_dir_object; use super::{ data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend}, data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash}, heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, }; -use crate::{bucket::metadata_sys, cmd::bucket_replication::queue_replication_heal}; +use crate::bucket::{ + object_lock::objectlock_sys::{ + enforce_retention_for_deletion, + BucketObjectLockSys, + }, utils::is_meta_bucketname, +}; +use crate::{ + bucket::{ + lifecycle::{ + bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}, + lifecycle::{self, Lifecycle, ExpirationOptions}, + bucket_lifecycle_ops::{ + self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, + expire_transitioned_object, + }, + }, + metadata_sys + }, + event_notification::{send_event, EventArgs}, + global::GLOBAL_LocalNodeName, heal::{data_scanner}, + store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, +}; +use crate::cmd::bucket_replication::queue_replication_heal; use crate::{ bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, cmd::bucket_replication::ReplicationStatusType, @@ -44,6 +70,7 @@ use crate::{ peer::is_reserved_or_invalid_bucket, store::ECStore, }; +use crate::event::name::EventName; use crate::{disk::DiskAPI, store_api::ObjectInfo}; use crate::{ disk::error::DiskError, @@ -56,7 +83,7 @@ use rand::Rng; use rmp_serde::{Deserializer, Serializer}; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus}; +use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration}; use serde::{Deserialize, Serialize}; use tokio::{ sync::{ @@ -515,8 +542,55 @@ impl ScannerItem { path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())]) } - pub async fn apply_versions_actions(&self, fives: &[FileInfo]) -> Result> { - let obj_infos = self.apply_newer_noncurrent_version_limit(fives).await?; + async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) { + let mut size = oi.get_actual_size().expect("err!"); + if self.debug { + info!("apply_lifecycle debug"); + } + if self.lifecycle.is_none() { + return (lifecycle::IlmAction::NoneAction, size as i64); + } + + let version_id = oi.version_id; + + let mut vc = None; + let mut lr = None; + let mut rcfg = None; + if !is_meta_bucketname(&self.bucket) { + vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap()); + lr = BucketObjectLockSys::get(&self.bucket).await; + rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await { + Some(replication_config) + } else { None }; + } + + let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; + if self.debug { + if !version_id.is_none() { + info!("lifecycle: {} (version-id={}), Initial scan: {}", self.object_path().to_string_lossy().to_string(), version_id.expect("err"), lc_evt.action); + } else { + info!("lifecycle: {} Initial scan: {}", self.object_path().to_string_lossy().to_string(), lc_evt.action); + } + } + + match lc_evt.action { + lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + size = 0; + } + lifecycle::IlmAction::DeleteAction => { + if !vc.unwrap().prefix_enabled(&oi.name) { + size = 0 + } + } + _ => () + } + + apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; + (lc_evt.action, size as i64) + } + + pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result> { + let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?; if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize { // todo } @@ -533,16 +607,22 @@ impl ScannerItem { Ok(obj_infos) } - pub async fn apply_newer_noncurrent_version_limit(&self, fives: &[FileInfo]) -> Result> { + pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result> { // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent); + + let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { + rcfg.mode.is_some() + } else { false }; + let vcfg = BucketVersioningSys::get(&self.bucket).await?; + let versioned = match BucketVersioningSys::get(&self.bucket).await { Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()), Err(_) => false, }; - let mut object_infos = Vec::with_capacity(fives.len()); + let mut object_infos = Vec::with_capacity(fivs.len()); if self.lifecycle.is_none() { - for info in fives.iter() { + for info in fivs.iter() { object_infos.push(ObjectInfo::from_file_info( info, &self.bucket, @@ -553,6 +633,54 @@ impl ScannerItem { return Ok(object_infos); } + let event = self.lifecycle.as_ref().expect("lifecycle err.").noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { + name: self.object_path().to_string_lossy().to_string(), + ..Default::default() + }).await; + let lim = event.newer_noncurrent_versions; + if lim == 0 || fivs.len() <= lim+1 { + for fi in fivs.iter() { + object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); + } + return Ok(object_infos); + } + + let overflow_versions = &fivs[lim+1..]; + for fi in fivs[..lim+1].iter() { + object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); + } + + let mut to_del = Vec::::with_capacity(overflow_versions.len()); + for fi in overflow_versions.iter() { + let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); + if lock_enabled && enforce_retention_for_deletion(&obj) { + //if enforce_retention_for_deletion(&obj) { + if self.debug { + if obj.version_id.is_some() { + info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting\n", obj.name); + } + } + object_infos.push(obj); + continue; + } + + if OffsetDateTime::now_utc().unix_timestamp() < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32).unix_timestamp() { + object_infos.push(obj); + continue; + } + + to_del.push(ObjectToDelete { + object_name: obj.name, + version_id: obj.version_id, + }); + } + + if to_del.len() > 0 { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await; + } // done().await; Ok(object_infos) @@ -560,7 +688,9 @@ impl ScannerItem { pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, usize) { let done = ScannerMetrics::time(ScannerMetric::Ilm); - //todo: lifecycle + + let (action, size) = self.apply_lifecycle(oi).await; + info!( "apply_actions {} {} {:?} {:?}", oi.bucket.clone(), @@ -581,6 +711,10 @@ impl ScannerItem { self.heal_replication(&oi, _size_s).await; done(); + if action.delete_all() { + return (true, 0); + } + (false, oi.size) } @@ -757,9 +891,9 @@ impl FolderScanner { let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name); // Todo: lifeCycle - let active_life_cycle = if let Some(lc) = self.old_cache.info.life_cycle.as_ref() { + let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() { if lc_has_active_rules(lc, &prefix) { - self.old_cache.info.life_cycle.clone() + self.old_cache.info.lifecycle.clone() } else { None } @@ -1401,7 +1535,151 @@ pub async fn scan_data_folder( Ok(s.new_cache) } -// pub fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: &ObjectLockConfiguration, rcfg: &ReplicationConfiguration， obj: &ObjectInfo) +pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: Option, rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, oi: &ObjectInfo) -> lifecycle::Event { + let event = lc.eval(&oi.to_lifecycle_opts()).await; + //if serverDebugLog { + info!("lifecycle: Secondary scan: {}", event.action); + //} + + let lock_enabled = if let Some(lr) = lr { + lr.mode.is_some() + } else { false }; + + match event.action { + lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + if lock_enabled { + return lifecycle::Event::default(); + } + } + lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => { + if oi.version_id.is_none() { + return lifecycle::Event::default(); + } + if lock_enabled && enforce_retention_for_deletion(oi) { + //if serverDebugLog { + if !oi.version_id.is_none() { + info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting", oi.name); + } + //} + return lifecycle::Event::default(); + } + if let Some(rcfg) = rcfg { + if rep_has_active_rules(&rcfg.0, &oi.name, true) { + return lifecycle::Event::default(); + } + } + } + _ => () + } + + event +} + +async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + if oi.delete_marker || oi.is_dir { + return false; + } + GLOBAL_TransitionState.queue_transition_task(oi, event, src).await; + true +} + +pub async fn apply_expiry_on_transitioned_object(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { + let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { + return false; + } + time_ilm(1); + + true +} + +pub async fn apply_expiry_on_non_transitioned_objects(api: Arc, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { + let mut opts = ObjectOptions { + expiration: ExpirationOptions {expire: true}, + ..Default::default() + }; + + if lc_event.action.delete_versioned() { + opts.version_id = Some(oi.version_id.expect("err").to_string()); + } + + opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; + opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await; + + if lc_event.action.delete_all() { + opts.delete_prefix = true; + opts.delete_prefix_object = true; + } + let dobj: ObjectInfo; + //let err: Error; + + let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + + let mut dobj = api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await.unwrap(); + if dobj.name == "" { + dobj = oi.clone(); + } + + //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags(); + //tags["version-id"] = dobj.version_id; + + let mut event_name = EventName::ObjectRemovedDelete; + if oi.delete_marker { + event_name = EventName::ObjectRemovedDeleteMarkerCreated; + } + match lc_event.action { + lifecycle::IlmAction::DeleteAllVersionsAction => { + event_name = EventName::ObjectRemovedDeleteAllVersions + } + lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + event_name = EventName::ILMDelMarkerExpirationDelete + } + _ => () + } + send_event(EventArgs { + event_name: event_name.as_ref().to_string(), + bucket_name: dobj.bucket.clone(), + object: dobj, + user_agent: "Internal: [ILM-Expiry]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + + if lc_event.action != lifecycle::IlmAction::NoneAction { + let mut num_versions = 1_u64; + if lc_event.action.delete_all() { + num_versions = oi.num_versions as u64; + } + time_ilm(num_versions); + } + + true +} + +async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_by_days(oi, event, src).await; + true +} + +pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut success = false; + match event.action { + lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAction + | lifecycle::IlmAction::DeleteRestoredAction | lifecycle::IlmAction::DeleteRestoredVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + success = apply_expiry_rule(event, src, oi).await; + } + lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + success = apply_transition_rule(event, src, oi).await; + } + _ => () + } + success +} + #[cfg(test)] mod tests { diff --git a/ecstore/src/heal/data_scanner_metric.rs b/ecstore/src/heal/data_scanner_metric.rs index 6d97106a..f59818af 100644 --- a/ecstore/src/heal/data_scanner_metric.rs +++ b/ecstore/src/heal/data_scanner_metric.rs @@ -15,14 +15,13 @@ use tokio::sync::{Mutex, RwLock}; use tracing::debug; use super::data_scanner::CurrentScannerCycle; +use crate::bucket::lifecycle::lifecycle; lazy_static! { pub static ref globalScannerMetrics: Arc = Arc::new(ScannerMetrics::new()); } -/// Scanner metric types, matching the Go version exactly -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[repr(u8)] +#[derive(Clone, Debug, PartialEq, PartialOrd)] pub enum ScannerMetric { // START Realtime metrics, that only records // last minute latencies and total operation count. @@ -196,7 +195,8 @@ pub struct ScannerMetrics { // All fields must be accessed atomically and aligned. operations: Vec, latency: Vec, - + actions: Vec, + actions_latency: Vec, // Current paths contains disk -> tracker mappings current_paths: Arc>>>, @@ -215,6 +215,8 @@ impl ScannerMetrics { Self { operations, latency, + actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(), + actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize], current_paths: Arc::new(RwLock::new(HashMap::new())), cycle_info: Arc::new(RwLock::new(None)), } @@ -222,16 +224,17 @@ impl ScannerMetrics { /// Log scanner action with custom metadata - compatible with existing usage pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap) { + let metric = metric as usize; let start_time = SystemTime::now(); move |_custom: &HashMap| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task for this) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -239,23 +242,24 @@ impl ScannerMetrics { // Log trace metrics if metric as u8 > ScannerMetric::StartTrace as u8 { - debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); + //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); } } } /// Time scanner action with size - returns function that takes size pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) { + let metric = metric as usize; let start_time = SystemTime::now(); move |size: u64| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics with size (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add_size(duration, size).await; }); @@ -265,16 +269,17 @@ impl ScannerMetrics { /// Time a scanner action - returns a closure to call when done pub fn time(metric: ScannerMetric) -> impl Fn() { + let metric = metric as usize; let start_time = SystemTime::now(); move || { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -284,17 +289,18 @@ impl ScannerMetrics { /// Time N scanner actions - returns function that takes count, then returns completion function pub fn time_n(metric: ScannerMetric) -> Box Box + Send + Sync> { + let metric = metric as usize; let start_time = SystemTime::now(); Box::new(move |count: usize| { Box::new(move || { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(count as u64, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -303,31 +309,53 @@ impl ScannerMetrics { }) } + pub fn time_ilm(a: lifecycle::IlmAction) -> Box Box + Send + Sync> { + let a_clone = a as usize; + if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { + return Box::new(move |_: u64| { + Box::new(move || {}) + }); + } + let start = SystemTime::now(); + Box::new(move |versions: u64| { + Box::new(move || { + let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); + tokio::spawn(async move { + globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed); + globalScannerMetrics.actions_latency[a_clone].add(duration).await; + }); + }) + }) + } + /// Increment time with specific duration pub async fn inc_time(metric: ScannerMetric, duration: Duration) { + let metric = metric as usize; // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics - if (metric as usize) < ScannerMetric::LastRealtime as usize { - globalScannerMetrics.latency[metric as usize].add(duration).await; + if (metric) < ScannerMetric::LastRealtime as usize { + globalScannerMetrics.latency[metric].add(duration).await; } } /// Get lifetime operation count for a metric pub fn lifetime(&self, metric: ScannerMetric) -> u64 { - if (metric as usize) >= ScannerMetric::Last as usize { + let metric = metric as usize; + if (metric) >= ScannerMetric::Last as usize { return 0; } - self.operations[metric as usize].load(Ordering::Relaxed) + self.operations[metric].load(Ordering::Relaxed) } /// Get last minute statistics for a metric pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem { - if (metric as usize) >= ScannerMetric::LastRealtime as usize { + let metric = metric as usize; + if (metric) >= ScannerMetric::LastRealtime as usize { return AccElem::default(); } - self.latency[metric as usize].total().await + self.latency[metric].total().await } /// Set current cycle information diff --git a/ecstore/src/heal/data_usage_cache.rs b/ecstore/src/heal/data_usage_cache.rs index 2e329f89..35aac655 100644 --- a/ecstore/src/heal/data_usage_cache.rs +++ b/ecstore/src/heal/data_usage_cache.rs @@ -129,6 +129,58 @@ const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERS }, ]; +#[derive(Clone, Copy, Default)] +pub struct TierStats { + pub total_size: u64, + pub num_versions: i32, + pub num_objects: i32, +} + +impl TierStats { + pub fn add(&self, u: &TierStats) -> TierStats { + TierStats { + total_size: self.total_size + u.total_size, + num_versions: self.num_versions + u.num_versions, + num_objects: self.num_objects + u.num_objects, + } + } +} + +#[derive(Clone)] +struct AllTierStats { + tiers: HashMap, +} + +impl AllTierStats { + pub fn new() -> Self { + Self { + tiers: HashMap::new(), + } + } + + fn add_sizes(&mut self, tiers: HashMap) { + for (tier, st) in tiers { + self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); + } + } + + fn merge(&mut self, other: AllTierStats) { + for (tier, st) in other.tiers { + self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); + } + } + + fn populate_stats(&self, stats: &mut HashMap) { + for (tier, st) in &self.tiers { + stats.insert(tier.clone(), TierStats { + total_size: st.total_size.clone(), + num_versions: st.num_versions.clone(), + num_objects: st.num_objects.clone(), + }); + } + } +} + // sizeHistogram is a size histogram. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SizeHistogram(Vec); @@ -346,8 +398,7 @@ pub struct DataUsageCacheInfo { pub last_update: Option, pub skip_healing: bool, #[serde(skip)] - pub life_cycle: Option, - // pub life_cycle: + pub lifecycle: Option, #[serde(skip)] pub updates: Option>, #[serde(skip)] diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 294c2669..178aa4f4 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -26,6 +26,13 @@ mod store_init; pub mod store_list_objects; mod store_utils; +pub mod checksum; +pub mod event; +pub mod event_notification; +pub mod client; +pub mod tier; +pub mod signer; + pub use global::new_object_layer_fn; pub use global::set_global_endpoints; pub use global::update_erasure_type; diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 0efbce66..548a60b3 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -1,3 +1,5 @@ +use s3s::header::X_AMZ_RESTORE; +use crate::error::ObjectApiError; use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::{ @@ -20,6 +22,8 @@ use crate::{ UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, error::{StorageError, to_object_err}, + event::name::EventName, event_notification::{send_event, EventArgs}, + bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, put_restore_opts, get_transitioned_object_reader,}, global::{ GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure, @@ -40,11 +44,16 @@ use crate::{ }, store_init::load_format_erasure, }; +use crate::client::{ + object_api_utils::extract_etag, + transition_api::ReaderImpl, +}; use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; use crate::{ heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, store_api::ListObjectVersionsInfo, }; +use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use bytesize::ByteSize; use chrono::Utc; use futures::future::join_all; @@ -91,6 +100,7 @@ use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; use workers::workers::Workers; +use crate::global::{GLOBAL_TierConfigMgr, GLOBAL_LocalNodeName}; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; @@ -3666,6 +3676,29 @@ impl SetDisks { Ok(()) } + + pub async fn update_restore_metadata(&self, bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> { + let mut oi = obj_info.clone(); + oi.metadata_only = true; + + if let Some(user_defined) = &mut oi.user_defined { + user_defined.remove(X_AMZ_RESTORE.as_str()); + } + + let version_id = oi.version_id.clone().map(|v| v.to_string()); + let obj = self.copy_object(bucket, object, bucket, object, &mut oi, &ObjectOptions { + version_id: version_id.clone(), + ..Default::default() + }, &ObjectOptions { + version_id: version_id, + ..Default::default() + }).await; + if let Err(err) = obj { + //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) + return Err(err); + } + Ok(()) + } } #[async_trait::async_trait] @@ -4125,6 +4158,45 @@ impl StorageAPI for SetDisks { src_opts.versioned || src_opts.version_suspended, )) } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + let disks = self.get_disks(0, 0).await?; + let write_quorum = disks.len() / 2 + 1; + + let mut futures = Vec::with_capacity(disks.len()); + let mut errs = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + futures.push(async move { + if let Some(disk) = disk { + match disk.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()).await { + Ok(r) => Ok(r), + Err(e) => Err(e), + } + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(err.into()); + } + Ok(()) + } + #[tracing::instrument(skip(self))] async fn delete_objects( &self, @@ -4318,6 +4390,21 @@ impl StorageAPI for SetDisks { Ok(oi) } + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + GLOBAL_MRFState.add_partial(PartialOperation { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: Some(version_id.to_string()), + queued: Utc::now(), + set_index: self.set_index, + pool_index: self.pool_index, + ..Default::default() + }) + .await; + Ok(()) + } + #[tracing::instrument(skip(self))] async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { // TODO: nslock @@ -4404,6 +4491,201 @@ impl StorageAPI for SetDisks { Ok(oi.user_tags) } + #[tracing::instrument(level = "debug", skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await { + Ok(client) => client, + Err(err) => { + return Err(Error::other(err.to_string())); + } + }; + + /*if !opts.no_lock { + let lk = self.new_ns_lock(bucket, object); + let lkctx = lk.get_lock(globalDeleteOperationTimeout)?; + //ctx = lkctx.Context() + //defer lk.Unlock(lkctx) + }*/ + + let (mut fi, mut meta_arr, mut online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true).await?; + /*if err != nil { + return Err(to_object_err(err, vec![bucket, object])); + }*/ + /*if fi.deleted { + if opts.version_id.is_none() { + return Err(to_object_err(DiskError::FileNotFound, vec![bucket, object])); + } + return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object])); + }*/ + if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() || !(opts.transition.etag == extract_etag(&fi.metadata)) { + return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object])); + } + if fi.transition_status == TRANSITION_COMPLETE { + return Ok(()); + } + + /*if fi.xlv1 { + if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) { + return err.expect("err"); + } + (fi, meta_arr, online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true); + if err != nil { + return to_object_err(err, vec![bucket, object]); + } + }*/ + //let traceFn = GLOBAL_LifecycleSys.trace(fi.to_object_info(bucket, object, opts.Versioned || opts.VersionSuspended)); + + let dest_obj = gen_transition_objname(bucket); + if let Err(err) = dest_obj { + //traceFn(ILMTransition, nil, err) + return Err(to_object_err(err, vec![])); + } + let dest_obj = dest_obj.unwrap(); + + let oi = ObjectInfo::from_file_info(&fi, &bucket, &object, opts.versioned || opts.version_suspended); + + let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); + //let h = HeaderMap::new(); + //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi}); + let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: Box::new(pr), object_info: oi}); + + let cloned_bucket = bucket.to_string(); + let cloned_object = object.to_string(); + let cloned_fi = fi.clone(); + let set_index = self.set_index; + let pool_index = self.pool_index; + tokio::spawn(async move { + if let Err(e) = Self::get_object_with_fileinfo( + &cloned_bucket, &cloned_object, 0, cloned_fi.size, &mut pw, cloned_fi, meta_arr, &online_disks, set_index, pool_index + ) + .await + { + error!("get_object_with_fileinfo err {:?}", e); + }; + }); + + let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size as i64, { + let mut m = HashMap::::new(); + m.insert("name".to_string(), object.to_string()); + m + }).await; + //pr.CloseWithError(err); + if let Err(err) = rv { + //traceFn(ILMTransition, nil, err) + return Err(StorageError::Io(err)); + } + let rv = rv.unwrap(); + fi.transition_status = TRANSITION_COMPLETE.to_string(); + fi.transitioned_objname = dest_obj; + fi.transition_tier = opts.transition.tier.clone(); + fi.transition_version_id = if rv=="" { None } else { Some(Uuid::parse_str(&rv)?) }; + let mut event_name = EventName::ObjectTransitionComplete.as_ref(); + + let disks = self.get_disks(0, 0).await?; + + if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await { + event_name = EventName::ObjectTransitionFailed.as_ref(); + } + + for disk in disks.iter() { + if let Some(disk) = disk { + if disk.is_online().await { + continue; + } + } + self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")).await; + break; + } + + let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + send_event(EventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: obj_info, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + //let tags = opts.lifecycle_audit_event.tags(); + //auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn) + Ok(()) + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option| -> Result<()> { + if rerr.is_none() { + return Ok(()); + } + self.update_restore_metadata(bucket, object, oi, opts).await?; + Err(rerr.unwrap()) + }; + let mut oi = ObjectInfo::default(); + let fi = self.get_object_fileinfo(&bucket, &object, &opts, true).await; + if let Err(err) = fi { + return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await; + } + let (mut actual_fi, _, _) = fi.unwrap(); + + oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended); + let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi); + /*if oi.parts.len() == 1 { + let mut rs: HTTPRangeSpec; + let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts); + //if err != nil { + // return set_restore_header_fn(&mut oi, Some(toObjectErr(err, bucket, object))); + //} + //defer gr.Close() + let hash_reader = HashReader::new(gr, gr.obj_info.size, "", "", gr.obj_info.size); + let p_reader = PutObjReader::new(StreamingBlob::from(Box::pin(hash_reader)), hash_reader.size()); + if let Err(err) = self.put_object(bucket, object, &mut p_reader, &ropts).await { + return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))); + } else { + return Ok(()); + } + } + + let res = self.new_multipart_upload(bucket, object, &ropts).await?; + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + + let mut uploaded_parts: Vec = vec![]; + let mut rs: HTTPRangeSpec; + let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts).await?; + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + + for part_info in oi.parts { + //let hr = HashReader::new(LimitReader(gr, part_info.size), part_info.size, "", "", part_info.size); + let hr = HashReader::new(gr, part_info.size as i64, part_info.size as i64, None, false); + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + let mut p_reader = PutObjReader::new(hr, hr.size()); + let p_info = self.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()).await?; + //if let Err(err) = p_info { + // return set_restore_header_fn(&mut oi, err); + //} + if p_info.size != part_info.size { + return set_restore_header_fn(&mut oi, Some(Error::from(ObjectApiError::InvalidObjectState(GenericError{bucket: bucket.to_string(), object: object.to_string(), ..Default::default()})))); + } + uploaded_parts.push(CompletePart { + part_num: p_info.part_num, + etag: p_info.etag, + }); + } + if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions { + mod_time: oi.mod_time, + ..Default::default() + }).await { + set_restore_header_fn(&mut oi, Some(err)); + }*/ + Ok(()) + } + #[tracing::instrument(level = "debug", skip(self))] async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { let (mut fi, _, disks) = self.get_object_fileinfo(bucket, object, opts, false).await?; diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index b4b64178..808946e1 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,6 +1,7 @@ #![allow(clippy::map_entry)] use std::{collections::HashMap, sync::Arc}; +use rustfs_filemeta::FileInfo; use crate::disk::error_reduce::count_errs; use crate::error::{Error, Result}; use crate::{ @@ -459,6 +460,11 @@ impl StorageAPI for Sets { )) } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + unimplemented!() + } + #[tracing::instrument(skip(self))] async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { if opts.delete_prefix && !opts.delete_prefix_object { @@ -572,6 +578,22 @@ impl StorageAPI for Sets { self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await } + #[tracing::instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object).transition_object(bucket, object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + self.get_disks_by_key(object).add_partial(bucket, object, version_id).await; + Ok(()) + } + + #[tracing::instrument(skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object).restore_transitioned_object(bucket, object, opts).await + } + #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -666,6 +688,8 @@ impl StorageAPI for Sets { .await } + + #[tracing::instrument(skip(self))] async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await diff --git a/ecstore/src/signer/mod.rs b/ecstore/src/signer/mod.rs new file mode 100644 index 00000000..105e92ca --- /dev/null +++ b/ecstore/src/signer/mod.rs @@ -0,0 +1,13 @@ +pub mod utils; +pub mod request_signature_v2; +pub mod request_signature_v4; +pub mod request_signature_streaming; +pub mod request_signature_streaming_unsigned_trailer; +pub mod ordered_qs; + +pub use request_signature_v2::sign_v2; +pub use request_signature_v2::pre_sign_v2; +pub use request_signature_v4::sign_v4; +pub use request_signature_v4::pre_sign_v4; +pub use request_signature_v4::sign_v4_trailer; +pub use request_signature_streaming::streaming_sign_v4; \ No newline at end of file diff --git a/ecstore/src/signer/ordered_qs.rs b/ecstore/src/signer/ordered_qs.rs new file mode 100644 index 00000000..7cc97e2b --- /dev/null +++ b/ecstore/src/signer/ordered_qs.rs @@ -0,0 +1,109 @@ +//! Ordered query strings + +use crate::signer::utils::stable_sort_by_first; + +/// Immutable query string container +#[derive(Debug, Default, Clone)] +pub struct OrderedQs { + /// Ascending query strings + qs: Vec<(String, String)>, +} + +/// [`OrderedQs`] +#[derive(Debug, thiserror::Error)] +#[error("ParseOrderedQsError: {inner}")] +pub struct ParseOrderedQsError { + /// url decode error + inner: serde_urlencoded::de::Error, +} + +impl OrderedQs { + /// Constructs [`OrderedQs`] from vec + /// + /// + strings must be url-decoded + #[cfg(test)] + #[must_use] + pub fn from_vec_unchecked(mut v: Vec<(String, String)>) -> Self { + stable_sort_by_first(&mut v); + Self { qs: v } + } + + /// Parses [`OrderedQs`] from query + /// + /// # Errors + /// Returns [`ParseOrderedQsError`] if query cannot be decoded + pub fn parse(query: &str) -> Result { + let result = serde_urlencoded::from_str::>(query); + let mut v = result.map_err(|e| ParseOrderedQsError { inner: e })?; + stable_sort_by_first(&mut v); + Ok(Self { qs: v }) + } + + #[must_use] + pub fn has(&self, name: &str) -> bool { + self.qs.binary_search_by_key(&name, |x| x.0.as_str()).is_ok() + } + + /// Gets query values by name. Time `O(logn)` + pub fn get_all(&self, name: &str) -> impl Iterator + use<'_> { + let qs = self.qs.as_slice(); + + let lower_bound = qs.partition_point(|x| x.0.as_str() < name); + let upper_bound = qs.partition_point(|x| x.0.as_str() <= name); + + qs[lower_bound..upper_bound].iter().map(|x| x.1.as_str()) + } + + pub fn get_unique(&self, name: &str) -> Option<&str> { + let qs = self.qs.as_slice(); + let lower_bound = qs.partition_point(|x| x.0.as_str() < name); + + let mut iter = qs[lower_bound..].iter(); + let pair = iter.next()?; + + if let Some(following) = iter.next() { + if following.0 == name { + return None; + } + } + + (pair.0.as_str() == name).then_some(pair.1.as_str()) + } +} + +impl AsRef<[(String, String)]> for OrderedQs { + fn as_ref(&self) -> &[(String, String)] { + self.qs.as_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tag() { + { + let query = "tagging"; + let qs = OrderedQs::parse(query).unwrap(); + assert_eq!(qs.as_ref(), &[("tagging".to_owned(), String::new())]); + + assert_eq!(qs.get_unique("taggin"), None); + assert_eq!(qs.get_unique("tagging"), Some("")); + assert_eq!(qs.get_unique("taggingg"), None); + } + + { + let query = "tagging&tagging"; + let qs = OrderedQs::parse(query).unwrap(); + assert_eq!( + qs.as_ref(), + &[("tagging".to_owned(), String::new()), ("tagging".to_owned(), String::new())] + ); + + assert_eq!(qs.get_unique("taggin"), None); + assert_eq!(qs.get_unique("tagging"), None); + assert_eq!(qs.get_unique("taggingg"), None); + } + } +} diff --git a/ecstore/src/signer/request_signature_streaming.rs b/ecstore/src/signer/request_signature_streaming.rs new file mode 100644 index 00000000..81175b74 --- /dev/null +++ b/ecstore/src/signer/request_signature_streaming.rs @@ -0,0 +1,71 @@ +use std::pin::Pin; +use std::sync::Mutex; +use futures::prelude::*; +use futures::task; +use bytes::{Bytes, BytesMut}; +use http::header::TRAILER; +use http::Uri; +use lazy_static::lazy_static; +use std::collections::HashMap; +use stdx::str::StrExt; +use std::fmt::Write; +use tracing::{error, info, warn, debug}; +use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; +use http::request::{self, Request}; +use http::HeaderMap; +use hyper::Method; + +use rustfs_utils::{ + crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256}, + hash::EMPTY_STRING_SHA256_HASH +}; +use crate::client::constants::UNSIGNED_PAYLOAD; +use super::request_signature_v4::{get_scope, get_signing_key, get_signature, SERVICE_TYPE_S3}; + +const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; +const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER"; +const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024; +const CHUNK_SIGCONST_LEN: i64 = 17; +const SIGNATURESTR_LEN: i64 = 64; +const CRLF_LEN: i64 = 2; +const TRAILER_KV_SEPARATOR: &str = ":"; +const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature"; + +lazy_static! { + static ref ignored_streaming_headers: HashMap = { + let mut m = >::new(); + m.insert("authorization".to_string(), true); + m.insert("user-agent".to_string(), true); + m.insert("content-type".to_string(), true); + m + }; +} + +fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &str, chunk_check_sum: &str) -> String { + let mut string_to_sign_parts = >::new(); + string_to_sign_parts.push(STREAMING_PAYLOAD_HDR.to_string()); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + string_to_sign_parts.push(t.format(&format).unwrap()); + string_to_sign_parts.push(get_scope(region, t, SERVICE_TYPE_S3)); + string_to_sign_parts.push(previous_sig.to_string()); + string_to_sign_parts.push(EMPTY_STRING_SHA256_HASH.to_string()); + string_to_sign_parts.push(chunk_check_sum.to_string()); + string_to_sign_parts.join("\n") +} + +fn build_chunk_signature(chunk_check_sum: &str, req_time: OffsetDateTime, region: &str, + previous_signature: &str, secret_access_key: &str +) -> String { + let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, + previous_signature, chunk_check_sum); + let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3); + get_signature(signing_key, &chunk_string_to_sign) +} + +pub fn streaming_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, + region: &str, data_len: i64, req_time: OffsetDateTime/*, sh256: md5simd.Hasher*/ +) -> request::Builder { + todo!(); +} diff --git a/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs new file mode 100644 index 00000000..3f84aabc --- /dev/null +++ b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs @@ -0,0 +1,6 @@ +use http::request; +use time::OffsetDateTime; + +pub fn streaming_unsigned_v4(mut req: request::Builder, session_token: &str, data_len: i64, req_time: OffsetDateTime) -> request::Builder{ + todo!(); +} diff --git a/ecstore/src/signer/request_signature_v2.rs b/ecstore/src/signer/request_signature_v2.rs new file mode 100644 index 00000000..8b4aa763 --- /dev/null +++ b/ecstore/src/signer/request_signature_v2.rs @@ -0,0 +1,202 @@ +use std::collections::HashMap; +use http::request; +use hyper::Uri; +use bytes::{Bytes, BytesMut}; +use std::fmt::Write; +use time::{OffsetDateTime, macros::format_description, format_description}; + +use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; + +use super::utils::get_host_addr; + +const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const SIGN_V2_ALGORITHM: &str = "AWS"; + +fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String { + let mut path = "".to_string(); + + //path = serde_urlencoded::to_string(req.uri_ref().unwrap().path().unwrap()).unwrap(); + path = req.uri_ref().unwrap().path().to_string(); + path +} + +pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, expires: i64, virtual_host: bool) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let d = OffsetDateTime::now_utc(); + let d = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + let epoch_expires = d.unix_timestamp() + expires; + + let mut headers = req.headers_mut().expect("err"); + let expires_str = headers.get("Expires"); + if expires_str.is_none() { + headers.insert("Expires", format!("{:010}", epoch_expires).parse().unwrap()); + } + + let string_to_sign = pre_string_to_sign_v2(&req, virtual_host); + let signature = hex(hmac_sha1(secret_access_key, string_to_sign)); + + let result = serde_urlencoded::from_str::>(req.uri_ref().unwrap().query().unwrap()); + let mut query = result.unwrap_or_default(); + if get_host_addr(&req).contains(".storage.googleapis.com") { + query.insert("GoogleAccessId".to_string(), access_key_id.to_string()); + } else { + query.insert("AWSAccessKeyId".to_string(), access_key_id.to_string()); + } + + query.insert("Expires".to_string(), format!("{:010}", epoch_expires)); + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + req +} + +fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> String { + let signature = hex(hmac_sha1(secret_access_key, policy_base64)); + signature +} + +pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, virtual_host: bool) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let d = OffsetDateTime::now_utc(); + let d2 = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let string_to_sign = string_to_sign_v2(&req, virtual_host); + let mut headers = req.headers_mut().expect("err"); + + let date = headers.get("Date").unwrap(); + if date.to_str().unwrap() == "" { + headers.insert("Date", d2.format(&format_description::well_known::Rfc2822).unwrap().to_string().parse().unwrap()); + } + + let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id); + let auth_header = format!("{}{}", auth_header, base64_encode(&hmac_sha1(secret_access_key, string_to_sign))); + + headers.insert("Authorization", auth_header.parse().unwrap()); + + req +} + +fn pre_string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String { + let mut buf = BytesMut::new(); + write_pre_sign_v2_headers(&mut buf, &req); + write_canonicalized_headers(&mut buf, &req); + write_canonicalized_resource(&mut buf, &req, virtual_host); + String::from_utf8(buf.to_vec()).unwrap() +} + +fn write_pre_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) { + buf.write_str(req.method_ref().unwrap().as_str()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Expires").unwrap().to_str().unwrap()); + buf.write_char('\n'); +} + +fn string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String { + let mut buf = BytesMut::new(); + write_sign_v2_headers(&mut buf, &req); + write_canonicalized_headers(&mut buf, &req); + write_canonicalized_resource(&mut buf, &req, virtual_host); + String::from_utf8(buf.to_vec()).unwrap() +} + +fn write_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) { + buf.write_str(req.method_ref().unwrap().as_str()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Date").unwrap().to_str().unwrap()); + buf.write_char('\n'); +} + +fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) { + let mut proto_headers = Vec::::new(); + let mut vals = HashMap::>::new(); + for k in req.headers_ref().expect("err").keys() { + let lk = k.as_str().to_lowercase(); + if lk.starts_with("x-amz") { + proto_headers.push(lk.clone()); + let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); + vals.insert(lk, vv); + } + } + proto_headers.sort(); + for k in proto_headers { + buf.write_str(&k); + buf.write_char(':'); + for (idx, v) in vals[&k].iter().enumerate() { + if idx > 0 { + buf.write_char(','); + } + buf.write_str(v); + } + buf.write_char('\n'); + } +} + +const INCLUDED_QUERY: &[&str] = &[ + "acl", + "delete", + "lifecycle", + "location", + "logging", + "notification", + "partNumber", + "policy", + "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", + "uploadId", + "uploads", + "versionId", + "versioning", + "versions", + "website", +]; + +fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virtual_host: bool) { + let request_url = req.uri_ref().unwrap(); + buf.write_str(&encode_url2path(req, virtual_host)); + if request_url.query().unwrap() != "" { + let mut n: i64 = 0; + let result = serde_urlencoded::from_str::>>(req.uri_ref().unwrap().query().unwrap()); + let mut vals = result.unwrap_or_default(); + for resource in INCLUDED_QUERY { + let vv = &vals[*resource]; + if vv.len() > 0 { + n += 1; + match n { + 1 => { + buf.write_char('?'); + } + _ => { + buf.write_char('&'); + buf.write_str(resource); + if vv[0].len() > 0 { + buf.write_char('='); + buf.write_str(&vv[0]); + } + } + } + } + } + } +} \ No newline at end of file diff --git a/ecstore/src/signer/request_signature_v4.rs b/ecstore/src/signer/request_signature_v4.rs new file mode 100644 index 00000000..29f055da --- /dev/null +++ b/ecstore/src/signer/request_signature_v4.rs @@ -0,0 +1,607 @@ +use bytes::{Bytes, BytesMut}; +use http::header::TRAILER; +use http::Uri; +use lazy_static::lazy_static; +use std::collections::HashMap; +use std::fmt::Write; +use tracing::{error, info, warn, debug}; +use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; +use http::request::{self, Request}; +use http::HeaderMap; + +use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use crate::client::constants::UNSIGNED_PAYLOAD; +use super::ordered_qs::OrderedQs; +use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; +use super::utils::stable_sort_by_first; +use super::utils::{get_host_addr, sign_v4_trim_all}; + +pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +pub const SERVICE_TYPE_S3: &str = "s3"; +pub const SERVICE_TYPE_STS: &str = "sts"; + +lazy_static! { + static ref v4_ignored_headers: HashMap = { + let mut m = >::new(); + m.insert("accept-encoding".to_string(), true); + m.insert("authorization".to_string(), true); + m.insert("user-agent".to_string(), true); + m + }; +} + +pub fn get_signing_key(secret: &str, loc: &str, t: OffsetDateTime, service_type: &str) -> [u8; 32] { + let mut s = "AWS4".to_string(); + s.push_str(secret); + let format = format_description!("[year][month][day]"); + let date = hmac_sha256(s.into_bytes(), t.format(&format).unwrap().into_bytes()); + let location = hmac_sha256(date, loc); + let service = hmac_sha256(location, service_type); + let signing_key = hmac_sha256(service, "aws4_request"); + signing_key +} + +pub fn get_signature(signing_key: [u8; 32], string_to_sign: &str) -> String { + hex(hmac_sha256(signing_key, string_to_sign)) +} + +pub fn get_scope(location: &str, t: OffsetDateTime, service_type: &str) -> String { + let format = format_description!("[year][month][day]"); + let mut ans = String::from(""); + ans.push_str(&t.format(&format).unwrap().to_string()); + ans.push('/'); + ans.push_str(location); // TODO: use a `Region` type + ans.push('/'); + ans.push_str(service_type); + ans.push_str("/aws4_request"); + ans +} + +fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, service_type: &str) -> String { + let scope = get_scope(location, t, service_type); + let mut s = access_key_id.to_string(); + s.push_str("/"); + s.push_str(&scope); + s +} + +fn get_hashed_payload(req: &request::Builder) -> String { + let headers = req.headers_ref().unwrap(); + let mut hashed_payload = ""; + if let Some(payload)= headers.get("X-Amz-Content-Sha256") { + hashed_payload = payload.to_str().unwrap(); + } + if hashed_payload == "" { + hashed_payload = UNSIGNED_PAYLOAD; + } + hashed_payload.to_string() +} + +fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap) -> String { + let mut headers = Vec::::new(); + let mut vals = HashMap::>::new(); + for k in req.headers_ref().expect("err").keys() { + if ignored_headers.get(&k.to_string()).is_some() { + continue; + } + headers.push(k.as_str().to_lowercase()); + let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); + vals.insert(k.as_str().to_lowercase(), vv); + } + if !header_exists("host", &headers) { + headers.push("host".to_string()); + } + headers.sort(); + + debug!("get_canonical_headers vals: {:?}", vals); + debug!("get_canonical_headers headers: {:?}", headers); + + let mut buf = BytesMut::new(); + for k in headers { + buf.write_str(&k); + buf.write_char(':'); + let k: &str = &k; + match k { + "host" => { + buf.write_str(&get_host_addr(&req)); + buf.write_char('\n'); + } + _ => { + for (idx, v) in vals[k].iter().enumerate() { + if idx > 0 { + buf.write_char(','); + } + buf.write_str(&sign_v4_trim_all(v)); + } + buf.write_char('\n'); + } + } + } + String::from_utf8(buf.to_vec()).unwrap() +} + +fn header_exists(key: &str, headers: &[String]) -> bool { + for k in headers { + if k == key { + return true; + } + } + false +} + +fn get_signed_headers(req: &request::Builder, ignored_headers: &HashMap) -> String { + let mut headers = Vec::::new(); + let headers_ref = req.headers_ref().expect("err"); + debug!("get_signed_headers headers: {:?}", headers_ref); + for (k, _) in headers_ref { + if ignored_headers.get(&k.to_string()).is_some() { + continue; + } + headers.push(k.as_str().to_lowercase()); + } + if !header_exists("host", &headers) { + headers.push("host".to_string()); + } + headers.sort(); + headers.join(";") +} + +fn get_canonical_request(req: &request::Builder, ignored_headers: &HashMap, hashed_payload: &str) -> String { + let mut canonical_query_string = "".to_string(); + if let Some(q) = req.uri_ref().unwrap().query() { + let mut query = q.split('&').map(|h| h.to_string()).collect::>(); + query.sort(); + canonical_query_string = query.join("&"); + canonical_query_string = canonical_query_string.replace("+", "%20"); + } + + let mut canonical_request = >::new(); + canonical_request.push(req.method_ref().unwrap().to_string()); + canonical_request.push(req.uri_ref().unwrap().path().to_string()); + canonical_request.push(canonical_query_string); + canonical_request.push(get_canonical_headers(&req, ignored_headers)); + canonical_request.push(get_signed_headers(&req, ignored_headers)); + canonical_request.push(hashed_payload.to_string()); + canonical_request.join("\n") +} + +fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &str, service_type: &str) -> String { + let mut string_to_sign = SIGN_V4_ALGORITHM.to_string(); + string_to_sign.push_str("\n"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + string_to_sign.push_str(&t.format(&format).unwrap()); + string_to_sign.push_str("\n"); + string_to_sign.push_str(&get_scope(location, t, service_type)); + string_to_sign.push_str("\n"); + string_to_sign.push_str(&hex_sha256(canonical_request.as_bytes(), |s| s.to_string())); + string_to_sign +} + +pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, expires: i64, t: OffsetDateTime) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + //let t = OffsetDateTime::now_utc(); + //let date = AmzDate::parse(timestamp).unwrap(); + let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + //let credential = get_scope(location, t, SERVICE_TYPE_S3); + let credential = get_credential(access_key_id, location, t, SERVICE_TYPE_S3); + let signed_headers = get_signed_headers(&req, &v4_ignored_headers); + + let mut query = >::new(); + if let Some(q) = req.uri_ref().unwrap().query() { + let result = serde_urlencoded::from_str::>(q); + query = result.unwrap_or_default(); + } + query.push(("X-Amz-Algorithm".to_string(), SIGN_V4_ALGORITHM.to_string())); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + query.push(("X-Amz-Date".to_string(), t.format(&format).unwrap().to_string())); + query.push(("X-Amz-Expires".to_string(), format!("{:010}", expires))); + query.push(("X-Amz-SignedHeaders".to_string(), signed_headers)); + query.push(("X-Amz-Credential".to_string(), credential)); + if session_token != "" { + query.push(("X-Amz-Security-Token".to_string(), session_token.to_string())); + } + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3); + //println!("canonical_request: \n{}\n", canonical_request); + //println!("string_to_sign: \n{}\n", string_to_sign); + let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3); + let signature = get_signature(signing_key, &string_to_sign); + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}&X-Amz-Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + req +} + +fn post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_access_key: &str, location: &str) -> String { + let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3); + let signature = get_signature(signing_key, policy_base64); + signature +} + +fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Builder { + sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) +} + +fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, service_type: &str, trailer: HeaderMap) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let t = OffsetDateTime::now_utc(); + let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let mut headers = req.headers_mut().expect("err"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap()); + + if session_token != "" { + headers.insert("X-Amz-Security-Token", session_token.parse().unwrap()); + } + + if trailer.len() > 0{ + for (k, _) in &trailer { + headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); + } + + headers.insert("Content-Encoding", "aws-chunked".parse().unwrap()); + headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap()); + } + + if service_type == SERVICE_TYPE_STS { + headers.remove("X-Amz-Content-Sha256"); + } + + let hashed_payload = get_hashed_payload(&req); + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &hashed_payload); + let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, service_type); + let signing_key = get_signing_key(secret_access_key, location, t, service_type); + let credential = get_credential(access_key_id, location, t2, service_type); + let signed_headers = get_signed_headers(&req, &v4_ignored_headers); + let signature = get_signature(signing_key, &string_to_sign); + //debug!("\n\ncanonical_request: \n{}\nstring_to_sign: \n{}\nsignature: \n{}\n\n", &canonical_request, &string_to_sign, &signature); + + let mut headers = req.headers_mut().expect("err"); + + let auth = format!("{} Credential={}, SignedHeaders={}, Signature={}", SIGN_V4_ALGORITHM, credential, signed_headers, signature); + headers.insert("Authorization", auth.parse().unwrap()); + + if trailer.len() > 0 { + //req.Trailer = trailer; + for (_, v) in &trailer { + headers.append(http::header::TRAILER, v.clone()); + } + return streaming_unsigned_v4(req, &session_token, content_len, t); + } + req +} + +fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: HeaderMap) { + if trailer.len() > 0 { + return; + } + let t = OffsetDateTime::now_utc(); + let t = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let mut headers = req.headers_mut().expect("err"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap()); + + for (k, _) in &trailer { + headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); + } + + headers.insert("Content-Encoding", "aws-chunked".parse().unwrap()); + headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap()); + + if trailer.len() > 0 { + for (_, v) in &trailer { + headers.append(http::header::TRAILER, v.clone()); + } + } + streaming_unsigned_v4(req, "", content_len, t); +} + +pub fn sign_v4(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str) -> request::Builder { + sign_v4_inner(req, content_len, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, HeaderMap::new()) +} + +pub fn sign_v4_trailer(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, trailer: HeaderMap) -> request::Builder { + sign_v4_inner(req, 0, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, trailer) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn example_list_objects() { + // let access_key_id = "AKIAIOSFODNN7EXAMPLE"; + let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "examplebucket"; + let region = "us-east-1"; + let service = "s3"; + let path = "/"; + + let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/?"); + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); + headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + let mut query = >::new(); + query.push(("max-keys".to_string(), "2".to_string())); + query.push(("prefix".to_string(), "J".to_string())); + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/\n", + "max-keys=2&prefix=J\n", + "host:examplebucket.s3.amazonaws.com\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", "20130524T000000Z", "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20130524T000000Z", "\n", + "20130524/us-east-1/s3/aws4_request\n", + "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + + assert_eq!(signature, "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"); + } + + #[test] + fn example_signature() { + // let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20250505T011054Z"; + let t = datetime!(2025-05-05 01:10:54 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + + let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + let mut query: Vec<(String, String)> = Vec::new(); + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/\n", + "\n", + "host:192.168.1.11:9020\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", "20250505T011054Z", "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + println!("string_to_sign: \n{}\n", string_to_sign); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20250505T011054Z", "\n", + "20250505/us-east-1/s3/aws4_request\n", + "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + println!("signature: \n{}\n", signature); + assert_eq!(signature, "df4116595e27b0dfd1103358947d9199378cc6386c4657abd8c5f0b11ebb4931"); + } + + #[test] + fn example_signature2() { + // let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20250507T051030Z"; + let t = datetime!(2025-05-07 05:10:30 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + + let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?list-type=2&encoding-type=url&prefix=mypre&delimiter=%2F&fetch-owner=true&max-keys=1"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + /*let uri = req.uri_ref().unwrap().clone(); + println!("{:?}", uri); + let mut canonical_query_string = "".to_string(); + if let Some(q) = uri.query() { + let result = serde_urlencoded::from_str::>(q); + let mut query = result.unwrap_or_default(); + query.sort(); + canonical_query_string = query.join("&"); + canonical_query_string.replace("+", "%20"); + } + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap());*/ +println!("{:?}", req.uri_ref().unwrap().query()); + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/\n", + "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n", + "host:192.168.1.11:9020\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", "20250507T051030Z", "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + println!("string_to_sign: \n{}\n", string_to_sign); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20250507T051030Z", "\n", + "20250507/us-east-1/s3/aws4_request\n", + "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + println!("signature: \n{}\n", signature); + assert_eq!(signature, "760278c9a77d5c245ac83d85917bddc3e3b14343091e8f4ad8edbbf73107d685"); + } + + #[test] + fn example_presigned_url() { + use hyper::Uri; + + let access_key_id = "AKIAIOSFODNN7EXAMPLE"; + let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/"; + let session_token = ""; + + let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/test.txt"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); + + req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", ®ion, 86400, t); + + let mut canonical_request = req.method_ref().unwrap().as_str().to_string(); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().path()); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().query().unwrap()); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_hashed_payload(&req)); + //println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/test.txt\n", + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404\n", + "host:examplebucket.s3.amazonaws.com\n", + "\n", + "host\n", + "UNSIGNED-PAYLOAD", + ) + ); + } + + #[test] + fn example_presigned_url2() { + use hyper::Uri; + + let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + let session_token = ""; + + let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/test.txt?delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + + req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", ®ion, 86400, t); + + let mut canonical_request = req.method_ref().unwrap().as_str().to_string(); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().path()); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().query().unwrap()); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_hashed_payload(&req)); + //println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/test.txt\n", + "delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=rustfsadmin%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=e4af975e4a7e2c0449451740c7e9a425123681d2a8830bfb188789ea19618b20\n", + "host:192.168.1.11:9020\n", + "\n", + "host\n", + "UNSIGNED-PAYLOAD", + ) + ); + } +} \ No newline at end of file diff --git a/ecstore/src/signer/utils.rs b/ecstore/src/signer/utils.rs new file mode 100644 index 00000000..dd73b877 --- /dev/null +++ b/ecstore/src/signer/utils.rs @@ -0,0 +1,31 @@ +use http::request; + +pub fn get_host_addr(req: &request::Builder) -> String { + let host = req.headers_ref().expect("err").get("host"); + let uri = req.uri_ref().unwrap(); + let req_host; + if let Some(port) = uri.port() { + req_host = format!("{}:{}", uri.host().unwrap(), port); + } else { + req_host = uri.host().unwrap().to_string(); + } + if host.is_some() && req_host != host.unwrap().to_str().unwrap().to_string() { + return host.unwrap().to_str().unwrap().to_string(); + } + /*if req.uri_ref().unwrap().host().is_some() { + return req.uri_ref().unwrap().host().unwrap(); + }*/ + req_host +} + +pub fn sign_v4_trim_all(input: &str) -> String { + let ss = input.split_whitespace().collect::>(); + ss.join(" ") +} + +pub fn stable_sort_by_first(v: &mut [(T, T)]) +where + T: Ord, +{ + v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0)); +} \ No newline at end of file diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index c24f2224..a996a5d3 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,5 +1,6 @@ #![allow(clippy::map_entry)] +use rustfs_filemeta::FileInfo; use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::config::GLOBAL_StorageClass; @@ -13,7 +14,7 @@ use crate::error::{ use crate::global::{ DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd, - set_global_deployment_id, set_object_layer, + set_global_deployment_id, set_object_layer, GLOBAL_TierConfigMgr, }; use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; @@ -26,7 +27,10 @@ use crate::rebalance::RebalanceMeta; use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::{ - bucket::metadata::BucketMetadata, + bucket::{ + metadata::BucketMetadata, + lifecycle::bucket_lifecycle_ops::TransitionState + }, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk}, endpoints::EndpointServerPools, peer::S3PeerSys, @@ -40,7 +44,7 @@ use crate::{ }; use rustfs_utils::crypto::base64_decode; use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; - +use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry; use crate::error::{Error, Result}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use futures::future::join_all; @@ -324,6 +328,14 @@ impl ECStore { } } + init_background_expiry(self.clone()).await; + + TransitionState::init(self.clone()).await; + + if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { + info!("TierConfigMgr init error: {}", err); + } + Ok(()) } @@ -1836,7 +1848,6 @@ impl StorageAPI for ECStore { return self.pools[idx].new_multipart_upload(bucket, object, opts).await; } } - let idx = self.get_pool_idx(bucket, object, -1).await?; if opts.data_movement && idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( @@ -1849,6 +1860,47 @@ impl StorageAPI for ECStore { self.pools[idx].new_multipart_upload(bucket, object, opts).await } + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + let object = encode_dir_object(object); + + if self.single_pool() { + self.pools[0].add_partial(bucket, object.as_str(), version_id).await; + } + + let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()).await?; + + self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; + Ok(()) + } + #[tracing::instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let object = encode_dir_object(object); + if self.single_pool() { + return self.pools[0].transition_object(bucket, &object, opts).await; + } + + //opts.skip_decommissioned = true; + //opts.no_lock = true; + let idx = self.get_pool_idx_existing_with_opts(bucket, &object, opts).await?; + + self.pools[idx].transition_object(bucket, &object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let object = encode_dir_object(object); + if self.single_pool() { + return self.pools[0].restore_transitioned_object(bucket, &object, opts).await; + } + + //opts.skip_decommissioned = true; + //opts.nolock = true; + let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?; + + self.pools[idx].restore_transitioned_object(bucket, &object, opts).await + } + #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -2075,6 +2127,18 @@ impl StorageAPI for ECStore { self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + check_del_obj_args(bucket, object)?; + + let object = rustfs_utils::path::encode_dir_object(object); + + if self.single_pool() { + return self.pools[0].delete_object_version(bucket, object.as_str(), fi, force_del_marker).await; + } + Ok(()) + } + #[tracing::instrument(skip(self))] async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { let object = encode_dir_object(object); diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 31f801de..b26d4178 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -2,14 +2,22 @@ use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::versioning::VersioningApi as _; use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::error::{Error, Result}; +use rustfs_utils::path::decode_dir_object; use crate::heal::heal_ops::HealSequence; use crate::store_utils::clean_metadata; -use crate::{disk::DiskStore, heal::heal_commands::HealOpts}; +use crate::{disk::DiskStore, heal::heal_commands::HealOpts,}; +use crate::{ + bucket::lifecycle::{ + lifecycle::TransitionOptions, + bucket_lifecycle_ops::TransitionedObject, + }, + bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, + bucket::lifecycle::lifecycle::ExpirationOptions, +}; use http::{HeaderMap, HeaderValue}; use madmin::heal_commands::HealResultItem; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; use rustfs_rio::{HashReader, Reader}; -use rustfs_utils::path::decode_dir_object; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt::Debug; @@ -236,6 +244,10 @@ pub struct ObjectOptions { pub replication_request: bool, pub delete_marker: bool, + pub transition: TransitionOptions, + pub expiration: ExpirationOptions, + pub lifecycle_audit_event: LcAuditEvent, + pub eval_metadata: Option>, } @@ -307,6 +319,7 @@ pub struct ObjectInfo { pub data_blocks: usize, pub version_id: Option, pub delete_marker: bool, + pub transitioned_object: TransitionedObject, pub user_tags: String, pub parts: Vec, pub is_latest: bool, @@ -340,6 +353,7 @@ impl Clone for ObjectInfo { data_blocks: self.data_blocks, version_id: self.version_id, delete_marker: self.delete_marker, + transitioned_object: self.transitioned_object.clone(), user_tags: self.user_tags.clone(), parts: self.parts.clone(), is_latest: self.is_latest, @@ -430,7 +444,15 @@ impl ObjectInfo { // TODO:expires // TODO:ReplicationState - // TODO:TransitionedObject + + let transitioned_object = TransitionedObject { + name: fi.transitioned_objname.clone(), + version_id: if let Some(transition_version_id) = fi.transition_version_id { transition_version_id.to_string() } else { "".to_string() }, + status: fi.transition_status.clone(), + free_version: fi.tier_free_version(), + tier: fi.transition_tier.clone(), + }; + let metadata = { let mut v = fi.metadata.clone(); @@ -473,6 +495,7 @@ impl ObjectInfo { etag, inlined, user_defined: metadata, + transitioned_object, ..Default::default() } } @@ -746,6 +769,7 @@ pub trait StorageAPI: ObjectIO { src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result; + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>; async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; async fn delete_objects( &self, @@ -820,6 +844,9 @@ pub trait StorageAPI: ObjectIO { async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; // DecomTieredObject async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>; + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; diff --git a/ecstore/src/tier/mod.rs b/ecstore/src/tier/mod.rs new file mode 100644 index 00000000..80e99e75 --- /dev/null +++ b/ecstore/src/tier/mod.rs @@ -0,0 +1,9 @@ +pub mod warm_backend_s3; +pub mod warm_backend_minio; +pub mod warm_backend_rustfs; +pub mod warm_backend; +pub mod tier_admin; +pub mod tier_config; +pub mod tier; +pub mod tier_gen; +pub mod tier_handlers; \ No newline at end of file diff --git a/ecstore/src/tier/tier.rs b/ecstore/src/tier/tier.rs new file mode 100644 index 00000000..a5afc65b --- /dev/null +++ b/ecstore/src/tier/tier.rs @@ -0,0 +1,443 @@ +use std::{ + collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,} +}; +use bytes::Bytes; +use serde::{Serialize, Deserialize}; +use time::OffsetDateTime; +use tokio::{select, sync::RwLock, time::interval}; +use rand::Rng; +use tracing::{info, debug, warn, error}; +use http::status::StatusCode; +use lazy_static::lazy_static; +use tokio::io::BufReader; + +use s3s::S3ErrorCode; +use crate::error::{Error, Result, StorageError}; +use rustfs_utils::path::{path_join, SLASH_SEPARATOR}; +use crate::{ + config::com::{read_config, CONFIG_PREFIX}, + disk::RUSTFS_META_BUCKET, + store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI +}; +use crate::client::admin_handler_utils::AdminError; +use crate::tier::{ + warm_backend::{check_warm_backend, new_warm_backend}, + tier_handlers::{ + ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, + }, + tier_admin::TierCreds, + tier_config::{TierType, TierConfig,}, +}; +use crate::new_object_layer_fn; +use rustfs_rio::HashReader; + +use super::{ + tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND}, + warm_backend::WarmBackendImpl, +}; + +const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); + +pub const TIER_CONFIG_FILE: &str = "tier-config.json"; +pub const TIER_CONFIG_FORMAT: u16 = 1; +pub const TIER_CONFIG_V1: u16 = 1; +pub const TIER_CONFIG_VERSION: u16 = 1; + +lazy_static! { + //pub static ref TIER_CONFIG_PATH: PathBuf = path_join(&[PathBuf::from(RUSTFS_CONFIG_PREFIX), PathBuf::from(TIER_CONFIG_FILE)]); +} + +const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; + +pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError { + code: "XRustFSAdminTierMissingCredentials", + message: "Specified remote credentials are empty", + status_code: StatusCode::FORBIDDEN, +}; + +pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError { + code: "XRustFSAdminTierBackendInUse", + message: "Specified remote tier is already in use", + status_code: StatusCode::CONFLICT, +}; + +pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError { + code: "XRustFSAdminTierTypeUnsupported", + message: "Specified tier type is unsupported", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError { + code: "XRustFSAdminTierBackendNotEmpty", + message: "Specified remote backend is not empty", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError { + code: "XRustFSAdminTierInvalidConfig", + message: "Unable to setup remote tier, check tier configuration", + status_code: StatusCode::BAD_REQUEST, +}; + +#[derive(Serialize, Deserialize)] +pub struct TierConfigMgr { + #[serde(skip)] + pub driver_cache: HashMap, + pub tiers: HashMap, + pub last_refreshed_at: OffsetDateTime, +} + +impl TierConfigMgr { + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + })) + } + + pub fn unmarshal(data: &[u8]) -> std::result::Result { + let cfg: TierConfigMgr = serde_json::from_slice(data)?; + //let mut cfg = TierConfigMgr(m); + //let mut cfg = m; + Ok(cfg) + } + + pub fn marshal(&self) -> std::result::Result { + let data = serde_json::to_vec(&self)?; + + //let mut data = Vec::with_capacity(self.msg_size()+4); + let mut data = Bytes::from(data); + + //LittleEndian::write_u16(&mut data[0..2], TIER_CONFIG_FORMAT); + //LittleEndian::write_u16(&mut data[2..4], TIER_CONFIG_VERSION); + + Ok(data) + } + + pub fn refreshed_at(&self) -> OffsetDateTime { + self.last_refreshed_at + } + + pub fn is_tier_valid(&self, tier_name: &str) -> bool { + let (_, valid) = self.is_tier_name_in_use(tier_name); + valid + } + + pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) { + if let Some(t) = self.tiers.get(tier_name) { + return (t.tier_type.clone(), true); + } + (TierType::Unsupported, false) + } + + pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + let tier_name = &tier.name; + if tier_name != tier_name.to_uppercase().as_str() { + return Err(ERR_TIER_NAME_NOT_UPPERCASE); + } + + let (_, b) = self.is_tier_name_in_use(tier_name); + if b { + return Err(ERR_TIER_ALREADY_EXISTS); + } + + let d = new_warm_backend(&tier, true).await?; + + if !force { + let in_use = d.in_use().await; + match in_use { + Ok(b) => { + if b { + return Err(ERR_TIER_BACKEND_IN_USE); + } + } + Err(err) => { + warn!("tier add failed, err: {:?}", err); + if err.to_string().contains("connect") { + return Err(ERR_TIER_CONNECT_ERR); + } else if err.to_string().contains("authorization") { + return Err(ERR_TIER_INVALID_CREDENTIALS); + } else if err.to_string().contains("bucket") { + return Err(ERR_TIER_BUCKET_NOT_FOUND); + } + return Err(ERR_TIER_PERM_ERR); + } + } + } + + self.driver_cache.insert(tier_name.to_string(), d); + self.tiers.insert(tier_name.to_string(), tier); + + Ok(()) + } + + pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> { + let d = self.get_driver(tier_name).await; + if let Err(err) = d { + match err { + ERR_TIER_NOT_FOUND => { + return Ok(()); + } + _ => { + return Err(err); + } + } + } + if !force { + let inuse = d.expect("err").in_use().await; + if let Err(err) = inuse { + return Err(ERR_TIER_PERM_ERR); + } else if inuse.expect("err") { + return Err(ERR_TIER_BACKEND_NOT_EMPTY); + } + } + self.tiers.remove(tier_name); + self.driver_cache.remove(tier_name); + Ok(()) + } + + pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> { + let d = match self.get_driver(tier_name).await { + Ok(d) => d, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + if let Err(err) = check_warm_backend(Some(d)).await { + return Err(std::io::Error::other(err)); + } else { + return Ok(()); + } + } + + pub fn empty(&self) -> bool { + self.list_tiers().len() == 0 + } + + pub fn tier_type(&self, tier_name: &str) -> String { + let cfg = self.tiers.get(tier_name); + if cfg.is_none() { + return "internal".to_string(); + } + cfg.expect("err").tier_type.to_string() + } + + pub fn list_tiers(&self) -> Vec { + let mut tier_cfgs = Vec::::new(); + for (_, tier) in self.tiers.iter() { + let tier = tier.clone(); + tier_cfgs.push(tier); + } + tier_cfgs + } + + pub fn get(&self, tier_name: &str) -> Option { + for (tier_name2, tier) in self.tiers.iter() { + if tier_name == tier_name2 { + return Some(tier.clone()); + } + } + None + } + + pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> { + let (tier_type, exists) = self.is_tier_name_in_use(tier_name); + if !exists { + return Err(ERR_TIER_NOT_FOUND); + } + + let mut cfg = self.tiers[tier_name].clone(); + match tier_type { + TierType::S3 => { + let mut s3 = cfg.s3.as_mut().expect("err"); + if creds.aws_role { + s3.aws_role = true + } + if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" { + s3.aws_role_arn = creds.aws_role_arn; + s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file; + } + if creds.access_key != "" && creds.secret_key != "" { + s3.access_key = creds.access_key; + s3.secret_key = creds.secret_key; + } + } + TierType::RustFS => { + let mut rustfs = cfg.rustfs.as_mut().expect("err"); + if creds.access_key == "" || creds.secret_key == "" { + return Err(ERR_TIER_MISSING_CREDENTIALS); + } + rustfs.access_key = creds.access_key; + rustfs.secret_key = creds.secret_key; + } + TierType::MinIO => { + let mut minio = cfg.minio.as_mut().expect("err"); + if creds.access_key == "" || creds.secret_key == "" { + return Err(ERR_TIER_MISSING_CREDENTIALS); + } + minio.access_key = creds.access_key; + minio.secret_key = creds.secret_key; + } + _ => () + } + + let d = new_warm_backend(&cfg, true).await?; + self.tiers.insert(tier_name.to_string(), cfg); + self.driver_cache.insert(tier_name.to_string(), d); + Ok(()) + } + + pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { + Ok(match self.driver_cache.entry(tier_name.to_string()) { + Entry::Occupied(e) => { + e.into_mut() + } + Entry::Vacant(e) => { + let t = self.tiers.get(tier_name); + if t.is_none() { + return Err(ERR_TIER_NOT_FOUND); + } + let d = new_warm_backend(t.expect("err"), false).await?; + e.insert(d) + } + }) + } + + pub async fn reload(&mut self, api: Arc) -> std::result::Result<(), std::io::Error> { + //let Some(api) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) }; + let new_config = load_tier_config(api).await; + + match &new_config { + Ok(_c) => {} + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + } + self.driver_cache.clear(); + self.tiers.clear(); + let new_config = new_config.expect("err"); + for (tier, cfg) in new_config.tiers { + self.tiers.insert(tier, cfg); + } + self.last_refreshed_at = OffsetDateTime::now_utc(); + Ok(()) + } + + #[tracing::instrument(level = "debug", name = "tier_save", skip(self))] + pub async fn save(&self) -> std::result::Result<(), std::io::Error> { + let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) }; + //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; + + self.save_tiering_config(api).await + } + + pub async fn save_tiering_config(&self, api: Arc) -> std::result::Result<(), std::io::Error> { + let data = self.marshal()?; + + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + + self.save_config(api, &config_file, data).await + } + + pub async fn save_config(&self, api: Arc, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> { + self.save_config_with_opts( + api, + file, + data, + &ObjectOptions { + max_parity: true, + ..Default::default() + }, + ) + .await + } + + pub async fn save_config_with_opts(&self, api: Arc, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> { + debug!("save tier config:{}", file); + let _ = api + .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) + .await?; + Ok(()) + } + + pub async fn refresh_tier_config(&mut self, api: Arc) { + //let r = rand.New(rand.NewSource(time.Now().UnixNano())); + let mut rng = rand::rng(); + let r = rng.random_range(0.0..1.0); + let rand_interval = || { + Duration::from_secs((r * 60_f64).round() as u64) + }; + + let mut t = interval(TIER_CFG_REFRESH + rand_interval()); + loop { + select! { + _ = t.tick() => { + if let Err(err) = self.reload(api.clone()).await { + info!("{}", err); + } + } + else => () + } + t.reset(); + } + } + + pub async fn init(&mut self, api: Arc) -> Result<()> { + self.reload(api).await?; + //if globalIsDistErasure { + // self.refresh_tier_config(api).await; + //} + Ok(()) + } +} + +async fn new_and_save_tiering_config(api: Arc) -> Result { + let mut cfg = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; + //lookup_configs(&mut cfg, api.clone()).await; + cfg.save_tiering_config(api).await?; + + Ok(cfg) +} + +#[tracing::instrument(level = "debug")] +async fn load_tier_config(api: Arc) -> std::result::Result { + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + let data = read_config(api.clone(), config_file.as_str()).await; + if let Err(err) = data { + if is_err_config_not_found(&err) { + warn!("config not found, start to init"); + let cfg = new_and_save_tiering_config(api).await?; + return Ok(cfg); + } else { + error!("read config err {:?}", &err); + return Err(std::io::Error::other(err)); + } + } + + let cfg; + let version = 1;//LittleEndian::read_u16(&data[2..4]); + match version { + TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => { + cfg = match TierConfigMgr::unmarshal(&data.unwrap()) { + Ok(cfg) => cfg, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + } + _ => { + return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version))); + } + } + + Ok(cfg) +} + +pub fn is_err_config_not_found(err: &StorageError) -> bool { + matches!(err, StorageError::ObjectNotFound(_, _)) +} \ No newline at end of file diff --git a/ecstore/src/tier/tier_admin.rs b/ecstore/src/tier/tier_admin.rs new file mode 100644 index 00000000..ee7c2cc5 --- /dev/null +++ b/ecstore/src/tier/tier_admin.rs @@ -0,0 +1,29 @@ +use std::{ + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use rand::Rng; +use tracing::warn; +use http::status::StatusCode; +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +#[derive(Default, Debug, Clone)] +#[serde(default)] +pub struct TierCreds { + #[serde(rename = "accessKey")] + pub access_key: String, + #[serde(rename = "secretKey")] + pub secret_key: String, + + #[serde(rename = "awsRole")] + pub aws_role: bool, + #[serde(rename = "awsRoleWebIdentityTokenFile")] + pub aws_role_web_identity_token_file: String, + #[serde(rename = "awsRoleArn")] + pub aws_role_arn: String, + + //azsp: ServicePrincipalAuth, + + //#[serde(rename = "credsJson")] + pub creds_json: Vec, +} \ No newline at end of file diff --git a/ecstore/src/tier/tier_config.rs b/ecstore/src/tier/tier_config.rs new file mode 100644 index 00000000..518338ec --- /dev/null +++ b/ecstore/src/tier/tier_config.rs @@ -0,0 +1,353 @@ +use std::fmt::Display; +use serde::{Serialize, Deserialize}; +use tracing::info; + +const C_TierConfigVer: &str = "v1"; + +const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty"; +const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config"; +const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version"; +const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type"; + +#[derive(Serialize, Deserialize)] +#[derive(Default, Debug, Clone)] +pub enum TierType { + #[default] + Unsupported, + #[serde(rename = "s3")] + S3, + #[serde(rename = "azure")] + Azure, + #[serde(rename = "gcs")] + GCS, + #[serde(rename = "rustfs")] + RustFS, + #[serde(rename = "minio")] + MinIO, +} + +impl Display for TierType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TierType::S3 => { + write!(f, "S3") + } + TierType::RustFS => { + write!(f, "RustFS") + } + TierType::MinIO => { + write!(f, "MinIO") + } + _ => { + write!(f, "Unsupported") + } + } + } +} + +impl TierType { + pub fn new(sc_type: &str) -> Self { + match sc_type { + "S3" => { + TierType::S3 + } + "RustFS" => { + TierType::RustFS + } + "MinIO" => { + TierType::MinIO + } + _ => { + TierType::Unsupported + } + } + } + + pub fn to_string(&self) -> String { + match self { + TierType::S3 => { + "s3".to_string() + } + TierType::RustFS => { + "rustfs".to_string() + } + TierType::MinIO => { + "minio".to_string() + } + _ => { + "unsupported".to_string() + } + } + } +} + +#[derive(Default, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct TierConfig { + #[serde(skip)] + pub version: String, + #[serde(rename = "type")] + pub tier_type: TierType, + #[serde(skip)] + pub name: String, + #[serde(rename = "s3", skip_serializing_if = "Option::is_none")] + pub s3: Option, + //TODO: azure: Option, + //TODO: gcs: Option, + #[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")] + pub rustfs: Option, + #[serde(rename = "minio", skip_serializing_if = "Option::is_none")] + pub minio: Option, +} + +impl Clone for TierConfig { + fn clone(&self) -> TierConfig { + let mut s3 = None; + //az TierAzure + //gcs TierGCS + let mut r = None; + let mut m = None; + match self.tier_type { + TierType::S3 => { + let mut s3_ = self.s3.as_ref().expect("err").clone(); + s3_.secret_key = "REDACTED".to_string(); + s3 = Some(s3_); + } + TierType::RustFS => { + let mut r_ = self.rustfs.as_ref().expect("err").clone(); + r_.secret_key = "REDACTED".to_string(); + r = Some(r_); + } + TierType::MinIO => { + let mut m_ = self.minio.as_ref().expect("err").clone(); + m_.secret_key = "REDACTED".to_string(); + m = Some(m_); + } + _ => () + } + TierConfig { + version: self.version.clone(), + tier_type: self.tier_type.clone(), + name: self.name.clone(), + s3: s3, + //azure: az, + //gcs: gcs, + rustfs: r, + minio: m, + } + } +} + +impl TierConfig { + pub fn unmarshal(data: &[u8]) -> Result { + /*let m: HashMap> = serde_json::from_slice(data)?; + let mut cfg = TierConfig(m); + cfg.set_defaults(); + Ok(cfg)*/ + todo!(); + } + + pub fn marshal(&self) -> Result, std::io::Error> { + let data = serde_json::to_vec(&self)?; + Ok(data) + } + + fn endpoint(&self) -> String { + match self.tier_type { + TierType::S3 => { + self.s3.as_ref().expect("err").endpoint.clone() + } + TierType::RustFS => { + self.rustfs.as_ref().expect("err").endpoint.clone() + } + TierType::MinIO => { + self.minio.as_ref().expect("err").endpoint.clone() + } + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn bucket(&self) -> String { + match self.tier_type { + TierType::S3 => { + self.s3.as_ref().expect("err").bucket.clone() + } + TierType::RustFS => { + self.rustfs.as_ref().expect("err").bucket.clone() + } + TierType::MinIO => { + self.minio.as_ref().expect("err").bucket.clone() + } + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn prefix(&self) -> String { + match self.tier_type { + TierType::S3 => { + self.s3.as_ref().expect("err").prefix.clone() + } + TierType::RustFS => { + self.rustfs.as_ref().expect("err").prefix.clone() + } + TierType::MinIO => { + self.minio.as_ref().expect("err").prefix.clone() + } + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn region(&self) -> String { + match self.tier_type { + TierType::S3 => { + self.s3.as_ref().expect("err").region.clone() + } + TierType::RustFS => { + self.rustfs.as_ref().expect("err").region.clone() + } + TierType::MinIO => { + self.minio.as_ref().expect("err").region.clone() + } + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } +} + +//type S3Options = impl Fn(TierS3) -> Pin>> + Send + Sync + 'static; + +#[derive(Serialize, Deserialize)] +#[derive(Default, Debug, Clone)] +#[serde(default)] +pub struct TierS3 { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, + #[serde(rename = "storageclass")] + pub storage_class: String, + #[serde(skip)] + pub aws_role: bool, + #[serde(skip)] + pub aws_role_web_identity_token_file: String, + #[serde(skip)] + pub aws_role_arn: String, + #[serde(skip)] + pub aws_role_session_name: String, + #[serde(skip)] + pub aws_role_duration_seconds: i32, +} + +impl TierS3 { + fn new(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec) -> Result + where + F: Fn(TierS3) -> Box> + Send + Sync + 'static + { + if name == "" { + return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); + } + let sc = TierS3 { + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + endpoint: "https://s3.amazonaws.com".to_string(), + region: "".to_string(), + storage_class: "".to_string(), + ..Default::default() + }; + + for option in options { + let option = option(sc.clone()); + let option = *option; + option?; + } + + Ok(TierConfig { + version: C_TierConfigVer.to_string(), + tier_type: TierType::S3, + name: name.to_string(), + s3: Some(sc), + ..Default::default() + }) + } +} + +#[derive(Serialize, Deserialize)] +#[derive(Default, Debug, Clone)] +#[serde(default)] +pub struct TierRustFS { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, + #[serde(rename = "storageclass")] + pub storage_class: String, +} + +#[derive(Serialize, Deserialize)] +#[derive(Default, Debug, Clone)] +#[serde(default)] +pub struct TierMinIO { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, +} + +impl TierMinIO { + fn new(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec) -> Result + where + F: Fn(TierMinIO) -> Box> + Send + Sync + 'static + { + if name == "" { + return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); + } + let m = TierMinIO { + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + endpoint: endpoint.to_string(), + ..Default::default() + }; + + for option in options { + let option = option(m.clone()); + let option = *option; + option?; + } + + Ok(TierConfig { + version: C_TierConfigVer.to_string(), + tier_type: TierType::MinIO, + name: name.to_string(), + minio: Some(m), + ..Default::default() + }) + } +} diff --git a/ecstore/src/tier/tier_config_gen.rs b/ecstore/src/tier/tier_config_gen.rs new file mode 100644 index 00000000..94f96161 --- /dev/null +++ b/ecstore/src/tier/tier_config_gen.rs @@ -0,0 +1,48 @@ +use std::{fmt::Display, pin::Pin, sync::Arc}; +use tracing::info; +use common::error::{Error, Result}; +use crate::bucket::tier_config::{TierType, TierConfig,}; + +impl TierType { + fn decode_msg(&self/*, dc *msgp.Reader*/) -> Result<()> { + todo!(); + } + + fn encode_msg(&self/*, en *msgp.Writer*/) -> Result<()> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result> { + todo!(); + } + + pub fn unmarshal_msg(&self, bts: &[u8]) -> Result> { + todo!(); + } + + pub fn msg_size() -> usize { + todo!(); + } +} + +impl TierConfig { + fn decode_msg(&self, dc *msgp.Reader) -> Result<()> { + todo!(); + } + + pub fn encode_msg(&self, en *msgp.Writer) -> Result<()> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result> { + todo!(); + } + + pub fn unmarshal_msg(&self, bts: &[u8]) -> Result> { + todo!(); + } + + fn msg_size(&self) -> usize { + todo!(); + } +} diff --git a/ecstore/src/tier/tier_gen.rs b/ecstore/src/tier/tier_gen.rs new file mode 100644 index 00000000..bbb5020e --- /dev/null +++ b/ecstore/src/tier/tier_gen.rs @@ -0,0 +1,23 @@ +use crate::tier::tier::TierConfigMgr; + +impl TierConfigMgr { + fn decode_msg(/*dc *msgp.Reader*/) -> Result<(), std::io::Error> { + todo!(); + } + + fn encode_msg(/*en *msgp.Writer*/) -> Result<(), std::io::Error> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + pub fn unmarshal_msg(buf: &[u8]) -> Result { + todo!(); + } + + pub fn msg_size(&self) -> usize { + 100 + } +} diff --git a/ecstore/src/tier/tier_handlers.rs b/ecstore/src/tier/tier_handlers.rs new file mode 100644 index 00000000..491dd2ea --- /dev/null +++ b/ecstore/src/tier/tier_handlers.rs @@ -0,0 +1,51 @@ +use crate::client::admin_handler_utils::AdminError; +use tracing::warn; +use http::status::StatusCode; + +pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { + code: "XRustFSAdminTierAlreadyExists", + message: "Specified remote tier already exists", + status_code: StatusCode::CONFLICT, +}; + +pub const ERR_TIER_NOT_FOUND: AdminError = AdminError { + code: "XRustFSAdminTierNotFound", + message: "Specified remote tier was not found", + status_code: StatusCode::NOT_FOUND, +}; + +pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError { + code: "XRustFSAdminTierNameNotUpperCase", + message: "Tier name must be in uppercase", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError { + code: "XRustFSAdminTierBucketNotFound", + message: "Remote tier bucket not found", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError { + code: "XRustFSAdminTierInvalidCredentials", + message: "Invalid remote tier credentials", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError { + code: "XRustFSAdminTierReserved", + message: "Cannot use reserved tier name", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_PERM_ERR: AdminError = AdminError { + code: "TierPermErr", + message: "Tier Perm Err", + status_code: StatusCode::OK, +}; + +pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError { + code: "TierConnectErr", + message: "Tier Connect Err", + status_code: StatusCode::OK, +}; \ No newline at end of file diff --git a/ecstore/src/tier/warm_backend.rs b/ecstore/src/tier/warm_backend.rs new file mode 100644 index 00000000..7fea5f3c --- /dev/null +++ b/ecstore/src/tier/warm_backend.rs @@ -0,0 +1,97 @@ +use std::collections::HashMap; +use bytes::Bytes; + +use crate::client::{ + admin_handler_utils::AdminError, + transition_api::{ReadCloser, ReaderImpl,}, +}; +use crate::error::is_err_bucket_not_found; +use tracing::{info, warn}; +use crate::tier::{ + tier_config::{TierType, TierConfig}, + tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR}, + tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,}, + warm_backend_s3::WarmBackendS3, + warm_backend_rustfs::WarmBackendRustFS, + warm_backend_minio::WarmBackendMinIO, +}; + +pub type WarmBackendImpl = Box; + +const PROBE_OBJECT: &str = "probeobject"; + +#[derive(Default)] +pub struct WarmBackendGetOpts { + pub start_offset: i64, + pub length: i64, +} + +#[async_trait::async_trait] +pub trait WarmBackend { + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result; + async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result; + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result; + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>; + async fn in_use(&self) -> Result; +} + +pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { + let w = w.expect("err"); + let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await; + if let Err(err) = remote_version_id { + return Err(ERR_TIER_PERM_ERR); + } + + let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await; + //xhttp.DrainBody(r); + if let Err(err) = r { + //if is_err_bucket_not_found(&err) { + // return Err(ERR_TIER_BUCKET_NOT_FOUND); + //} + /*else if is_err_signature_does_not_match(err) { + return Err(ERR_TIER_MISSING_CREDENTIALS); + }*/ + //else { + return Err(ERR_TIER_PERM_ERR); + //} + } + if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await { + return Err(ERR_TIER_PERM_ERR); + }; + Ok(()) +} + +pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { + let mut d: Option = None; + match tier.tier_type { + TierType::S3 => { + let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + info!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + TierType::RustFS => { + let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + warn!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + TierType::MinIO => { + let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + warn!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + _ => { + return Err(ERR_TIER_TYPE_UNSUPPORTED); + } + } + + Ok(d.expect("err")) +} \ No newline at end of file diff --git a/ecstore/src/tier/warm_backend_minio.rs b/ecstore/src/tier/warm_backend_minio.rs new file mode 100644 index 00000000..bedb6135 --- /dev/null +++ b/ecstore/src/tier/warm_backend_minio.rs @@ -0,0 +1,129 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use tracing::warn; +use crate::client::{ + admin_handler_utils::AdminError, + transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore}, + credentials::{Credentials, SignatureType, Static, Value}, + api_put_object::PutObjectOptions, +}; +use crate::tier::{ + tier_config::TierMinIO, + warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend_s3::WarmBackendS3, +}; + + +const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; +const MAX_PARTS_COUNT: i64 = 10000; +const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; + +pub struct WarmBackendMinIO(WarmBackendS3); + +impl WarmBackendMinIO { + pub async fn new(conf: &TierMinIO, tier: &str) -> Result { + if conf.access_key == "" || conf.secret_key == "" { + return Err(std::io::Error::other("both access and secret keys are required")); + } + + if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let u = match url::Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(e) => { + return Err(std::io::Error::other(e.to_string())); + } + }; + + let creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + trailing_headers: true, + ..Default::default() + }; + let scheme = u.scheme(); + let default_port = if scheme == "https" { + 443 + } else { + 80 + }; + let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; + //client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self(WarmBackendS3 { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), + storage_class: "".to_string(), + })) + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendMinIO { + async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result { + let part_size = optimal_part_size(length)?; + let client = self.0.client.clone(); + let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }).await?; + //self.ToObjectError(err, object) + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + self.0.get(object, rv, opts).await + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + self.0.remove(object, rv).await + } + + async fn in_use(&self) -> Result { + self.0.in_use().await + } +} + +fn optimal_part_size(object_size: i64) -> Result { + let mut object_size = object_size; + if object_size == -1 { + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other("entity too large")); + } + + let configured_part_size = MIN_PART_SIZE; + let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64; + part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64; + + let part_size = part_size_flt as i64; + if part_size == 0 { + return Ok(MIN_PART_SIZE); + } + Ok(part_size) +} diff --git a/ecstore/src/tier/warm_backend_rustfs.rs b/ecstore/src/tier/warm_backend_rustfs.rs new file mode 100644 index 00000000..33f463ec --- /dev/null +++ b/ecstore/src/tier/warm_backend_rustfs.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use tracing::warn; + +use crate::client::{ + admin_handler_utils::AdminError, + transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore}, + credentials::{Credentials, SignatureType, Static, Value}, + api_put_object::PutObjectOptions, +}; +use crate::tier::{ + tier_config::TierRustFS, + warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend_s3::WarmBackendS3, +}; + +const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; +const MAX_PARTS_COUNT: i64 = 10000; +const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; + +pub struct WarmBackendRustFS(WarmBackendS3); + +impl WarmBackendRustFS { + pub async fn new(conf: &TierRustFS, tier: &str) -> Result { + if conf.access_key == "" || conf.secret_key == "" { + return Err(std::io::Error::other("both access and secret keys are required")); + } + + if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let u = match url::Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(e) => return Err(std::io::Error::other(e)), + }; + + let creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + trailing_headers: true, + ..Default::default() + }; + let scheme = u.scheme(); + let default_port = if scheme == "https" { + 443 + } else { + 80 + }; + let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; + //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self(WarmBackendS3 { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), + storage_class: "".to_string(), + })) + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendRustFS { + async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result { + let part_size = optimal_part_size(length)?; + let client = self.0.client.clone(); + let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }).await?; + //self.ToObjectError(err, object) + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + self.0.get(object, rv, opts).await + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + self.0.remove(object, rv).await + } + + async fn in_use(&self) -> Result { + self.0.in_use().await + } +} + +fn optimal_part_size(object_size: i64) -> Result { + let mut object_size = object_size; + if object_size == -1 { + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other("entity too large")); + } + + let configured_part_size = MIN_PART_SIZE; + let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64; + part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64; + + let part_size = part_size_flt as i64; + if part_size == 0 { + return Ok(MIN_PART_SIZE); + } + Ok(part_size) +} diff --git a/ecstore/src/tier/warm_backend_s3.rs b/ecstore/src/tier/warm_backend_s3.rs new file mode 100644 index 00000000..2d04da51 --- /dev/null +++ b/ecstore/src/tier/warm_backend_s3.rs @@ -0,0 +1,153 @@ +use std::collections::HashMap; +use std::sync::Arc; +use url::Url; + +use crate::error::ErrorResponse; +use crate::error::error_resp_to_object_err; +use crate::client::{ + api_get_options::GetObjectOptions, + credentials::{Credentials, Static, Value, SignatureType}, + transition_api::{ReaderImpl, ReadCloser}, + api_put_object::PutObjectOptions, + api_remove::RemoveObjectOptions, + transition_api::{Options, TransitionClient, TransitionCore,}, +}; +use rustfs_utils::path::SLASH_SEPARATOR; +use crate::tier::{ + tier_config::TierS3, + warm_backend::{WarmBackend, WarmBackendGetOpts,} +}; + +pub struct WarmBackendS3 { + pub client: Arc, + pub core: TransitionCore, + pub bucket: String, + pub prefix: String, + pub storage_class: String, +} + +impl WarmBackendS3 { + pub async fn new(conf: &TierS3, tier: &str) -> Result { + let u = match Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + + if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" { + return Err(std::io::Error::other("both the token file and the role ARN are required")); + } + else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" { + return Err(std::io::Error::other("both the access and secret keys are required")); + } + else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") { + return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file")); + } + else if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let mut creds: Credentials; + + if conf.access_key != "" && conf.secret_key != "" { + //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, ""); + creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + } + else { + return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); + } + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + ..Default::default() + }; + let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?; + //client.set_appinfo(format!("s3-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.clone().trim_matches('/').to_string(), + storage_class: conf.storage_class.clone(), + }) + } + + fn to_object_err(&self, err: ErrorResponse, params: Vec<&str>) -> std::io::Error { + let mut object = ""; + if params.len() >= 1 { + object = params.first().cloned().unwrap_or_default(); + } + + error_resp_to_object_err(err, vec![&self.bucket, &self.get_dest(object)]) + } + + pub fn get_dest(&self, object: &str) -> String { + let mut dest_obj = object.to_string(); + if self.prefix != "" { + dest_obj = format!("{}/{}", &self.prefix, object); + } + return dest_obj; + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendS3 { + async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap) -> Result { + let client = self.client.clone(); + let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions { + send_content_md5: true, + storage_class: self.storage_class.clone(), + user_metadata: meta, + ..Default::default() + }).await?; + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + let mut gopts = GetObjectOptions::default(); + + if rv != "" { + gopts.version_id = rv.to_string(); + } + if opts.start_offset >= 0 && opts.length > 0 { + if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset+opts.length-1) { + return Err(std::io::Error::other(err)); + } + } + let c = TransitionCore(Arc::clone(&self.client)); + let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?; + + Ok(r) + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + let mut ropts = RemoveObjectOptions::default(); + if rv != "" { + ropts.version_id = rv.to_string(); + } + let client = self.client.clone(); + let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await; + Err(std::io::Error::other(err.expect("err"))) + } + + async fn in_use(&self) -> Result { + let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?; + + Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) + } +} \ No newline at end of file diff --git a/reader/Cargo.toml b/reader/Cargo.toml new file mode 100644 index 00000000..6ad7c7ac --- /dev/null +++ b/reader/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "reader" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +tracing.workspace = true +s3s.workspace = true +thiserror.workspace = true +bytes.workspace = true +pin-project-lite.workspace = true +hex-simd = "0.8.0" +base64-simd = "0.8.0" +md-5.workspace = true +sha2 = { version = "0.11.0-pre.4" } +futures.workspace = true +async-trait.workspace = true +common.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/reader/src/error.rs b/reader/src/error.rs new file mode 100644 index 00000000..9c5017ee --- /dev/null +++ b/reader/src/error.rs @@ -0,0 +1,12 @@ +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ReaderError { + #[error("stream input error {0}")] + StreamInput(String), + // + #[error("etag: expected ETag {0} does not match computed ETag {1}")] + VerifyError(String, String), + #[error("Bad checksum: Want {0} does not match calculated {1}")] + ChecksumMismatch(String, String), + #[error("Bad sha256: Expected {0} does not match calculated {1}")] + SHA256Mismatch(String, String), +} diff --git a/reader/src/hasher.rs b/reader/src/hasher.rs new file mode 100644 index 00000000..6366a64c --- /dev/null +++ b/reader/src/hasher.rs @@ -0,0 +1,184 @@ +use md5::{Digest as Md5Digest, Md5}; +use sha2::{ + digest::{Reset, Update}, + Digest, Sha256 as sha_sha256, +}; +pub trait Hasher { + fn write(&mut self, bytes: &[u8]); + fn reset(&mut self); + fn sum(&mut self) -> String; + fn size(&self) -> usize; + fn block_size(&self) -> usize; +} + +#[derive(Default)] +pub enum HashType { + #[default] + Undefined, + Uuid(Uuid), + Md5(MD5), + Sha256(Sha256), +} + +impl Hasher for HashType { + fn write(&mut self, bytes: &[u8]) { + match self { + HashType::Md5(md5) => md5.write(bytes), + HashType::Sha256(sha256) => sha256.write(bytes), + HashType::Uuid(uuid) => uuid.write(bytes), + HashType::Undefined => (), + } + } + + fn reset(&mut self) { + match self { + HashType::Md5(md5) => md5.reset(), + HashType::Sha256(sha256) => sha256.reset(), + HashType::Uuid(uuid) => uuid.reset(), + HashType::Undefined => (), + } + } + + fn sum(&mut self) -> String { + match self { + HashType::Md5(md5) => md5.sum(), + HashType::Sha256(sha256) => sha256.sum(), + HashType::Uuid(uuid) => uuid.sum(), + HashType::Undefined => "".to_owned(), + } + } + + fn size(&self) -> usize { + match self { + HashType::Md5(md5) => md5.size(), + HashType::Sha256(sha256) => sha256.size(), + HashType::Uuid(uuid) => uuid.size(), + HashType::Undefined => 0, + } + } + + fn block_size(&self) -> usize { + match self { + HashType::Md5(md5) => md5.block_size(), + HashType::Sha256(sha256) => sha256.block_size(), + HashType::Uuid(uuid) => uuid.block_size(), + HashType::Undefined => 64, + } + } +} + +#[derive(Debug)] +pub struct Sha256 { + hasher: sha_sha256, +} + +impl Sha256 { + pub fn new() -> Self { + Self { + hasher: sha_sha256::new(), + } + } +} +impl Default for Sha256 { + fn default() -> Self { + Self::new() + } +} + +impl Hasher for Sha256 { + fn write(&mut self, bytes: &[u8]) { + Update::update(&mut self.hasher, bytes); + } + + fn reset(&mut self) { + Reset::reset(&mut self.hasher); + } + + fn sum(&mut self) -> String { + hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) + } + + fn size(&self) -> usize { + 32 + } + + fn block_size(&self) -> usize { + 64 + } +} + +#[derive(Debug)] +pub struct MD5 { + hasher: Md5, +} + +impl MD5 { + pub fn new() -> Self { + Self { hasher: Md5::new() } + } +} +impl Default for MD5 { + fn default() -> Self { + Self::new() + } +} + +impl Hasher for MD5 { + fn write(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + fn reset(&mut self) {} + + fn sum(&mut self) -> String { + hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) + } + + fn size(&self) -> usize { + 32 + } + + fn block_size(&self) -> usize { + 64 + } +} + +pub struct Uuid { + id: String, +} + +impl Uuid { + pub fn new(id: String) -> Self { + Self { id } + } +} + +impl Hasher for Uuid { + fn write(&mut self, _bytes: &[u8]) {} + + fn reset(&mut self) {} + + fn sum(&mut self) -> String { + self.id.clone() + } + + fn size(&self) -> usize { + self.id.len() + } + + fn block_size(&self) -> usize { + 64 + } +} + +pub fn sum_sha256_hex(data: &[u8]) -> String { + let mut hash = Sha256::new(); + hash.write(data); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) +} + +pub fn sum_md5_base64(data: &[u8]) -> String { + let mut hash = MD5::new(); + hash.write(data); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) +} \ No newline at end of file diff --git a/reader/src/lib.rs b/reader/src/lib.rs new file mode 100644 index 00000000..433caaa2 --- /dev/null +++ b/reader/src/lib.rs @@ -0,0 +1,7 @@ +pub mod error; +pub mod hasher; +pub mod reader; + +pub fn hex(data: impl AsRef<[u8]>) -> String { + hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower) +} diff --git a/reader/src/reader.rs b/reader/src/reader.rs new file mode 100644 index 00000000..2f731d52 --- /dev/null +++ b/reader/src/reader.rs @@ -0,0 +1,559 @@ +use bytes::Bytes; +use s3s::StdError; +use std::any::Any; +use std::io::Read; +use std::{collections::VecDeque, io::Cursor}; + +use std::pin::Pin; +use std::task::Poll; + +use crate::{ + error::ReaderError, + hasher::{HashType, Uuid}, +}; + +// use futures::stream::Stream; +use std::io::{Error, Result}; +use super::hasher::{Hasher, Sha256, MD5}; +use futures::Stream; + +pin_project_lite::pin_project! { + #[derive(Default)] + pub struct EtagReader { + #[pin] + inner: S, + md5: HashType, + checksum:Option, + bytes_read:usize, + } +} + +impl EtagReader { + pub fn new(inner: S, etag: Option, force_md5: Option) -> Self { + let md5 = { + if let Some(m) = force_md5 { + HashType::Uuid(Uuid::new(m)) + } else { + HashType::Md5(MD5::new()) + } + }; + Self { + inner, + md5, + checksum: etag, + bytes_read: 0, + } + } + + pub fn etag(&mut self) -> String { + self.md5.sum() + } +} + +impl Stream for EtagReader +where + S: Stream>, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let this = self.project(); + let poll = this.inner.poll_next(cx); + + if let Poll::Ready(ref res) = poll { + match res { + Some(Ok(bytes)) => { + *this.bytes_read += bytes.len(); + this.md5.write(bytes); + } + Some(Err(err)) => { + return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string()))))); + } + None => { + if let Some(etag) = this.checksum { + let got = this.md5.sum(); + if got.as_str() != etag.as_str() { + return Poll::Ready(Some(Err(Box::new(ReaderError::VerifyError(etag.to_owned(), got))))); + } + } + } + } + } + + poll + } +} + +pin_project_lite::pin_project! { + #[derive(Default)] + pub struct HashReader { + #[pin] + inner: S, + sha256: Option, + md5: Option, + md5_hex:Option, + sha256_hex:Option, + size:usize, + actual_size: usize, + bytes_read:usize, + } +} + +impl HashReader { + pub fn new(inner: S, size: usize, md5_hex: Option, sha256_hex: Option, actual_size: usize) -> Self { + let md5 = { + if md5_hex.is_some() { + Some(MD5::new()) + } else { + None + } + }; + let sha256 = { + if sha256_hex.is_some() { + Some(Sha256::new()) + } else { + None + } + }; + Self { + inner, + size, + actual_size, + md5_hex, + sha256_hex, + bytes_read: 0, + md5, + sha256, + } + } +} + +impl Stream for HashReader +where + S: Stream>, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let this = self.project(); + let poll = this.inner.poll_next(cx); + + if let Poll::Ready(ref res) = poll { + match res { + Some(Ok(bytes)) => { + *this.bytes_read += bytes.len(); + if let Some(sha) = this.sha256 { + sha.write(bytes); + } + + if let Some(md5) = this.md5 { + md5.write(bytes); + } + } + Some(Err(err)) => { + return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string()))))); + } + None => { + if let Some(hash) = this.sha256 { + if let Some(hex) = this.sha256_hex { + let got = hash.sum(); + let src = hex.as_str(); + if src != got.as_str() { + println!("sha256 err src:{},got:{}", src, got); + return Poll::Ready(Some(Err(Box::new(ReaderError::SHA256Mismatch(src.to_string(), got))))); + } + } + } + + if let Some(hash) = this.md5 { + if let Some(hex) = this.md5_hex { + let got = hash.sum(); + let src = hex.as_str(); + if src != got.as_str() { + // TODO: ERR + println!("md5 err src:{},got:{}", src, got); + return Poll::Ready(Some(Err(Box::new(ReaderError::ChecksumMismatch(src.to_string(), got))))); + } + } + } + } + } + } + + // println!("poll {:?}", poll); + + poll + } +} + +#[async_trait::async_trait] +pub trait Reader { + async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result; + async fn seek(&mut self, offset: usize) -> Result<()>; + async fn read_exact(&mut self, buf: &mut [u8]) -> Result; + async fn read_all(&mut self) -> Result> { + let mut data = Vec::new(); + + Ok(data) + } + fn as_any(&self) -> &dyn Any; +} + +#[derive(Debug)] +pub struct BufferReader { + pub inner: Cursor>, + pos: usize, +} + +impl BufferReader { + pub fn new(inner: Vec) -> Self { + Self { + inner: Cursor::new(inner), + pos: 0, + } + } +} + +#[async_trait::async_trait] +impl Reader for BufferReader { + #[tracing::instrument(level = "debug", skip(self, buf))] + async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result { + self.seek(offset).await?; + self.read_exact(buf).await + } + #[tracing::instrument(level = "debug", skip(self))] + async fn seek(&mut self, offset: usize) -> Result<()> { + if self.pos != offset { + self.inner.set_position(offset as u64); + } + + Ok(()) + } + #[tracing::instrument(level = "debug", skip(self))] + async fn read_exact(&mut self, buf: &mut [u8]) -> Result { + let bytes_read = self.inner.read_exact(buf)?; + self.pos += buf.len(); + //Ok(bytes_read) + Ok(0) + } + + async fn read_all(&mut self) -> Result> { + let mut data = Vec::new(); + self.inner.read_to_end(&mut data)?; + + Ok(data) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pin_project_lite::pin_project! { + pub struct ChunkedStream { + #[pin] + inner: S, + chuck_size: usize, + streams: VecDeque, + remaining:Vec, + } +} + +impl ChunkedStream { + pub fn new(inner: S, chuck_size: usize) -> Self { + Self { + inner, + chuck_size, + streams: VecDeque::new(), + remaining: Vec::new(), + } + } +} + +impl Stream for ChunkedStream +where + S: Stream> + Send + Sync, + // E: std::error::Error + Send + Sync, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let (items, op_items) = self.inner.size_hint(); + let this = self.project(); + + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + + let poll = this.inner.poll_next(cx); + + match poll { + Poll::Ready(res_op) => match res_op { + Some(res) => match res { + Ok(bytes) => { + let chuck_size = *this.chuck_size; + let mut bytes = bytes; + + // println!("get len {}", bytes.len()); + // 如果有剩余 + if !this.remaining.is_empty() { + let need_size = chuck_size - this.remaining.len(); + // 传入的数据大小需要补齐的大小，使用传入数据补齐 + if bytes.len() >= need_size { + let add_bytes = bytes.split_to(need_size); + this.remaining.extend_from_slice(&add_bytes); + this.streams.push_back(Bytes::from(this.remaining.clone())); + this.remaining.clear(); + } else { + // 不够，直接追加 + let need_size = bytes.len(); + let add_bytes = bytes.split_to(need_size); + this.remaining.extend_from_slice(&add_bytes); + } + } + + loop { + if bytes.len() < chuck_size { + break; + } + let chuck = bytes.split_to(chuck_size); + this.streams.push_back(chuck); + } + + if !bytes.is_empty() { + this.remaining.extend_from_slice(&bytes); + } + + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + + if items > 0 || op_items.is_some() { + return Poll::Pending; + } + + if !this.remaining.is_empty() { + let b = this.remaining.clone(); + this.remaining.clear(); + return Poll::Ready(Some(Ok(Bytes::from(b)))); + } + Poll::Ready(None) + } + Err(err) => Poll::Ready(Some(Err(err))), + }, + None => { + // println!("get empty"); + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + if !this.remaining.is_empty() { + let b = this.remaining.clone(); + this.remaining.clear(); + return Poll::Ready(Some(Ok(Bytes::from(b)))); + } + Poll::Ready(None) + } + }, + Poll::Pending => { + // println!("get Pending"); + Poll::Pending + } + } + + // if let Poll::Ready(Some(res)) = poll { + // warn!("poll res ..."); + // match res { + // Ok(bytes) => { + // let chuck_size = *this.chuck_size; + // let mut bytes = bytes; + // if this.remaining.len() > 0 { + // let need_size = chuck_size - this.remaining.len(); + // let add_bytes = bytes.split_to(need_size); + // this.remaining.extend_from_slice(&add_bytes); + // warn!("poll push_back remaining ...1"); + // this.streams.push_back(Bytes::from(this.remaining.clone())); + // this.remaining.clear(); + // } + + // loop { + // if bytes.len() < chuck_size { + // break; + // } + // let chuck = bytes.split_to(chuck_size); + // warn!("poll push_back ...1"); + // this.streams.push_back(chuck); + // } + + // warn!("poll remaining extend_from_slice...1"); + // this.remaining.extend_from_slice(&bytes); + // } + // Err(err) => return Poll::Ready(Some(Err(err))), + // } + // } + + // if let Some(b) = this.streams.pop_front() { + // warn!("poll pop_front ..."); + // return Poll::Ready(Some(Ok(b))); + // } + + // if this.remaining.len() > 0 { + // let b = this.remaining.clone(); + // this.remaining.clear(); + + // warn!("poll remaining ...1"); + // return Poll::Ready(Some(Ok(Bytes::from(b)))); + // } + // Poll::Pending + } + + fn size_hint(&self) -> (usize, Option) { + let mut items = self.streams.len(); + if !self.remaining.is_empty() { + items += 1; + } + (items, Some(items)) + } +} + +#[cfg(test)] +mod test { + + use super::*; + use futures::StreamExt; + + #[tokio::test] + async fn test_etag_reader() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 32]; // 65536 + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; + + let mut stream = futures::stream::iter(chunk_results); + + let mut hash_reader = EtagReader::new(&mut stream, None, None); + + // let chunk_size = 8; + + // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size); + + loop { + match hash_reader.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + println!("etag:{}", hash_reader.etag()); + + // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45 + // println!("md5: {:?}", hash_reader.hex()); + } + + #[tokio::test] + async fn test_hash_reader() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 32]; // 65536 + let size = data1.len() + data2.len(); + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; + + let mut stream = futures::stream::iter(chunk_results); + + let mut hash_reader = HashReader::new( + &mut stream, + size, + Some("d94c485610a7a00a574df55e45d3cc0c".to_string()), + Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()), + 0, + ); + + // let chunk_size = 8; + + // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size); + + loop { + match hash_reader.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + // BUG: borrow of moved value: `md5_stream` + + // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45 + // println!("md5: {:?}", hash_reader.hex()); + } + + #[tokio::test] + async fn test_chunked_stream() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 33]; // 65536 + let data3 = vec![4u8; 5]; // 65536 + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + let chunk3 = Bytes::from(data3); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2), Ok(chunk3)]; + + let mut stream = futures::stream::iter(chunk_results); + // let mut hash_reader = HashReader::new( + // &mut stream, + // size, + // Some("d94c485610a7a00a574df55e45d3cc0c".to_string()), + // Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()), + // 0, + // ); + + let chunk_size = 8; + + let mut etag_reader = EtagReader::new(&mut stream, None, None); + + let mut chunked_stream = ChunkedStream::new(&mut etag_reader, chunk_size); + + loop { + match chunked_stream.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + println!("etag:{}", etag_reader.etag()); + } +} diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index 88676b49..1e2caf17 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -65,6 +65,7 @@ pub mod service_account; pub mod sts; pub mod trace; pub mod user; +pub mod tier; use urlencoding::decode; #[derive(Debug, Serialize, Default)] diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs new file mode 100644 index 00000000..456a64d8 --- /dev/null +++ b/rustfs/src/admin/handlers/tier.rs @@ -0,0 +1,364 @@ +use std::str::from_utf8; + +use http::{HeaderMap, StatusCode}; +use iam::get_global_action_cred; +use matchit::Params; +use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; +use serde::Deserialize; +use serde_urlencoded::from_bytes; +use tracing::{warn, debug, info}; + +use crypto::{encrypt_data, decrypt_data}; +use crate::{ + admin::{router::Operation, utils::has_space_be}, + auth::{check_key_valid, get_session_token}, +}; +use ecstore::{ + client::admin_handler_utils::AdminError, + config::storageclass, + global::GLOBAL_TierConfigMgr, + tier::{ + tier_admin::TierCreds, + tier_config::{TierConfig, TierType}, + tier_handlers::{ + ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, ERR_TIER_CONNECT_ERR, + ERR_TIER_INVALID_CREDENTIALS, + }, + tier::{ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_BACKEND_IN_USE}, + }, +}; + +#[derive(Debug, Deserialize, Default)] +pub struct AddTierQuery { + #[serde(rename = "accessKey")] + pub access_key: Option, + pub status: Option, + pub tier: Option, + pub force: String, +} + +pub struct AddTier {} +#[async_trait::async_trait] +impl Operation for AddTier { + #[tracing::instrument(level = "debug", skip(self))] + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let mut input = req.input; + let body = match input.store_all_unlimited().await { + Ok(b) => b, + Err(e) => { + warn!("get body failed, e: {:?}", e); + return Err(s3_error!(InvalidRequest, "get body failed")); + } + }; + + let mut args: TierConfig = serde_json::from_slice(&body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?; + + match args.tier_type { + TierType::S3 => { + args.name = args.s3.clone().unwrap().name; + } + TierType::RustFS => { + args.name = args.rustfs.clone().unwrap().name; + } + TierType::MinIO => { + args.name = args.minio.clone().unwrap().name; + } + _ => (), + } + debug!("add tier args {:?}", args); + + let mut force: bool = false; + let force_str = query.force; + if force_str != "" { + force = force_str.parse().unwrap(); + } + match args.name.as_str() { + storageclass::STANDARD | storageclass::RRS => { + warn!("tier reserved name, args.name: {}", args.name); + return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); + } + &_ => () + } + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.add(args, force).await { + Err(ERR_TIER_ALREADY_EXISTS) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameAlreadyExist".into()), "tier name already exists!")); + } + Err(ERR_TIER_NAME_NOT_UPPERCASE) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameNotUppercase".into()), "tier name not uppercase!")); + } + Err(ERR_TIER_BACKEND_IN_USE) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse!".into()), "tier name backend in use!")); + } + Err(ERR_TIER_CONNECT_ERR) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierConnectError".into()), "tier connect error!")); + } + Err(ERR_TIER_INVALID_CREDENTIALS) => { + return Err(S3Error::with_message(S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), ERR_TIER_INVALID_CREDENTIALS.message)); + } + Err(e) => { + warn!("tier_config_mgr add failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier add failed")); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct EditTier {} +#[async_trait::async_trait] +impl Operation for EditTier { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + //{"accesskey":"gggggg","secretkey":"jjjjjjj"} + //{"detailedMessage":"failed to perform PUT: The Access Key Id you provided does not exist in our records.","message":"an error occurred, please try again"} + //200 OK + let mut input = req.input; + let body = match input.store_all_unlimited().await { + Ok(b) => b, + Err(e) => { + warn!("get body failed, e: {:?}", e); + return Err(s3_error!(InvalidRequest, "get body failed")); + } + }; + + let creds: TierCreds = serde_json::from_slice(&body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?; + + debug!("edit tier args {:?}", creds); + + let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.edit(&tier_name, creds).await { + Err(ERR_TIER_NOT_FOUND) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); + } + Err(ERR_TIER_MISSING_CREDENTIALS) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierMissingCredentials".into()), "tier missing credentials!")); + } + Err(e) => { + warn!("tier_config_mgr edit failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier edit failed")); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +#[derive(Debug, Deserialize, Default)] +pub struct BucketQuery { + #[serde(rename = "bucket")] + pub bucket: String, +} +pub struct ListTiers {} +#[async_trait::async_trait] +impl Operation for ListTiers { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: BucketQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + BucketQuery::default() + } + }; + + //{"items":[{"minio":{"accesskey":"minioadmin","bucket":"mblock2","endpoint":"http://192.168.1.11:9020","name":"COLDTIER","objects":"0","prefix":"mypre/","secretkey":"REDACTED","usage":"0 B","versions":"0"},"status":true,"type":"minio"}]} + let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await; + let tiers = tier_config_mgr.list_tiers(); + + let data = serde_json::to_vec(&tiers) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tiers err {}", e)))?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +pub struct RemoveTier {} +#[async_trait::async_trait] +impl Operation for RemoveTier { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + warn!("handle RemoveTier"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let sys_cred = get_global_action_cred() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; + + let mut force: bool = false; + let force_str = query.force; + if force_str != "" { + force = force_str.parse().unwrap(); + } + + let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.remove(&tier_name, force).await { + Err(ERR_TIER_NOT_FOUND) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found.")); + } + Err(ERR_TIER_BACKEND_NOT_EMPTY) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used.")); + } + Err(e) => { + warn!("tier_config_mgr remove failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier remove failed")); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct VerifyTier {} +#[async_trait::async_trait] +impl Operation for VerifyTier { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle RemoveTier"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let sys_cred = get_global_action_cred() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + tier_config_mgr.verify(&query.tier.unwrap()); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct GetTierInfo {} +#[async_trait::async_trait] +impl Operation for GetTierInfo { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle GetTierInfo"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await; + let info = tier_config_mgr.get(&query.tier.unwrap()); + + let data = serde_json::to_vec(&info) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tier err {}", e)))?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 7f25d355..05ae8e13 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -7,7 +7,7 @@ pub mod utils; use handlers::{ group, policys, pools, rebalance, service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount}, - sts, user, + sts, user, tier, }; use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler}; @@ -311,5 +311,38 @@ fn register_user_route(r: &mut S3Router) -> std::io::Result<()> AdminOperation(&policys::SetPolicyForUserOrGroup {}), )?; + // ? + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(), + AdminOperation(&tier::ListTiers {}), + )?; + // ? + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/tier-stats").as_str(), + AdminOperation(&tier::GetTierInfo {}), + )?; + // ?force=xxx + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(), + AdminOperation(&tier::RemoveTier {}), + )?; + // ?force=xxx + // body: AddOrUpdateTierReq + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(), + AdminOperation(&tier::AddTier {}), + )?; + // ? + // body: AddOrUpdateTierReq + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(), + AdminOperation(&tier::EditTier {}), + )?; + Ok(()) } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index c8f3fa2f..0385b53b 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -17,6 +17,7 @@ use chrono::Utc; use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder; use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; use datafusion::arrow::json::writer::JsonArray; +use ecstore::bucket::error::BucketMetadataError; use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; @@ -52,6 +53,7 @@ use ecstore::bucket::utils::serialize; use ecstore::cmd::bucket_replication::ReplicationStatusType; use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; +use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; use futures::pin_mut; use futures::{Stream, StreamExt}; use http::HeaderMap; @@ -93,6 +95,14 @@ use tracing::warn; use transform_stream::AsyncTryStream; use uuid::Uuid; +use ecstore::bucket::{ + lifecycle::{ + lifecycle::Lifecycle, + bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS + }, + object_lock::objectlock_sys::BucketObjectLockSys, +}; + macro_rules! try_ { ($result:expr) => { match $result { @@ -1590,12 +1600,26 @@ impl S3 for FS { .. } = req.input; - // warn!("lifecycle_configuration {:?}", &lifecycle_configuration); + let rcfg = metadata_sys::get_object_lock_config(&bucket).await; + if rcfg.is_err() { + return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockIsNotExist".into()), "bucket lock is not exist.")); + } + let rcfg = rcfg.expect("get_lifecycle_config err!").0; - // TODO: objcetLock + //info!("lifecycle_configuration: {:?}", &lifecycle_configuration); let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; + if let Err(err) = input_cfg.validate(&rcfg).await { + //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed.")); + return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), format!("{}", err.to_string()))); + } + + if let Err(err) = validate_transition_tier(&input_cfg).await { + //warn!("lifecycle_configuration add failed, err: {:?}", err); + return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), format!("{}", err.to_string()))); + } + let data = try_!(serialize(&input_cfg)); metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) .await @@ -2036,6 +2060,27 @@ impl S3 for FS { })) } + async fn get_object_attributes( + &self, + req: S3Request, + ) -> S3Result> { + let GetObjectAttributesInput { bucket, key, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + if let Err(e) = store.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()).await { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); + } + + Ok(S3Response::new(GetObjectAttributesOutput { + delete_marker: None, + object_parts: None, + ..Default::default() + })) + } + async fn put_object_acl(&self, req: S3Request) -> S3Result> { let PutObjectAclInput { bucket,