sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-16 17:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

ilm feature add

Browse Source
This commit is contained in:
likewu
2025-06-22 23:04:40 +08:00
parent 8452c11e9a
commit cc71f40a6d
93 changed files with 12534 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
.vscode/launch.json vendored
View File

@@ -20,33 +20,19 @@
}
},
"env": {
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug",
"RUSTFS_VOLUMES": "./target/volume/test{0...3}",
"RUSTFS_ADDRESS": "[::]:9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "[::]:9002",
"RUSTFS_SERVER_DOMAINS": "localhost:9000",
"RUSTFS_TLS_PATH": "./deploy/certs",
"RUSTFS_OBS_CONFIG": "./deploy/config/obs.example.toml",
"RUSTFS__OBSERVABILITY__ENDPOINT": "http://localhost:4317",
"RUSTFS__OBSERVABILITY__USE_STDOUT": "true",
"RUSTFS__OBSERVABILITY__SAMPLE_RATIO": "2.0",
"RUSTFS__OBSERVABILITY__METER_INTERVAL": "30",
"RUSTFS__OBSERVABILITY__SERVICE_NAME": "rustfs",
"RUSTFS__OBSERVABILITY__SERVICE_VERSION": "0.1.0",
"RUSTFS__OBSERVABILITY__ENVIRONMENT": "develop",
"RUSTFS__OBSERVABILITY__LOGGER_LEVEL": "info",
"RUSTFS__SINKS__FILE__ENABLED": "true",
"RUSTFS__SINKS__FILE__PATH": "./deploy/logs/rustfs.log",
"RUSTFS__SINKS__WEBHOOK__ENABLED": "false",
"RUSTFS__SINKS__WEBHOOK__ENDPOINT": "",
"RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN": "",
"RUSTFS__SINKS__KAFKA__ENABLED": "false",
"RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS": "",
"RUSTFS__SINKS__KAFKA__TOPIC": "",
"RUSTFS__LOGGER__QUEUE_CAPACITY": "10"
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug"
},
"args": [
"--access-key",
"AKEXAMPLERUSTFS",
"--secret-key",
"SKEXAMPLERUSTFS",
"--address",
"0.0.0.0:9010",
"--domain-name",
"127.0.0.1:9010",
"./target/volume/test{0...4}"
],
"cwd": "${workspaceFolder}"
},
{
@@ -86,6 +72,19 @@
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"name": "Debug executable target/debug/rustfs",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
//"stopAtEntry": false,
//"preLaunchTask": "cargo build",
"sourceLanguages": [
"rust"
],
}
]
}

58
Cargo.lock generated
View File

@@ -3612,6 +3612,7 @@ dependencies = [
name = "ecstore"
version = "0.0.1"
dependencies = [
"async-channel",
"async-trait",
"aws-sdk-s3",
"backon",
@@ -3624,18 +3625,25 @@ dependencies = [
"common",
"crc32fast",
"criterion",
"enumset",
"flatbuffers 25.2.10",
"futures",
"glob",
"hex-simd",
"highway",
"hmac 0.12.1",
"http 1.3.1",
"http-body-util",
"hyper 1.6.0",
"hyper-rustls 0.27.7",
"hyper-util",
"lazy_static",
"lock",
"madmin",
"md-5",
"netif",
"nix 0.30.1",
"num",
"num_cpus",
"once_cell",
"path-absolutize",
@@ -3644,6 +3652,7 @@ dependencies = [
"policy",
"protos",
"rand 0.9.1",
"reader",
"reed-solomon-erasure",
"reed-solomon-simd",
"regex",
@@ -3655,13 +3664,18 @@ dependencies = [
"rustfs-rio",
"rustfs-rsc",
"rustfs-utils",
"rustls 0.23.27",
"s3s",
"serde",
"serde-xml-rs 0.8.1",
"serde_json",
"sha2 0.11.0-pre.5",
"serde_urlencoded",
"sha1 0.10.6",
"sha2 0.10.9",
"shadow-rs",
"siphasher 1.0.1",
"smallvec",
"std-next",
"tempfile",
"thiserror 2.0.12",
"time",
@@ -3669,6 +3683,7 @@ dependencies = [
"tokio-stream",
"tokio-util",
"tonic",
"tower",
"tracing",
"tracing-error",
"transform-stream",
@@ -4852,6 +4867,7 @@ dependencies = [
"http 1.3.1",
"hyper 1.6.0",
"hyper-util",
"log",
"rustls 0.23.27",
"rustls-native-certs 0.8.1",
"rustls-pki-types",
@@ -7832,6 +7848,25 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "reader"
version = "0.0.1"
dependencies = [
"async-trait",
"base64-simd",
"bytes",
"common",
"futures",
"hex-simd",
"md-5",
"pin-project-lite",
"s3s",
"sha2 0.11.0-pre.5",
"thiserror 2.0.12",
"tokio",
"tracing",
]
[[package]]
name = "readme-rustdocifier"
version = "0.1.1"
@@ -8362,6 +8397,7 @@ dependencies = [
"rmp",
"rmp-serde",
"rustfs-utils",
"s3s",
"serde",
"thiserror 2.0.12",
"time",
@@ -8473,7 +8509,7 @@ dependencies = [
"regex",
"reqwest",
"serde",
"serde-xml-rs",
"serde-xml-rs 0.6.0",
"sha2 0.10.9",
"urlencoding",
]
@@ -8484,9 +8520,13 @@ version = "0.0.1"
dependencies = [
"base64-simd",
"blake3",
"common",
"crc32fast",
"hex-simd",
"highway",
"hmac 0.12.1",
"hyper 1.6.0",
"hyper-util",
"lazy_static",
"local-ip-address",
"md-5",
@@ -8497,7 +8537,9 @@ dependencies = [
"rustls 0.23.27",
"rustls-pemfile 2.2.0",
"rustls-pki-types",
"s3s",
"serde",
"sha1 0.10.6",
"sha2 0.10.9",
"siphasher 1.0.1",
"tempfile",
@@ -8908,6 +8950,18 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "serde-xml-rs"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53630160a98edebde0123eb4dfd0fce6adff091b2305db3154a9e920206eb510"
dependencies = [
"log",
"serde",
"thiserror 1.0.69",
"xml-rs",
]
[[package]]
name = "serde_derive"
version = "1.0.219"

9
Cargo.toml
View File

@@ -21,7 +21,7 @@ members = [
"crates/zip",
"crates/filemeta",
"crates/rio",
"reader",
]
resolver = "2"
@@ -61,6 +61,7 @@ rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" }
rustfs-disk = { path = "crates/disk", version = "0.0.1" }
rustfs-error = { path = "crates/error", version = "0.0.1" }
workers = { path = "./common/workers", version = "0.0.1" }
reader = { path = "./reader", version = "0.0.1" }
aes-gcm = { version = "0.10.3", features = ["std"] }
arc-swap = "1.7.1"
argon2 = { version = "0.5.3", features = ["std"] }
@@ -105,6 +106,7 @@ hyper-util = { version = "0.1.14", features = [
"server-auto",
"server-graceful",
] }
hyper-rustls = "0.27.5"
http = "1.3.1"
http-body = "1.0.1"
humantime = "2.2.0"
@@ -189,9 +191,13 @@ scopeguard = "1.2.0"
shadow-rs = { version = "1.1.1", default-features = false }
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
serde-xml-rs = "0.8.1"
serde_urlencoded = "0.7.1"
serde_with = "3.12.0"
sha1 = "0.10.6"
sha2 = "0.10.9"
hmac = "0.12.1"
std-next = "0.1.8"
siphasher = "1.0.1"
smallvec = { version = "1.15.1", features = ["serde"] }
snafu = "0.8.6"
@@ -215,6 +221,7 @@ tokio-rustls = { version = "0.26.2", default-features = false }
tokio-stream = { version = "0.1.17" }
tokio-tar = "0.3.1"
tokio-util = { version = "0.7.15", features = ["io", "compat"] }
async-channel = "2.3.1"
tower = { version = "0.5.2", features = ["timeout"] }
tower-http = { version = "0.6.6", features = ["cors"] }
tracing = "0.1.41"

2
common/lock/src/local_locker.rs
View File

@@ -7,7 +7,7 @@ use std::{
use crate::{Locker, lock_args::LockArgs};
const MAX_DELETE_LIST: usize = 1000;
pub const MAX_DELETE_LIST: usize = 1000;
#[derive(Clone, Debug)]
struct LockRequesterInfo {

2
crates/filemeta/Cargo.toml
View File

@@ -21,6 +21,8 @@ byteorder = "1.5.0"
tracing.workspace = true
thiserror.workspace = true
s3s.workspace = true
[dev-dependencies]
criterion = { version = "0.5", features = ["html_reports"] }

47
crates/filemeta/src/fileinfo.rs
View File

@@ -18,6 +18,10 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
pub const NULL_VERSION_ID: &str = "null";
// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
pub const TIER_FV_ID: &str = "tier-free-versionID";
pub const TIER_FV_MARKER: &str = "tier-free-marker";
pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
pub struct ObjectPartInfo {
pub etag: String,
@@ -147,11 +151,10 @@ pub struct FileInfo {
pub version_id: Option<Uuid>,
pub is_latest: bool,
pub deleted: bool,
// Transition related fields
pub transition_status: Option<String>,
pub transitioned_obj_name: Option<String>,
pub transition_tier: Option<String>,
pub transition_version_id: Option<String>,
pub transition_status: String,
pub transitioned_objname: String,
pub transition_tier: String,
pub transition_version_id: Option<Uuid>,
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
@@ -220,7 +223,11 @@ impl FileInfo {
}
pub fn get_etag(&self) -> Option<String> {
self.metadata.get("etag").cloned()
if let Some(meta) = self.metadata.get("etag") {
Some(meta.clone())
} else {
None
}
}
pub fn write_quorum(&self, quorum: usize) -> usize {
@@ -301,6 +308,30 @@ impl FileInfo {
self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
}
pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string());
}
pub fn tier_free_version_id(&self) -> String {
self.metadata[&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID)].clone()
}
pub fn set_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string());
}
pub fn set_skip_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string());
}
pub fn skip_tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID))
}
pub fn tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER))
}
pub fn set_inline_data(&mut self) {
self.metadata
.insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
@@ -319,7 +350,7 @@ impl FileInfo {
/// Check if the object is remote (transitioned to another tier)
pub fn is_remote(&self) -> bool {
!self.transition_tier.as_ref().is_none_or(|s| s.is_empty())
!self.transition_tier.is_empty()
}
/// Get the data directory for this object
@@ -375,7 +406,7 @@ impl FileInfo {
pub fn transition_info_equals(&self, other: &FileInfo) -> bool {
self.transition_status == other.transition_status
&& self.transition_tier == other.transition_tier
&& self.transitioned_obj_name == other.transitioned_obj_name
&& self.transitioned_objname == other.transitioned_objname
&& self.transition_version_id == other.transition_version_id
}

138
crates/filemeta/src/filemeta.rs
View File

@@ -7,6 +7,7 @@ use crate::headers::{
};
use byteorder::ByteOrder;
use rmp::Marker;
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::hash::Hasher;
@@ -36,6 +37,19 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2;
const META_DATA_READ_DEFAULT: usize = 4 << 10;
const MSGP_UINT32_SIZE: usize = 5;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
pub const FREE_VERSION: &str = "free-version";
pub const TRANSITION_STATUS: &str = "transition-status";
pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const TRANSITION_TIER: &str = "transition-tier";
const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
// type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>;
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -466,6 +480,40 @@ impl FileMeta {
Err(Error::other("add_version failed"))
}
pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
let mod_time = ver.get_mod_time().unwrap().nanosecond();
if !ver.valid() {
return Err(Error::other("attempted to add invalid version"));
}
let encoded = ver.marshal_msg()?;
if self.versions.len()+1 > 100 {
return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object"));
}
self.versions.push(FileMetaShallowVersion {
header: FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
..Default::default()
},
..Default::default()
});
let len = self.versions.len();
for (i, existing) in self.versions.iter().enumerate() {
if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
let vers = self.versions[i..len-1].to_vec();
self.versions[i+1..].clone_from_slice(vers.as_slice());
self.versions[i] = FileMetaShallowVersion {
header: ver.header(),
meta: encoded,
};
return Ok(());
}
}
Err(Error::other("addVersion: Internal error, unable to add version"))
}
// delete_version deletes version, returns data_dir
pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
let mut ventry = FileMetaVersion::default();
@@ -501,6 +549,42 @@ impl FileMeta {
}
}
for (i, version) in self.versions.iter().enumerate() {
if version.header.version_type != VersionType::Object || version.header.version_id != fi.version_id {
continue;
}
let mut ver = self.get_idx(i)?;
if fi.expire_restored {
ver.object.as_mut().unwrap().remove_restore_hdrs();
let _ = self.set_idx(i, ver.clone());
} else if fi.transition_status == TRANSITION_COMPLETE {
ver.object.as_mut().unwrap().set_transition(fi);
ver.object.as_mut().unwrap().reset_inline_data();
self.set_idx(i, ver.clone())?;
} else {
let vers = self.versions[i+1..].to_vec();
self.versions.extend(vers.iter().cloned());
let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi);
if to_free {
self.add_version_filemata(free_version)?;
}
}
if fi.deleted {
self.add_version_filemata(ventry)?;
}
if self.shared_data_dir_count(ver.object.as_ref().unwrap().version_id, ver.object.as_ref().unwrap().data_dir) > 0 {
return Ok(None);
}
return Ok(ver.object.as_ref().unwrap().data_dir);
}
if fi.deleted {
self.add_version_filemata(ventry)?;
}
Err(Error::FileVersionNotFound)
}
@@ -1842,10 +1926,17 @@ impl MetaObject {
}
}
/// Set transition metadata
pub fn set_transition(&mut self, _fi: &FileInfo) {
// Implementation for object lifecycle transitions
// This would handle storage class transitions
pub fn set_transition(&mut self, fi: &FileInfo) {
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec());
}
pub fn remove_restore_hdrs(&mut self) {
self.meta_user.remove(X_AMZ_RESTORE.as_str());
self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS);
self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE);
}
pub fn uses_data_dir(&self) -> bool {
@@ -1881,6 +1972,45 @@ impl MetaObject {
let bytes = hash.to_le_bytes();
[bytes[0], bytes[1], bytes[2], bytes[3]]
}
pub fn init_free_version(&self, fi: &FileInfo) -> (FileMetaVersion, bool) {
if fi.skip_tier_free_version() {
return (FileMetaVersion::default(), false);
}
if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) {
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
let vid = Uuid::parse_str(&fi.tier_free_version_id());
if let Err(err) = vid {
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string());
}
let vid = vid.unwrap();
let mut free_entry = FileMetaVersion {
version_type: VersionType::Delete,
write_version: 0,
..Default::default()
};
free_entry.delete_marker = Some(MetaDeleteMarker {
version_id: Some(vid),
mod_time: self.mod_time,
meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
});
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]);
let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER);
let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME);
let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID);
let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
for (k, v) in &self.meta_sys {
if aa.contains(&k) {
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone());
}
}
return (free_entry, true);
}
}
(FileMetaVersion::default(), false)
}
}
impl From<FileInfo> for MetaObject {

2
crates/filemeta/src/lib.rs
View File

@@ -1,5 +1,5 @@
mod error;
mod fileinfo;
pub mod fileinfo;
mod filemeta;
mod filemeta_inline;
pub mod headers;

8
crates/utils/Cargo.toml
View File

@@ -23,12 +23,18 @@ rustls = { workspace = true, optional = true }
rustls-pemfile = { workspace = true, optional = true }
rustls-pki-types = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
sha2 = { workspace = true, optional = true }
siphasher = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["io-util", "macros"] }
tracing = { workspace = true }
url = { workspace = true , optional = true}
hyper.workspace = true
hyper-util.workspace = true
common.workspace = true
sha1 = { workspace = true }
sha2 = { workspace = true, optional = true }
hmac.workspace = true
s3s.workspace = true
[dev-dependencies]

79
crates/utils/src/crypto.rs
View File

@@ -1,3 +1,8 @@
use std::mem::MaybeUninit;
use hex_simd::{AsOut, AsciiCase};
use hyper::body::Bytes;
pub fn base64_encode(input: &[u8]) -> String {
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
}
@@ -24,6 +29,80 @@ pub fn hex(data: impl AsRef<[u8]>) -> String {
// h.finish().unwrap()
// }
/// verify sha256 checksum string
pub fn is_sha256_checksum(s: &str) -> bool {
// TODO: optimize
let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f');
s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex)
}
/// `hmac_sha1(key, data)`
pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
use hmac::{Hmac, Mac};
use sha1::Sha1;
let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).unwrap();
m.update(data.as_ref());
m.finalize().into_bytes().into()
}
/// `hmac_sha256(key, data)`
pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
use hmac::{Hmac, Mac};
use sha2::Sha256;
let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).unwrap();
m.update(data.as_ref());
m.finalize().into_bytes().into()
}
/// `f(hex(src))`
fn hex_bytes32<R>(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R {
let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64];
let ans = hex_simd::encode_as_str(src.as_ref(), buf.as_out(), AsciiCase::Lower);
f(ans)
}
#[cfg(not(all(feature = "openssl", not(windows))))]
fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> {
use sha2::{Digest, Sha256};
<Sha256 as Digest>::digest(data)
}
#[cfg(all(feature = "openssl", not(windows)))]
fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
use openssl::hash::{Hasher, MessageDigest};
let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
h.update(data).unwrap();
h.finish().unwrap()
}
#[cfg(not(all(feature = "openssl", not(windows))))]
fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> {
use sha2::{Digest, Sha256};
let mut h = <Sha256 as Digest>::new();
chunk.iter().for_each(|data| h.update(data));
h.finalize()
}
#[cfg(all(feature = "openssl", not(windows)))]
fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8]> {
use openssl::hash::{Hasher, MessageDigest};
let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
chunk.iter().for_each(|data| h.update(data).unwrap());
h.finish().unwrap()
}
/// `f(hex(sha256(data)))`
pub fn hex_sha256<R>(data: &[u8], f: impl FnOnce(&str) -> R) -> R {
hex_bytes32(sha256(data).as_ref(), f)
}
/// `f(hex(sha256(chunk)))`
pub fn hex_sha256_chunk<R>(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R {
hex_bytes32(sha256_chunk(chunk).as_ref(), f)
}
#[test]
fn test_base64_encoding_decoding() {
let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000";

2
crates/utils/src/hash.rs
View File

@@ -61,6 +61,8 @@ impl HashAlgorithm {
use crc32fast::Hasher;
use siphasher::sip::SipHasher;
pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
// 你的密钥，必须是 16 字节

2
crates/utils/src/lib.rs
View File

@@ -7,6 +7,8 @@ pub mod net;
#[cfg(feature = "net")]
pub use net::*;
pub mod retry;
#[cfg(feature = "io")]
pub mod io;

111
crates/utils/src/net.rs
View File

@@ -1,11 +1,17 @@
use hyper::client::conn::http2::Builder;
use hyper_util::rt::TokioExecutor;
use lazy_static::lazy_static;
use std::{
collections::HashSet,
collections::{HashMap, HashSet},
fmt::Display,
net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
};
use url::{Host, Url};
//use hyper::{client::conn::http2::Builder, rt::Executor};
//use tonic::{SharedExec, UserAgent};
//use hyper_util::rt::TokioTimer;
use url::Host;
use s3s::header::X_AMZ_STORAGE_CLASS;
lazy_static! {
static ref LOCAL_IPS: Vec<IpAddr> = must_get_local_ips().unwrap();
@@ -105,6 +111,107 @@ pub fn must_get_local_ips() -> std::io::Result<Vec<IpAddr>> {
}
}
pub fn get_default_location(u: Url, region_override: &str) -> String {
todo!();
}
pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Error> {
let mut scheme = "https";
if !secure {
scheme = "http";
}
let endpoint_url_str = format!("{scheme}://{endpoint}");
let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { return Err(std::io::Error::other("url parse error.")); };
//is_valid_endpoint_url(endpoint_url)?;
Ok(endpoint_url)
}
pub const DEFAULT_DIAL_TIMEOUT: i64 = 5;
pub fn new_remotetarget_http_transport(insecure: bool) -> Builder<TokioExecutor> {
todo!();
}
lazy_static! {
static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("attributes".to_string(), true);
m.insert("partNumber".to_string(), true);
m.insert("versionId".to_string(), true);
m.insert("response-cache-control".to_string(), true);
m.insert("response-content-disposition".to_string(), true);
m.insert("response-content-encoding".to_string(), true);
m.insert("response-content-language".to_string(), true);
m.insert("response-content-type".to_string(), true);
m.insert("response-expires".to_string(), true);
m
};
static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("content-type".to_string(), true);
m.insert("cache-control".to_string(), true);
m.insert("content-encoding".to_string(), true);
m.insert("content-disposition".to_string(), true);
m.insert("content-language".to_string(), true);
m.insert("x-amz-website-redirect-location".to_string(), true);
m.insert("x-amz-object-lock-mode".to_string(), true);
m.insert("x-amz-metadata-directive".to_string(), true);
m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
m.insert("expires".to_string(), true);
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
SUPPORTED_QUERY_VALUES[qs_key]
}
const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-";
pub fn is_custom_query_value(qs_key: &str) -> bool {
qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX)
}
pub fn is_storageclass_header(header_key: &str) -> bool {
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
}
pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-")
}
pub fn is_rustfs_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-rustfs-")
}
pub fn is_rustfs_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-rustfs-")
}
#[derive(Debug, Clone)]
pub struct XHost {
pub name: String,

5
crates/utils/src/path.rs
View File

@@ -253,6 +253,11 @@ pub fn dir(path: &str) -> String {
let (a, _) = split(path);
clean(a)
}
pub fn trim_etag(etag: &str) -> String {
etag.trim_matches('"').to_string()
}
#[cfg(test)]
mod tests {
use super::*;

179
crates/utils/src/retry.rs Normal file
View File

@@ -0,0 +1,179 @@
//use tokio_stream::Stream;
use std::future::Future;
use std::time::{Duration, Instant};
use std::pin::Pin;
use std::task::{Context, Poll};
// MaxRetry is the maximum number of retries before stopping.
pub const MAX_RETRY: i64 = 10;
// MaxJitter will randomize over the full exponential backoff time
pub const MAX_JITTER: f64 = 1.0;
// NoJitter disables the use of jitter for randomizing the exponential backoff time
pub const NO_JITTER: f64 = 0.0;
// DefaultRetryUnit - default unit multiplicative per retry.
// defaults to 200 * time.Millisecond
//const DefaultRetryUnit = 200 * time.Millisecond;
// DefaultRetryCap - Each retry attempt never waits no longer than
// this maximum time duration.
//const DefaultRetryCap = time.Second;
/*
struct Delay {
when: Instant,
}
impl Future for Delay {
type Output = &'static str;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>)
-> Poll<&'static str>
{
if Instant::now() >= self.when {
println!("Hello world");
Poll::Ready("done")
} else {
// Ignore this line for now.
cx.waker().wake_by_ref();
Poll::Pending
}
}
}
struct RetryTimer {
rem: usize,
delay: Delay,
}
impl RetryTimer {
fn new() -> Self {
Self {
rem: 3,
delay: Delay { when: Instant::now() }
}
}
}
impl Stream for RetryTimer {
type Item = ();
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>)
-> Poll<Option<()>>
{
if self.rem == 0 {
// No more delays
return Poll::Ready(None);
}
match Pin::new(&mut self.delay).poll(cx) {
Poll::Ready(_) => {
let when = self.delay.when + Duration::from_millis(10);
self.delay = Delay { when };
self.rem -= 1;
Poll::Ready(Some(()))
}
Poll::Pending => Poll::Pending,
}
}
}*/
pub fn new_retry_timer(max_retry: i32, base_sleep: Duration, max_sleep: Duration, jitter: f64) -> Vec<i32> {
/*attemptCh := make(chan int)
exponentialBackoffWait := func(attempt int) time.Duration {
// normalize jitter to the range [0, 1.0]
if jitter < NoJitter {
jitter = NoJitter
}
if jitter > MaxJitter {
jitter = MaxJitter
}
// sleep = random_between(0, min(maxSleep, base * 2 ** attempt))
sleep := baseSleep * time.Duration(1<<uint(attempt))
if sleep > maxSleep {
sleep = maxSleep
}
if jitter != NoJitter {
sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
}
return sleep
}
go func() {
defer close(attemptCh)
for i := 0; i < maxRetry; i++ {
select {
case attemptCh <- i + 1:
case <-ctx.Done():
return
}
select {
case <-time.After(exponentialBackoffWait(i)):
case <-ctx.Done():
return
}
}
}()
return attemptCh*/
todo!();
}
/*var retryableS3Codes = map[string]struct{}{
"RequestError": {},
"RequestTimeout": {},
"Throttling": {},
"ThrottlingException": {},
"RequestLimitExceeded": {},
"RequestThrottled": {},
"InternalError": {},
"ExpiredToken": {},
"ExpiredTokenException": {},
"SlowDown": {},
}
fn isS3CodeRetryable(s3Code string) (ok bool) {
_, ok = retryableS3Codes[s3Code]
return ok
}
var retryableHTTPStatusCodes = map[int]struct{}{
http.StatusRequestTimeout: {},
429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
499: {}, // client closed request, retry. A non-standard status code introduced by nginx.
http.StatusInternalServerError: {},
http.StatusBadGateway: {},
http.StatusServiceUnavailable: {},
http.StatusGatewayTimeout: {},
520: {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected.
// Add more HTTP status codes here.
}
fn isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
_, ok = retryableHTTPStatusCodes[httpStatusCode]
return ok
}
fn isRequestErrorRetryable(ctx context.Context, err error) bool {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// Retry if internal timeout in the HTTP call.
return ctx.Err() == nil
}
if ue, ok := err.(*url.Error); ok {
e := ue.Unwrap()
switch e.(type) {
// x509: certificate signed by unknown authority
case x509.UnknownAuthorityError:
return false
}
switch e.Error() {
case "http: server gave HTTP response to HTTPS client":
return false
}
}
return true
}*/

17
ecstore/Cargo.toml
View File

@@ -34,9 +34,12 @@ serde.workspace = true
time.workspace = true
bytesize.workspace = true
serde_json.workspace = true
serde-xml-rs.workspace = true
serde_urlencoded.workspace = true
tracing-error.workspace = true
s3s.workspace = true
http.workspace = true
http-body-util = "0.1.1"
highway = { workspace = true }
url.workspace = true
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
@@ -56,14 +59,25 @@ tokio-util = { workspace = true, features = ["io", "compat"] }
crc32fast = { workspace = true }
siphasher = { workspace = true }
base64-simd = { workspace = true }
sha2 = { version = "0.11.0-pre.4" }
sha1 = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
path-clean = { workspace = true }
tempfile.workspace = true
hyper.workspace = true
hyper-util.workspace = true
hyper-rustls.workspace = true
rustls.workspace = true
tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
tokio-stream = { workspace = true }
tonic.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
tower.workspace = true
async-channel.workspace = true
num = "0.4.3"
enumset = "1.1.5"
hmac.workspace = true
std-next.workspace = true
num_cpus = { workspace = true }
rand.workspace = true
pin-project-lite.workspace = true
@@ -80,6 +94,7 @@ shadow-rs.workspace = true
rustfs-filemeta.workspace = true
rustfs-utils ={workspace = true, features=["full"]}
rustfs-rio.workspace = true
reader = { workspace = true }
[target.'cfg(not(windows))'.dependencies]
nix = { workspace = true }

2
ecstore/src/admin_server_info.rs
View File

@@ -23,7 +23,7 @@ use protos::{
};
use std::{
collections::{HashMap, HashSet},
time::SystemTime,
time::{SystemTime, UNIX_EPOCH},
};
use time::OffsetDateTime;
use tonic::Request;

32
ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs Normal file
View File

@@ -0,0 +1,32 @@
use super::lifecycle;
#[derive(Debug, Clone, Default)]
pub enum LcEventSrc {
#[default]
None,
Heal,
Scanner,
Decom,
Rebal,
S3HeadObject,
S3GetObject,
S3ListObjects,
S3PutObject,
S3CopyObject,
S3CompleteMultipartUpload,
}
#[derive(Clone, Debug, Default)]
pub struct LcAuditEvent {
pub event: lifecycle::Event,
pub source: LcEventSrc,
}
impl LcAuditEvent {
pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
Self {
event,
source,
}
}
}

815
ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs Normal file
View File

@@ -0,0 +1,815 @@
use std::any::{Any, TypeId};
use std::env;
use std::io::{Cursor, Write};
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, AtomicI64, Ordering};
use std::sync::{Arc, Mutex};
use futures::Future;
use lazy_static::lazy_static;
use s3s::Body;
use std::collections::HashMap;
use tracing::{error, info, warn};
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use uuid::Uuid;
use http::HeaderMap;
use tokio::select;
use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{mpsc, RwLock};
use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender};
use s3s::dto::BucketLifecycleConfiguration;
use crate::error::Error;
use crate::event::name::EventName;
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,};
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::client::object_api_utils::{new_getobjectreader,};
use crate::event_notification::{send_event, EventArgs};
use crate::heal::{
data_scanner_metric::ScannerMetrics,
data_scanner::{
apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects,
},
data_usage_cache::TierStats,
};
use crate::global::GLOBAL_LocalNodeName;
use crate::bucket::{
metadata_sys::get_lifecycle_config,
versioning_sys::BucketVersioningSys,
};
use crate::tier::warm_backend::WarmBackendGetOpts;
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats};
use super::tier_sweeper::{delete_object_from_remote_tier, Jentry};
use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent};
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn = Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
static XXHASH_SEED: u64 = 0;
const DISABLED: &str = "Disabled";
//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class.";
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
lazy_static! {
pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
pub static ref GLOBAL_TransitionState: Arc<TransitionState> = TransitionState::new();
}
pub struct LifecycleSys;
impl LifecycleSys {
pub fn new() -> Arc<Self> {
Arc::new(Self)
}
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
Some(lc)
}
pub fn trace(oi: &ObjectInfo) -> TraceFn
{
todo!();
}
}
struct ExpiryTask {
obj_info: ObjectInfo,
event: lifecycle::Event,
src: LcEventSrc,
}
impl ExpiryOp for ExpiryTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes());
hasher.flush();
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
}
fn as_any(&self) -> &dyn Any {
self
}
}
struct ExpiryStats {
missed_expiry_tasks: AtomicI64,
missed_freevers_tasks: AtomicI64,
missed_tier_journal_tasks: AtomicI64,
workers: AtomicI64,
}
impl ExpiryStats {
pub fn missed_tasks(&self) -> i64 {
self.missed_expiry_tasks.load(Ordering::SeqCst)
}
fn missed_free_vers_tasks(&self) -> i64 {
self.missed_freevers_tasks.load(Ordering::SeqCst)
}
fn missed_tier_journal_tasks(&self) -> i64 {
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
}
fn num_workers(&self) -> i64 {
self.workers.load(Ordering::SeqCst)
}
}
pub trait ExpiryOp: 'static {
fn op_hash(&self) -> u64;
fn as_any(&self) -> &dyn Any;
}
#[derive(Debug, Default, Clone)]
pub struct TransitionedObject {
pub name: String,
pub version_id: String,
pub tier: String,
pub free_version: bool,
pub status: String,
}
struct FreeVersionTask(ObjectInfo);
impl ExpiryOp for FreeVersionTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes());
let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes());
hasher.flush();
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
}
fn as_any(&self) -> &dyn Any {
self
}
}
struct NewerNoncurrentTask {
bucket: String,
versions: Vec<ObjectToDelete>,
event: lifecycle::Event,
}
impl ExpiryOp for NewerNoncurrentTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}", self.bucket).as_bytes());
let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes());
hasher.flush();
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
}
fn as_any(&self) -> &dyn Any {
self
}
}
pub struct ExpiryState {
tasks_tx: Vec<Sender<Option<ExpiryOpType>>>,
tasks_rx: Vec<Arc<tokio::sync::Mutex<Receiver<Option<ExpiryOpType>>>>>,
stats: Option<ExpiryStats>,
}
impl ExpiryState {
#[allow(clippy::new_ret_no_self)]
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
tasks_tx: vec![],
tasks_rx: vec![],
stats: Some(ExpiryStats {
missed_expiry_tasks: AtomicI64::new(0),
missed_freevers_tasks: AtomicI64::new(0),
missed_tier_journal_tasks: AtomicI64::new(0),
workers: AtomicI64::new(0),
}),
}))
}
pub async fn pending_tasks(&self) -> usize {
let rxs = &self.tasks_rx;
if rxs.len() == 0 {
return 0;
}
let mut tasks=0;
for rx in rxs.iter() {
tasks += rx.lock().await.len();
}
tasks
}
pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
let wrkr = self.get_worker_ch(je.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
}
let wrkr = wrkr.expect("err");
select! {
//_ -> GlobalContext.Done() => ()
_ = wrkr.send(Some(Box::new(je.clone()))) => (),
else => {
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
}
}
return Ok(());
}
pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
let task = FreeVersionTask(oi);
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
}
}
}
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
}
}
}
pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
if versions.len() == 0 {
return;
}
let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
}
}
}
pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
if self.tasks_tx.len() == 0 {
return None;
}
Some(self.tasks_tx[h as usize %self.tasks_tx.len()].clone())
}
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
return;
}
let mut state = GLOBAL_ExpiryState.write().await;
while state.tasks_tx.len() < n {
let (tx, mut rx) = mpsc::channel(10000);
let api = api.clone();
let rx = Arc::new(tokio::sync::Mutex::new(rx));
state.tasks_tx.push(tx);
state.tasks_rx.push(rx.clone());
*state.stats.as_mut().expect("err").workers.get_mut() += 1;
tokio::spawn(async move {
let mut rx = rx.lock().await;
//let mut expiry_state = GLOBAL_ExpiryState.read().await;
ExpiryState::worker(&mut *rx, api).await;
});
}
let mut l = state.tasks_tx.len();
while l > n {
let worker = state.tasks_tx[l-1].clone();
worker.send(None).await.unwrap_or(());
state.tasks_tx.remove(l-1);
state.tasks_rx.remove(l-1);
*state.stats.as_mut().expect("err").workers.get_mut() -= 1;
l -= 1;
}
}
pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
loop {
select! {
_ = tokio::signal::ctrl_c() => {
info!("got ctrl+c, exits");
break;
}
v = rx.recv() => {
if v.is_none() {
break;
}
let v = v.expect("err!");
if v.is_none() {
//rx.close();
//drop(rx);
let _ = rx;
return;
}
let v = v.expect("err!");
if v.as_any().is::<ExpiryTask>() {
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
if v.obj_info.transitioned_object.status != "" {
apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
} else {
apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
}
}
else if v.as_any().is::<NewerNoncurrentTask>() {
let v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
//delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
}
else if v.as_any().is::<Jentry>() {
//transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
}
else if v.as_any().is::<FreeVersionTask>() {
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
let oi = v.0.clone();
}
else {
//info!("Invalid work type - {:?}", v);
todo!();
}
}
}
}
}
}
struct TransitionTask {
obj_info: ObjectInfo,
src: LcEventSrc,
event: lifecycle::Event,
}
impl ExpiryOp for TransitionTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
//let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
hasher.flush();
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
}
fn as_any(&self) -> &dyn Any {
self
}
}
pub struct TransitionState {
transition_tx: A_Sender<Option<TransitionTask>>,
transition_rx: A_Receiver<Option<TransitionTask>>,
pub num_workers: AtomicI64,
kill_tx: A_Sender<()>,
kill_rx: A_Receiver<()>,
active_tasks: AtomicI64,
missed_immediate_tasks: AtomicI64,
last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
}
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
//type RetKill = impl Future<Output = Option<()>> + Send + 'static;
//type RetTransitionTask = impl Future<Output = Option<Option<TransitionTask>>> + Send + 'static;
impl TransitionState {
#[allow(clippy::new_ret_no_self)]
pub fn new() -> Arc<Self> {
let (tx1, rx1) = bounded(100000);
let (tx2, rx2) = bounded(1);
Arc::new(Self {
transition_tx: tx1,
transition_rx: rx1,
num_workers: AtomicI64::new(0),
kill_tx: tx2,
kill_rx: rx2,
active_tasks: AtomicI64::new(0),
missed_immediate_tasks: AtomicI64::new(0),
last_day_stats: Arc::new(Mutex::new(HashMap::new())),
})
}
pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()};
select! {
//_ -> t.ctx.Done() => (),
_ = self.transition_tx.send(Some(task)) => (),
else => {
match src {
LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => {
self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst);
}
_ => ()
}
},
}
}
pub async fn init(api: Arc<ECStore>) {
let mut n = 10;//globalAPIConfig.getTransitionWorkers();
let tw = 10;//globalILMConfig.getTransitionWorkers();
if tw > 0 {
n = tw;
}
//let mut transition_state = GLOBAL_TransitionState.write().await;
//self.objAPI = objAPI
Self::update_workers(api, n).await;
}
pub fn pending_tasks(&self) -> usize {
//let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap();
let transition_rx = &GLOBAL_TransitionState.transition_rx;
transition_rx.len()
}
pub fn active_tasks(&self) -> i64 {
self.active_tasks.load(Ordering::SeqCst)
}
pub fn missed_immediate_tasks(&self) -> i64 {
self.missed_immediate_tasks.load(Ordering::SeqCst)
}
pub async fn worker(api: Arc<ECStore>) {
loop {
select! {
_ = GLOBAL_TransitionState.kill_rx.recv() => {
return;
}
task = GLOBAL_TransitionState.transition_rx.recv() => {
if task.is_err() {
break;
}
let task = task.expect("err!");
if task.is_none() {
//self.transition_rx.close();
//drop(self.transition_rx);
return;
}
let task = task.expect("err!");
if task.as_any().is::<TransitionTask>() {
let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
if !err.to_string().contains("use of closed network connection") {
error!("Transition to {} failed for {}/{} version:{} with {}",
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string());
}
}
} else {
let mut ts = TierStats {
total_size: task.obj_info.size as u64,
num_versions: 1,
..Default::default()
};
if task.obj_info.is_latest {
ts.num_objects = 1;
}
GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
}
GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
}
}
else => ()
}
}
}
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.last_day_stats.lock().unwrap();
tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default());
}
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
let tier_stats = self.last_day_stats.lock().unwrap();
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
for (tier, st) in tier_stats.iter() {
res.insert(tier.clone(), st.clone());
}
res
}
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
Self::update_workers_inner(api, n).await;
}
pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
let mut n = n;
if n == 0 {
n = 100;
}
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
while num_workers < n {
let clone_api = api.clone();
tokio::spawn(async move {
TransitionState::worker(clone_api).await;
});
num_workers = num_workers + 1;
GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
}
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
while num_workers > n {
let worker = GLOBAL_TransitionState.kill_tx.clone();
worker.send(()).await;
num_workers = num_workers - 1;
GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
}
}
}
struct AuditTierOp {
tier: String,
time_to_responsens: i64,
output_bytes: i64,
error: String,
}
impl AuditTierOp {
#[allow(clippy::new_ret_no_self)]
pub async fn new() -> Result<Self, std::io::Error> {
Ok(Self {
tier: String::from("tier"),
time_to_responsens: 0,
output_bytes: 0,
error: String::from(""),
})
}
pub fn string(&self) -> String {
format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error)
}
}
pub async fn init_background_expiry(api: Arc<ECStore>) {
let mut workers = num_cpus::get() / 2;
//globalILMConfig.getExpirationWorkers()
if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") {
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
workers = num_expirations;
}
}
if workers == 0 {
workers = 100;
}
//let expiry_state = GLOBAL_ExpiryStSate.write().await;
ExpiryState::resize_workers(workers, api).await;
}
pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
for rule in &lc.rules {
if let Some(transitions) = &rule.transitions {
for transition in transitions {
if let Some(storage_class) = &transition.storage_class {
if storage_class.as_str() != "" {
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
}
}
}
}
if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
for noncurrent_version_transition in noncurrent_version_transitions {
if let Some(storage_class) = &noncurrent_version_transition.storage_class {
if storage_class.as_str() != "" {
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
}
}
}
}
}
Ok(())
}
pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
if !lc.is_none() {
let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
match event.action {
lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
if oi.delete_marker || oi.is_dir {
return;
}
GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
}
_ => ()
}
}
}
pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result<ObjectInfo, std::io::Error> {
//let traceFn = GLOBAL_LifecycleSys.trace(oi);
let mut opts = ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
expiration: ExpirationOptions {expire: true},
..Default::default()
};
if lc_event.action == IlmAction::DeleteVersionAction {
opts.version_id = oi.version_id.map(|id| id.to_string());
}
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
if lc_event.action == IlmAction::DeleteRestoredAction {
opts.transition.expire_restored = true;
match api.delete_object(&oi.bucket, &oi.name, opts).await {
Ok(dobj) => {
//audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
return Ok(dobj);
}
Err(err) => return Err(std::io::Error::other(err)),
}
}
let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await;
if ret.is_ok() {
opts.skip_decommissioned = true;
} else {
//transitionLogIf(ctx, err);
}
let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?;
//defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
let mut event_name = EventName::ObjectRemovedDelete;
if oi.delete_marker {
event_name = EventName::ObjectRemovedDeleteMarkerCreated;
}
let obj_info = ObjectInfo {
name: oi.name.clone(),
version_id: oi.version_id,
delete_marker: oi.delete_marker,
..Default::default()
};
send_event(EventArgs {
event_name: event_name.as_ref().to_string(),
bucket_name: obj_info.bucket.clone(),
object: obj_info,
user_agent: "Internal: [ILM-Expiry]".to_string(),
host: GLOBAL_LocalNodeName.to_string(),
..Default::default()
});
Ok(dobj)
}
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
let us = Uuid::new_v4().to_string();
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes());
hasher.flush();
let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice());
let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us);
Ok(obj)
}
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
let time_ilm = ScannerMetrics::time_ilm(lae.event.action);
let opts = ObjectOptions {
transition: TransitionOptions {
status: lifecycle::TRANSITION_PENDING.to_string(),
tier: lae.event.storage_class,
etag: oi.etag.clone().expect("err").to_string(),
..Default::default()
},
//lifecycle_audit_event: lae,
version_id: Some(oi.version_id.expect("err").to_string()),
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
mod_time: oi.mod_time,
..Default::default()
};
time_ilm(1);
api.transition_object(&oi.bucket, &oi.name, &opts).await
}
pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn {
todo!();
}
pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result<GetObjectReader, std::io::Error> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
Ok(d) => d,
Err(err) => return Err(std::io::Error::other(err)),
};
let ret = new_getobjectreader(rs, &oi, opts, &h);
if let Err(err) = ret {
return Err(error_resp_to_object_err(err, vec![bucket, object]));
}
let (get_fn, off, length) = ret.expect("err");
let mut gopts = WarmBackendGetOpts::default();
if off >= 0 && length >= 0 {
gopts.start_offset = off;
gopts.length = length;
}
//return Ok(HttpFileReader::new(rs, &oi, opts, &h));
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?;
Ok(get_fn(reader, h))
}
pub fn post_restore_opts(r: http::Request<Body>, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
todo!();
}
pub fn put_restore_opts(bucket: &str, object: &str, rreq: &RestoreObjectRequest, oi: &ObjectInfo) -> ObjectOptions {
todo!();
}
pub trait LifecycleOps {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts;
}
impl LifecycleOps for ObjectInfo {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
lifecycle::ObjectOpts {
name: self.name.clone(),
user_tags: self.user_tags.clone(),
version_id: self.version_id.expect("err").to_string(),
mod_time: self.mod_time,
size: self.size,
is_latest: self.is_latest,
num_versions: self.num_versions,
delete_marker: self.delete_marker,
successor_mod_time: self.successor_mod_time,
//restore_ongoing: self.restore_ongoing,
//restore_expires: self.restore_expires,
transition_status: self.transitioned_object.status.clone(),
..Default::default()
}
}
}
#[derive(Debug, Default, Clone)]
pub struct S3Location {
pub bucketname: String,
//pub encryption: Encryption,
pub prefix: String,
pub storage_class: String,
//pub tagging: Tags,
pub user_metadata: HashMap<String, String>,
}
#[derive(Debug, Default, Clone)]
pub struct OutputLocation(pub S3Location);
#[derive(Debug, Default, Clone)]
pub struct RestoreObjectRequest {
pub days: i64,
pub ror_type: String,
pub tier: String,
pub description: String,
//pub select_parameters: SelectParameters,
pub output_location: OutputLocation,
}
const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;

702
ecstore/src/bucket/lifecycle/lifecycle.rs Normal file
View File

@@ -0,0 +1,702 @@
use std::cmp::Ordering;
use std::env;
use std::fmt::Display;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration,
ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition,
};
use time::macros::{datetime, offset};
use time::{self, OffsetDateTime, Duration};
use crate::bucket::lifecycle::rule::TransitionOps;
use super::bucket_lifecycle_ops::RestoreObjectRequest;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IlmAction {
NoneAction = 0,
DeleteAction,
DeleteVersionAction,
TransitionAction,
TransitionVersionAction,
DeleteRestoredAction,
DeleteRestoredVersionAction,
DeleteAllVersionsAction,
DelMarkerDeleteAllVersionsAction,
ActionCount,
}
impl IlmAction {
pub fn delete_restored(&self) -> bool {
*self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
}
pub fn delete_versioned(&self) -> bool {
*self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
}
pub fn delete_all(&self) -> bool {
*self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
}
pub fn delete(&self) -> bool {
if self.delete_restored() {
return true;
}
*self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
}
}
impl Display for IlmAction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
#[async_trait::async_trait]
pub trait RuleValidate {
fn validate(&self) -> Result<(), std::io::Error>;
}
#[async_trait::async_trait]
impl RuleValidate for LifecycleRule {
/*fn validate_id(&self) -> Result<()> {
if self.id.len() > 255 {
return errInvalidRuleID;
}
Ok(())
}
fn validate_status(&self) -> Result<()> {
if self.Status.len() == 0 {
return errEmptyRuleStatus;
}
if self.Status != Enabled && self.Status != Disabled {
return errInvalidRuleStatus;
}
Ok(())
}
fn validate_expiration(&self) -> Result<()> {
self.Expiration.Validate();
}
fn validate_noncurrent_expiration(&self) -> Result<()> {
self.NoncurrentVersionExpiration.Validate()
}
fn validate_prefix_and_filter(&self) -> Result<()> {
if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() {
return errXMLNotWellFormed;
}
if !self.Prefix.set {
return self.Filter.Validate();
}
Ok(())
}
fn validate_transition(&self) -> Result<()> {
self.Transition.Validate()
}
fn validate_noncurrent_transition(&self) -> Result<()> {
self.NoncurrentVersionTransition.Validate()
}
fn get_prefix(&self) -> String {
if p := self.Prefix.String(); p != "" {
return p
}
if p := self.Filter.Prefix.String(); p != "" {
return p
}
if p := self.Filter.And.Prefix.String(); p != "" {
return p
}
"".to_string()
}*/
fn validate(&self) -> Result<(), std::io::Error> {
/*self.validate_id()?;
self.validate_status()?;
self.validate_expiration()?;
self.validate_noncurrent_expiration()?;
self.validate_prefix_and_filter()?;
self.validate_transition()?;
self.validate_noncurrent_transition()?;
if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
return errInvalidRuleDelMarkerExpiration
}
if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
return errXMLNotWellFormed
}*/
Ok(())
}
}
#[async_trait::async_trait]
pub trait Lifecycle {
async fn has_transition(&self) -> bool;
fn has_expiry(&self) -> bool;
async fn has_active_rules(&self, prefix: &str) -> bool;
async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
async fn eval(&self, obj: &ObjectOpts) -> Event;
async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event;
//fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event;
}
#[async_trait::async_trait]
impl Lifecycle for BucketLifecycleConfiguration {
async fn has_transition(&self) -> bool {
for rule in self.rules.iter() {
if !rule.transitions.is_none() {
return true;
}
}
false
}
fn has_expiry(&self) -> bool {
for rule in self.rules.iter() {
if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() {
return true;
}
}
false
}
async fn has_active_rules(&self, prefix: &str) -> bool {
if self.rules.len() == 0 {
return false;
}
for rule in self.rules.iter() {
if rule.status.as_str() == ExpirationStatus::DISABLED {
continue;
}
let rule_prefix = rule.prefix.as_ref().expect("err!");
if prefix.len() > 0 && rule_prefix.len() > 0 {
if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix) {
continue;
}
}
let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
return true;
}
if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
return true;
}
if !rule.noncurrent_version_transitions.is_none() {
return true;
}
let rule_expiration = rule.expiration.as_ref().expect("err!");
if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
return true;
}
if !rule_expiration.date.is_none() {
return true;
}
if rule_expiration.expired_object_delete_marker.expect("err!") {
return true;
}
let rule_transitions: &[Transition]= &rule.transitions.as_ref().expect("err!");
let rule_transitions_0 = rule_transitions[0].clone();
if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
return true;
}
if !rule.transitions.is_none() {
return true;
}
}
false
}
async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error> {
if self.rules.len() > 1000 {
return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES));
}
if self.rules.len() == 0 {
return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE));
}
for r in &self.rules {
r.validate()?;
if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
if let Some(expiration) = r.expiration.as_ref() {
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (!expired_object_delete_marker) {
return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
}
} /*else {
if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED {
return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED));
}
}*/
}
}
}
for (i,_) in self.rules.iter().enumerate() {
if i == self.rules.len()-1 {
break;
}
let other_rules = &self.rules[i+1..];
for other_rule in other_rules {
if self.rules[i].id == other_rule.id {
return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
}
}
}
Ok(())
}
async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>> {
if obj.name == "" {
return None;
}
let mut rules = Vec::<LifecycleRule>::new();
for rule in self.rules.iter() {
if rule.status.as_str() == ExpirationStatus::DISABLED {
continue;
}
if let Some(prefix) = rule.prefix.clone() {
if !obj.name.starts_with(prefix.as_str()) {
continue;
}
}
/*if !rule.filter.test_tags(obj.user_tags) {
continue;
}*/
//if !obj.delete_marker && !rule.filter.BySize(obj.size) {
if !obj.delete_marker && false{
continue;
}
rules.push(rule.clone());
}
Some(rules)
}
async fn eval(&self, obj: &ObjectOpts) -> Event {
self.eval_inner(obj, OffsetDateTime::now_utc()).await
}
async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event {
let mut events = Vec::<Event>::new();
if obj.mod_time.expect("err").unix_timestamp() == 0 {
return Event::default();
}
if let Some(restore_expires) = obj.restore_expires {
if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() {
let mut action = IlmAction::DeleteRestoredAction;
if !obj.is_latest {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event{
action: action,
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
}
if let Some(ref lc_rules) = self.filter_rules(obj).await {
for rule in lc_rules.iter() {
if obj.expired_object_deletemarker() {
if let Some(expiration) = rule.expiration.as_ref() {
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
events.push(Event{
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(now),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
break;
}
}
if let Some(expiration) = rule.expiration.as_ref() {
if let Some(days) = expiration.days {
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days/*, date*/);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
break;
}
}
}
}
if obj.is_latest {
if let Some(ref expiration) = rule.expiration {
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
if obj.delete_marker && expired_object_delete_marker {
let due = expiration.next_due(obj);
if let Some(due) = due {
if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
events.push(Event{
action: IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
}
continue;
}
}
}
}
if !obj.is_latest {
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
if newer_noncurrent_versions > 0 {
continue;
}
}
}
}
if !obj.is_latest {
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days {
if noncurrent_days != 0 {
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
}
}
}
}
}
if !obj.is_latest {
if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions {
if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class {
if storage_class.as_str() != "" {
if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE {
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) {
events.push(Event {
action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"),
due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(),
..Default::default()
});
}
}
}
}
}
}
if obj.is_latest && !obj.delete_marker {
if let Some(ref expiration) = rule.expiration {
if let Some(ref date) = expiration.date {
let date0 = OffsetDateTime::from(date.clone());
if date0.unix_timestamp() != 0 {
if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(date0),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
}
} else if let Some(days) = expiration.days {
if days != 0 {
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
let mut event = Event{
action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
};
/*if rule.expiration.expect("err!").delete_all.val {
event.action = IlmAction::DeleteAllVersionsAction
}*/
events.push(event);
}
}
}
}
if obj.transition_status != TRANSITION_COMPLETE {
if let Some(ref transitions) = rule.transitions {
let due = transitions[0].next_due(obj);
if let Some(due) = due {
if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) {
events.push(Event{
action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due),
storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
});
}
}
}
}
}
}
}
if events.len() > 0 {
events.sort_by(|a, b| {
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() {
match a.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => {
return Ordering::Less;
}
_ => ()
}
match b.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => {
return Ordering::Greater;
}
_ => ()
}
return Ordering::Less;
}
if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
return Ordering::Less;
}
return Ordering::Greater;
});
return events[0].clone();
}
Event::default()
}
async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event {
if let Some(filter_rules) = self.filter_rules(obj).await {
for rule in filter_rules.iter() {
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
if newer_noncurrent_versions == 0 {
continue;
}
return Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: newer_noncurrent_versions as usize,
due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(),
};
} else {
return Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: 0,
due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(),
};
}
}
}
}
Event::default()
}
}
#[async_trait::async_trait]
pub trait LifecycleCalculate {
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime>;
}
#[async_trait::async_trait]
impl LifecycleCalculate for LifecycleExpiration {
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
if !obj.is_latest || !obj.delete_marker {
return None;
}
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
}
}
#[async_trait::async_trait]
impl LifecycleCalculate for NoncurrentVersionTransition {
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
if obj.is_latest || self.storage_class.is_none() {
return None;
}
if self.noncurrent_days.is_none() {
return obj.successor_mod_time;
}
Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap()))
}
}
#[async_trait::async_trait]
impl LifecycleCalculate for Transition {
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
if !obj.is_latest || self.days.is_none() {
return None;
}
if let Some(date) = self.date.clone() {
return Some(date.into());
}
if self.days.is_none() {
return obj.mod_time;
}
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
}
}
pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
if days == 0 {
return mod_time;
}
let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug
let mut hour = 3600;
if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
hour = num_hour;
}
}
//t.Truncate(24 * hour)
t
}
#[derive(Default)]
pub struct ObjectOpts {
pub name: String,
pub user_tags: String,
pub mod_time: Option<OffsetDateTime>,
pub size: usize,
pub version_id: String,
pub is_latest: bool,
pub delete_marker: bool,
pub num_versions: usize,
pub successor_mod_time: Option<OffsetDateTime>,
pub transition_status: String,
pub restore_ongoing: bool,
pub restore_expires: Option<OffsetDateTime>,
pub versioned: bool,
pub version_suspended: bool,
}
impl ObjectOpts {
pub fn expired_object_deletemarker(&self) -> bool {
self.delete_marker && self.num_versions == 1
}
}
#[derive(Debug, Clone)]
pub struct Event {
pub action: IlmAction,
pub rule_id: String,
pub due: Option<OffsetDateTime>,
pub noncurrent_days: u32,
pub newer_noncurrent_versions: usize,
pub storage_class: String,
}
impl Default for Event {
fn default() -> Self {
Self {
action: IlmAction::NoneAction,
rule_id: "".into(),
due: Some(OffsetDateTime::UNIX_EPOCH),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
}
}
}
#[derive(Debug, Clone, Default)]
pub struct ExpirationOptions {
pub expire: bool
}
impl ExpirationOptions {
fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>, std::io::Error> {
todo!();
}
fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>, std::io::Error> {
todo!();
}
fn msg_size(&self) -> i64 {
1 + 7 + 10
}
}
#[derive(Debug, Clone)]
pub struct TransitionOptions {
pub status: String,
pub tier: String,
pub etag: String,
pub restore_request: RestoreObjectRequest,
pub restore_expiry: OffsetDateTime,
pub expire_restored: bool,
}
impl Default for TransitionOptions {
fn default() -> Self {
Self {
status: Default::default(),
tier: Default::default(),
etag: Default::default(),
restore_request: Default::default(),
restore_expiry: OffsetDateTime::now_utc(),
expire_restored: Default::default(),
}
}
}

6
ecstore/src/bucket/lifecycle/mod.rs Normal file
View File

@@ -0,0 +1,6 @@
pub mod rule;
pub mod lifecycle;
pub mod tier_sweeper;
pub mod tier_last_day_stats;
pub mod bucket_lifecycle_ops;
pub mod bucket_lifecycle_audit;

51
ecstore/src/bucket/lifecycle/rule.rs Normal file
View File

@@ -0,0 +1,51 @@
use s3s::dto::{
LifecycleRuleFilter, Transition,
};
const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter {
fn test_tags(&self, user_tags: &str) -> bool;
fn by_size(&self, sz: i64) -> bool;
}
impl Filter for LifecycleRuleFilter {
fn test_tags(&self, user_tags: &str) -> bool {
true
}
fn by_size(&self, sz: i64) -> bool {
true
}
}
pub trait TransitionOps {
fn validate(&self) -> Result<(), std::io::Error>;
}
impl TransitionOps for Transition {
fn validate(&self) -> Result<(), std::io::Error> {
if !self.date.is_none() && self.days.expect("err!") > 0 {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if self.storage_class.is_none() {
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
}
Ok(())
}
}
#[cfg(test)]
mod test {
use super::*;
#[tokio::test]
async fn test_rule() {
//assert!(skip_access_checks(p.to_str().unwrap()));
}
}

86
ecstore/src/bucket/lifecycle/tier_last_day_stats.rs Normal file
View File

@@ -0,0 +1,86 @@
use sha2::Sha256;
use std::collections::HashMap;
use time::OffsetDateTime;
use tracing::{error, warn};
use std::ops::Sub;
use crate::heal::data_usage_cache::TierStats;
pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
#[derive(Clone)]
pub struct LastDayTierStats {
bins: [TierStats; 24],
updated_at: OffsetDateTime,
}
impl Default for LastDayTierStats {
fn default() -> Self {
Self {
bins: Default::default(),
updated_at: OffsetDateTime::now_utc(),
}
}
}
impl LastDayTierStats {
pub fn add_stats(&mut self, ts: TierStats) {
let mut now = OffsetDateTime::now_utc();
self.forward_to(&mut now);
let now_idx = now.hour() as usize;
self.bins[now_idx] = self.bins[now_idx].add(&ts);
}
fn forward_to(&mut self, t: &mut OffsetDateTime) {
if t.unix_timestamp() == 0 {
*t = OffsetDateTime::now_utc();
}
let since = t.sub(self.updated_at).whole_hours();
if since < 1 {
return;
}
let (idx, mut last_idx) = (t.hour(), self.updated_at.hour());
self.updated_at = *t;
if since >= 24 {
self.bins = [TierStats::default(); 24];
return;
}
while last_idx != idx {
last_idx = (last_idx + 1) % 24;
self.bins[last_idx as usize] = TierStats::default();
}
}
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
let mut cl = self.clone();
let mut cm = m.clone();
let mut merged = LastDayTierStats::default();
if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {
cm.forward_to(&mut cl.updated_at);
merged.updated_at = cl.updated_at;
} else {
cl.forward_to(&mut cm.updated_at);
merged.updated_at = cm.updated_at;
}
for (i, _) in cl.bins.iter().enumerate() {
merged.bins[i] = cl.bins[i].add(&cm.bins[i]);
}
merged
}
}
#[cfg(test)]
mod test {
}

130
ecstore/src/bucket/lifecycle/tier_sweeper.rs Normal file
View File

@@ -0,0 +1,130 @@
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use std::any::Any;
use std::io::{Cursor, Write};
use crate::global::GLOBAL_TierConfigMgr;
use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
use super::lifecycle::{self, ObjectOpts};
static XXHASH_SEED: u64 = 0;
#[derive(Default)]
struct ObjSweeper {
object: String,
bucket: String,
version_id: String,
versioned: bool,
suspended: bool,
transition_status: String,
transition_tier: String,
transition_version_id: String,
remote_object: String,
}
impl ObjSweeper {
#[allow(clippy::new_ret_no_self)]
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
Ok(Self {
object: object.into(),
bucket: bucket.into(),
..Default::default()
})
}
pub fn with_version(&mut self, vid: String) -> &Self {
self.version_id = vid;
self
}
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
self.versioned = versioned;
self.suspended = suspended;
self
}
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts{
version_id: self.version_id.clone(),
versioned: self.versioned,
version_suspended: self.suspended,
..Default::default()
};
if self.suspended && self.version_id == "" {
opts.version_id = String::from("");
}
opts
}
pub fn set_transition_state(&mut self, info: TransitionedObject) {
self.transition_tier = info.tier;
self.transition_status = info.status;
self.remote_object = info.name;
self.transition_version_id = info.version_id;
}
pub fn should_remove_remote_object(&self) -> Option<Jentry> {
if self.transition_status != lifecycle::TRANSITION_COMPLETE {
return None;
}
let mut del_tier = false;
if !self.versioned || self.suspended { // 1, 2.a, 2.b
del_tier = true;
} else if self.versioned && self.version_id != "" { // 3.a
del_tier = true;
}
if del_tier {
return Some(Jentry {
obj_name: self.remote_object.clone(),
version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(),
});
}
None
}
pub async fn sweep(&self) {
let je = self.should_remove_remote_object();
if !je.is_none() {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
}
}
}
#[derive(Debug, Clone)]
pub struct Jentry {
obj_name: String,
version_id: String,
tier_name: String,
}
impl ExpiryOp for Jentry {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
let _ = hasher.write(format!("{}", self.tier_name).as_bytes());
let _ = hasher.write(format!("{}", self.obj_name).as_bytes());
hasher.flush();
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
}
fn as_any(&self) -> &dyn Any {
self
}
}
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
let w = match config_mgr.get_driver(tier_name).await {
Ok(w) => w,
Err(e) => return Err(std::io::Error::other(e)),
};
w.remove(obj_name, rv_id).await
}
#[cfg(test)]
mod test {
}

1
ecstore/src/bucket/mod.rs
View File

@@ -10,3 +10,4 @@ pub mod target;
pub mod utils;
pub mod versioning;
pub mod versioning_sys;
pub mod lifecycle;

3
ecstore/src/bucket/object_lock/mod.rs
View File

@@ -1,3 +1,6 @@
pub mod objectlock;
pub mod objectlock_sys;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
pub trait ObjectLockApi {

95
ecstore/src/bucket/object_lock/objectlock.rs Normal file
View File

@@ -0,0 +1,95 @@
use std::any::{Any, TypeId};
use std::collections::HashMap;
use time::{OffsetDateTime, format_description};
use tracing::{error, warn};
use s3s::dto::{
ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold,
Date,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
};
//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date";
//const AMZ_OBJECTLOCK_MODE: &str = "X-Amz-Object-Lock-Mode";
//const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold";
const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config";
const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
pub fn utc_now_ntp() -> OffsetDateTime {
return OffsetDateTime::now_utc();
}
pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRetention {
let mode: ObjectLockRetentionMode;
let mut retain_until_date: Date = Date::from(OffsetDateTime::UNIX_EPOCH);
let mut mode_str = meta.get(X_AMZ_OBJECT_LOCK_MODE.as_str().to_lowercase().as_str());
if mode_str.is_none() {
mode_str = Some(&meta[X_AMZ_OBJECT_LOCK_MODE.as_str()]);
}
if let Some(mode_str) = mode_str {
mode = parse_ret_mode(mode_str.as_str());
} else {
return ObjectLockRetention {mode: None, retain_until_date: None};
}
let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
if till_str.is_none() {
till_str = Some(&meta[X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()]);
}
if let Some(till_str) = till_str {
let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
if t.is_err() {
retain_until_date = Date::from(t.expect("err")); //TODO: utc
}
}
ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)}
}
pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
let mut hold_str = meta.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_lowercase().as_str());
if hold_str.is_none() {
hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
}
if let Some(hold_str) = hold_str {
return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))};
}
ObjectLockLegalHold {status: None}
}
pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
let mut mode;
match mode_str.to_uppercase().as_str() {
"GOVERNANCE" => {
mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE);
}
"COMPLIANCE" => {
mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE);
}
_ => unreachable!()
}
mode
}
pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
let mut st;
match hold_str {
"ON" => {
st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON);
}
"OFF" => {
st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF);
}
_ => unreachable!()
}
st
}

55
ecstore/src/bucket/object_lock/objectlock_sys.rs Normal file
View File

@@ -0,0 +1,55 @@
use std::any::{Any, TypeId};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::{error, warn};
use s3s::dto::{
DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
};
use crate::store_api::ObjectInfo;
use crate::bucket::metadata_sys::get_object_lock_config;
use super::objectlock;
pub struct BucketObjectLockSys {}
impl BucketObjectLockSys {
#[allow(clippy::new_ret_no_self)]
pub async fn new() -> Arc<Self> {
Arc::new(Self {})
}
pub async fn get(bucket: &str) -> Option<DefaultRetention> {
if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule {
return object_lock_rule.default_retention;
}
None
}
}
pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
if obj_info.delete_marker {
return false;
}
let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err"));
match lhold.status {
Some(st) if st.as_str()==ObjectLockLegalHoldStatus::ON => {
return true;
}
_ => ()
}
let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err"));
match ret.mode {
Some(r) if (r.as_str() == ObjectLockRetentionMode::COMPLIANCE || r.as_str() == ObjectLockRetentionMode::GOVERNANCE) => {
let t = objectlock::utc_now_ntp();
if OffsetDateTime::from(ret.retain_until_date.expect("err!")).unix_timestamp() > t.unix_timestamp() {
return true;
}
}
_ => ()
}
false
}

310
ecstore/src/checksum.rs Normal file
View File

@@ -0,0 +1,310 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use std::ops::{BitAnd, BitOr};
use lazy_static::lazy_static;
use reader::hasher::{Hasher, Sha256};
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
use crate::client::{
api_put_object::PutObjectOptions,
api_s3_datatypes::ObjectPart,
};
use rustfs_utils::crypto::{base64_decode, base64_encode};
use crate::{
disk::DiskAPI,
store_api::GetObjectReader,
};
use enumset::{enum_set, EnumSet, EnumSetType};
#[derive(Debug, EnumSetType, Default)]
#[enumset(repr = "u8")]
pub enum ChecksumMode {
#[default]
ChecksumNone,
ChecksumSHA256,
ChecksumSHA1,
ChecksumCRC32,
ChecksumCRC32C,
ChecksumCRC64NVME,
ChecksumFullObject,
}
lazy_static! {
static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
let mut s = EnumSet::all();
s.remove(ChecksumMode::ChecksumFullObject);
s
};
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
}
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
impl ChecksumMode {
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
pub fn base(&self) -> ChecksumMode {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() {
1_u8 => {
ChecksumMode::ChecksumNone
}
2_u8 => {
ChecksumMode::ChecksumSHA256
}
4_u8 => {
ChecksumMode::ChecksumSHA1
}
8_u8 => {
ChecksumMode::ChecksumCRC32
}
16_u8 => {
ChecksumMode::ChecksumCRC32C
}
32_u8 => {
ChecksumMode::ChecksumCRC64NVME
}
_ => panic!("enum err."),
}
}
pub fn is(&self, t: ChecksumMode) -> bool {
*self & t == t
}
pub fn key(&self) -> String {
//match c & checksumMask {
match self {
ChecksumMode::ChecksumCRC32 => {
return X_AMZ_CHECKSUM_CRC32.to_string();
}
ChecksumMode::ChecksumCRC32C => {
return X_AMZ_CHECKSUM_CRC32C.to_string();
}
ChecksumMode::ChecksumSHA1 => {
return X_AMZ_CHECKSUM_SHA1.to_string();
}
ChecksumMode::ChecksumSHA256 => {
return X_AMZ_CHECKSUM_SHA256.to_string();
}
ChecksumMode::ChecksumCRC64NVME => {
return AMZ_CHECKSUM_CRC64NVME.to_string();
}
_ => {
return "".to_string();
}
}
}
pub fn can_composite(&self) -> bool {
todo!();
}
pub fn can_merge_crc(&self) -> bool {
todo!();
}
pub fn full_object_requested(&self) -> bool {
todo!();
}
pub fn key_capitalized(&self) -> String {
self.key()
}
pub fn raw_byte_len(&self) -> usize {
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
4
}
else if u == ChecksumMode::ChecksumSHA1 as u8 {
4//sha1.size
}
else if u == ChecksumMode::ChecksumSHA256 as u8 {
4//sha256.size
}
else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
4//crc64.size
}
else {
0
}
}
pub fn hasher(&self) -> Result<Box<dyn Hasher>, std::io::Error> {
match /*C_ChecksumMask & **/self {
/*ChecksumMode::ChecksumCRC32 => {
return Ok(Box::new(crc32fast::Hasher::new()));
}*/
/*ChecksumMode::ChecksumCRC32C => {
return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli))));
}
ChecksumMode::ChecksumSHA1 => {
return Ok(Box::new(sha1::new()));
}*/
ChecksumMode::ChecksumSHA256 => {
return Ok(Box::new(Sha256::new()));
}
/*ChecksumMode::ChecksumCRC64NVME => {
return Ok(Box::new(crc64nvme.New());
}*/
_ => return Err(std::io::Error::other("unsupported checksum type")),
}
}
pub fn is_set(&self) -> bool {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
s.len() == 1
}
pub fn set_default(&mut self, t: ChecksumMode) {
if !self.is_set() {
*self = t;
}
}
pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
if !self.is_set() {
return Ok("".to_string());
}
let mut h = self.hasher()?;
h.write(b);
Ok(base64_encode(h.sum().as_bytes()))
}
pub fn to_string(&self) -> String {
//match c & checksumMask {
match self {
ChecksumMode::ChecksumCRC32 => {
return "CRC32".to_string();
}
ChecksumMode::ChecksumCRC32C => {
return "CRC32C".to_string();
}
ChecksumMode::ChecksumSHA1 => {
return "SHA1".to_string();
}
ChecksumMode::ChecksumSHA256 => {
return "SHA256".to_string();
}
ChecksumMode::ChecksumNone => {
return "".to_string();
}
ChecksumMode::ChecksumCRC64NVME => {
return "CRC64NVME".to_string();
}
_=> {
return "<invalid>".to_string();
}
}
}
pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
let mut h = self.hasher()?;
Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
}
pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
let mut h = self.hasher()?;
Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
}
pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
if !self.can_composite() {
return Err(std::io::Error::other("cannot do composite checksum"));
}
p.sort_by(|i, j| {
if i.part_num < j.part_num {
std::cmp::Ordering::Less
} else if i.part_num > j.part_num {
std::cmp::Ordering::Greater
} else {
std::cmp::Ordering::Equal
}
});
let c = self.base();
let mut crc_bytes = Vec::<u8>::with_capacity(p.len()*self.raw_byte_len() as usize);
let mut h = self.hasher()?;
h.write(&crc_bytes);
Ok(Checksum {checksum_type: self.clone(), r: h.sum().as_bytes().to_vec()})
}
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
todo!();
}
}
#[derive(Default)]
struct Checksum {
checksum_type: ChecksumMode,
r: Vec<u8>,
}
impl Checksum {
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
if t.is_set() && b.len() == t.raw_byte_len() {
return Checksum {checksum_type: t, r: b.to_vec()};
}
Checksum::default()
}
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
let b = match base64_decode(s.as_bytes()) {
Ok(b) => b,
Err(err) => return Err(std::io::Error::other(err.to_string())),
};
if t.is_set() && b.len() == t.raw_byte_len() {
return Ok(Checksum {checksum_type: t, r: b});
}
Ok(Checksum::default())
}
fn is_set(&self) -> bool {
self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
}
fn encoded(&self) -> String {
if !self.is_set() {
return "".to_string();
}
base64_encode(&self.r)
}
fn raw(&self) -> Option<Vec<u8>> {
if !self.is_set() {
return None;
}
Some(self.r.clone())
}
}
pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
opts.user_metadata.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
if opts.auto_checksum.full_object_requested() {
opts.user_metadata.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
}
}
pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
let crc = opts.auto_checksum.composite_checksum(all_parts)?;
opts.user_metadata = {
let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key(), crc.encoded());
hm
}
} else if opts.auto_checksum.can_merge_crc() {
let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
opts.user_metadata = {
let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
hm
}
}
Ok(())
}

33
ecstore/src/client/admin_handler_utils.rs Normal file
View File

@@ -0,0 +1,33 @@
use http::status::StatusCode;
use std::fmt::{self, Display, Formatter};
#[derive(Default, thiserror::Error, Debug, PartialEq)]
pub struct AdminError {
pub code: &'static str,
pub message: &'static str,
pub status_code: StatusCode,
}
impl Display for AdminError {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{}", self.message)
}
}
impl AdminError {
pub fn new(code: &'static str, message: &'static str, status_code: StatusCode) -> Self {
Self {
code,
message,
status_code,
}
}
pub fn msg(message: &'static str) -> Self {
Self {
code: "InternalError",
message,
status_code: StatusCode::INTERNAL_SERVER_ERROR,
}
}
}

113
ecstore/src/client/api_bucket_policy.rs Normal file
View File

@@ -0,0 +1,113 @@
#![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, StatusCode};
use bytes::Bytes;
use crate::client::{
api_error_response::http_resp_to_error_response,
transition_api::{RequestMetadata, TransitionClient, ReaderImpl}
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
impl TransitionClient {
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
if policy == "" {
return self.remove_bucket_policy(bucket_name).await;
}
self.put_bucket_policy(bucket_name, policy).await
}
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
content_length: policy.len() as i64,
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_md5_base64: "".to_string(),
content_sha256_hex: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
};
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
//defer closeResponse(resp)
//if resp != nil {
if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
}
//}
Ok(())
}
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
//defer closeResponse(resp)
if resp.status() != StatusCode::NO_CONTENT {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
}
Ok(())
}
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
Ok(bucket_policy)
}
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
Ok(policy)
}
}

252
ecstore/src/client/api_error_response.rs Normal file
View File

@@ -0,0 +1,252 @@
#![allow(clippy::map_entry)]
use std::fmt::Display;
use http::StatusCode;
use serde::{Serialize, Deserialize};
use serde::{ser::Serializer, de::Deserializer};
use s3s::S3ErrorCode;
use s3s::Body;
const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[serde(default, rename_all = "PascalCase")]
pub struct ErrorResponse {
#[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
pub code: S3ErrorCode,
pub message: String,
pub bucket_name: String,
pub key: String,
pub resource: String,
pub request_id: String,
pub host_id: String,
pub region: String,
pub server: String,
#[serde(skip)]
pub status_code: StatusCode,
}
fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
where
S: Serializer
{
s.serialize_str("")
}
fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where
D: Deserializer<'de>
{
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
}
impl Default for ErrorResponse {
fn default() -> Self {
ErrorResponse {
code: S3ErrorCode::Custom("".into()),
message: Default::default(),
bucket_name: Default::default(),
key: Default::default(),
resource: Default::default(),
request_id: Default::default(),
host_id: Default::default(),
region: Default::default(),
server: Default::default(),
status_code: Default::default(),
}
}
}
impl Display for ErrorResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
if let Some(err) = err.get_ref() {
if err.is::<ErrorResponse>() {
err.downcast_ref::<ErrorResponse>().expect("err!").clone()
} else {
ErrorResponse::default()
}
} else {
ErrorResponse::default()
}
}
pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucket_name: &str, object_name: &str) -> ErrorResponse {
let err_body = String::from_utf8(b).unwrap();
//let err_body = xml_decode_and_body(resp.body, &err_resp);
let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
let mut err_resp = ErrorResponse::default();
if err_resp_.is_err() {
match resp.status() {
StatusCode::NOT_FOUND => {
if object_name == "" {
err_resp = ErrorResponse {
status_code: resp.status(),
code: S3ErrorCode::NoSuchBucket,
message: "The specified bucket does not exist.".to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
};
} else {
err_resp = ErrorResponse {
status_code: resp.status(),
code: S3ErrorCode::NoSuchKey,
message: "The specified key does not exist.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
}
StatusCode::FORBIDDEN => {
err_resp = ErrorResponse {
status_code: resp.status(),
code: S3ErrorCode::AccessDenied,
message: "Access Denied.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
StatusCode::CONFLICT => {
err_resp = ErrorResponse {
status_code: resp.status(),
code: S3ErrorCode::BucketNotEmpty,
message: "Bucket not empty.".to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
};
}
StatusCode::PRECONDITION_FAILED => {
err_resp = ErrorResponse {
status_code: resp.status(),
code: S3ErrorCode::PreconditionFailed,
message: "Pre condition failed.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
_ => {
let mut msg = resp.status().to_string();
if err_body.len() > 0 {
msg = err_body;
}
err_resp = ErrorResponse{
status_code: resp.status(),
code: S3ErrorCode::Custom(resp.status().to_string().into()),
message: msg,
bucket_name: bucket_name.to_string(),
..Default::default()
};
}
}
} else {
err_resp = err_resp_.unwrap();
}
err_resp.status_code = resp.status();
if let Some(server_name) = resp.headers().get("Server") {
err_resp.server = server_name.to_str().expect("err").to_string();
}
let code = resp.headers().get("x-minio-error-code");
if code.is_some() {
err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
}
let desc = resp.headers().get("x-minio-error-desc");
if desc.is_some() {
err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
}
if err_resp.request_id == "" {
if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
}
}
if err_resp.host_id == "" {
if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
}
}
if err_resp.region == "" {
if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
}
}
if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
}
err_resp
}
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument,
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.".to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
}
}
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", total_size, max_object_size);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooLarge,
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", total_size);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooSmall,
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", total_read, total_size);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::Custom("UnexpectedEOF".into()),
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_invalid_argument(message: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument,
message: message.to_string(),
request_id: "rustfs".to_string(),
..Default::default()
}
}
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::NOT_IMPLEMENTED,
code: S3ErrorCode::Custom("APINotSupported".into()),
message: message.to_string(),
request_id: "rustfs".to_string(),
..Default::default()
}
}

198
ecstore/src/client/api_get_object.rs Normal file
View File

@@ -0,0 +1,198 @@
#![allow(clippy::map_entry)]
use bytes::Bytes;
use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor;
use crate::client::{
transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient},
api_error_response::err_invalid_argument,
api_get_options::GetObjectOptions,
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
impl TransitionClient {
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
todo!();
}
pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: opts.to_query_values(),
custom_header: opts.header(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
let resp = &resp;
let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
let b = resp.body().bytes().expect("err").to_vec();
Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
}
}
#[derive(Default)]
struct GetRequest {
pub buffer: Vec<u8>,
pub offset: i64,
pub did_offset_change: bool,
pub been_read: bool,
pub is_read_at: bool,
pub is_read_op: bool,
pub is_first_req: bool,
pub setting_object_info: bool,
}
struct GetResponse {
pub size: i64,
//pub error: error,
pub did_read: bool,
pub object_info: ObjectInfo,
}
#[derive(Default)]
struct Object {
//pub reqch: chan<- getRequest,
//pub resch: <-chan getResponse,
//pub cancel: context.CancelFunc,
pub curr_offset: i64,
pub object_info: ObjectInfo,
pub seek_data: bool,
pub is_closed: bool,
pub is_started: bool,
//pub prev_err: error,
pub been_read: bool,
pub object_info_set: bool,
}
impl Object {
pub fn new() -> Object {
Self {
..Default::default()
}
}
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
todo!()
}
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
self.curr_offset += bytes_read;
Ok(())
}
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
let mut read_req = GetRequest {
is_read_op: true,
been_read: self.been_read,
buffer: b.to_vec(),
..Default::default()
};
if !self.is_started {
read_req.is_first_req = true;
}
read_req.did_offset_change = self.seek_data;
read_req.offset = self.curr_offset;
let response = self.do_get_request(&read_req)?;
let bytes_read = response.size;
let oerr = self.set_offset(bytes_read);
Ok(response.size)
}
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
if !self.is_started || !self.object_info_set {
let _ = self.do_get_request(&GetRequest {
is_first_req: !self.is_started,
setting_object_info: !self.object_info_set,
..Default::default()
})?;
}
Ok(self.object_info.clone())
}
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
self.curr_offset = offset;
let mut read_at_req = GetRequest {
is_read_op: true,
is_read_at: true,
did_offset_change: true,
been_read: self.been_read,
offset,
buffer: b.to_vec(),
..Default::default()
};
if !self.is_started {
read_at_req.is_first_req = true;
}
let response = self.do_get_request(&read_at_req)?;
let bytes_read = response.size;
if !self.object_info_set {
self.curr_offset += bytes_read;
} else {
let oerr = self.set_offset(bytes_read);
}
Ok(response.size)
}
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
if !self.is_started || !self.object_info_set {
let seek_req = GetRequest {
is_read_op: false,
offset: offset,
is_first_req: true,
..Default::default()
};
let _ = self.do_get_request(&seek_req);
}
let mut new_offset = self.curr_offset;
match whence {
0 => {
new_offset = offset;
}
1 => {
new_offset += offset;
}
2 => {
new_offset = self.object_info.size as i64 + offset as i64;
}
_ => {
return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
}
}
self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
self.curr_offset = new_offset;
Ok(self.curr_offset)
}
fn close(&mut self) -> Result<(), std::io::Error> {
self.is_closed = true;
Ok(())
}
}

127
ecstore/src/client/api_get_options.rs Normal file
View File

@@ -0,0 +1,127 @@
#![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, HeaderName, HeaderValue};
use time::OffsetDateTime;
use tracing::warn;
use crate::client::api_error_response::err_invalid_argument;
#[derive(Default)]
pub struct AdvancedGetOptions {
replication_deletemarker: bool,
is_replication_ready_for_deletemarker: bool,
replication_proxy_request: String,
}
pub struct GetObjectOptions {
pub headers: HashMap<String, String>,
pub req_params: HashMap<String, String>,
//pub server_side_encryption: encrypt.ServerSide,
pub version_id: String,
pub part_number: i64,
pub checksum: bool,
pub internal: AdvancedGetOptions,
}
type StatObjectOptions = GetObjectOptions;
impl Default for GetObjectOptions {
fn default() -> Self {
Self {
headers: HashMap::new(),
req_params: HashMap::new(),
//server_side_encryption: encrypt.ServerSide::default(),
version_id: "".to_string(),
part_number: 0,
checksum: false,
internal: AdvancedGetOptions::default(),
}
}
}
impl GetObjectOptions {
pub fn header(&self) -> HeaderMap {
let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
for (k, v) in &self.headers {
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
headers.insert(header_name, v.parse().expect("err"));
} else {
warn!("Invalid header name: {}", k);
}
}
if self.checksum {
headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
}
headers
}
pub fn set(&self, key: &str, value: &str) {
//self.headers[http.CanonicalHeaderKey(key)] = value;
}
pub fn set_req_param(&mut self, key: &str, value: &str) {
self.req_params.insert(key.to_string(), value.to_string());
}
pub fn add_req_param(&mut self, key: &str, value: &str) {
self.req_params.insert(key.to_string(), value.to_string());
}
pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
self.set("If-Match", &format!("\"{etag}\""));
Ok(())
}
pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
self.set("If-None-Match", &format!("\"{etag}\""));
Ok(())
}
pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
if mod_time.unix_timestamp() == 0 {
return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
}
self.set("If-Unmodified-Since", &mod_time.to_string());
Ok(())
}
pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
if mod_time.unix_timestamp() == 0 {
return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
}
self.set("If-Modified-Since", &mod_time.to_string());
Ok(())
}
pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
if start == 0 && end < 0 {
self.set("Range", &format!("bytes={}", end));
}
else if 0 < start && end == 0 {
self.set("Range", &format!("bytes={}-", start));
}
else if 0 <= start && start <= end {
self.set("Range", &format!("bytes={}-{}", start, end));
}
else {
return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end))));
}
Ok(())
}
pub fn to_query_values(&self) -> HashMap<String, String> {
let mut url_values = HashMap::new();
if self.version_id != "" {
url_values.insert("versionId".to_string(), self.version_id.clone());
}
if self.part_number > 0 {
url_values.insert("partNumber".to_string(), self.part_number.to_string());
}
for (key, value) in self.req_params.iter() {
url_values.insert(key.to_string(), value.to_string());
}
url_values
}
}

229
ecstore/src/client/api_list.rs Normal file
View File

@@ -0,0 +1,229 @@
#![allow(clippy::map_entry)]
use std::collections::HashMap;
use bytes::Bytes;
use http::{HeaderMap, StatusCode};
use crate::client::{
api_error_response::http_resp_to_error_response,
credentials,
api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart},
transition_api::{ReaderImpl, TransitionClient, RequestMetadata,},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::store_api::BucketInfo;
impl TransitionClient {
pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
todo!();
}
pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketV2Result, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("list-type".to_string(), "2".to_string());
if metadata {
url_values.insert("metadata".to_string(), "true".to_string());
}
if start_after != "" {
url_values.insert("start-after".to_string(), start_after.to_string());
}
url_values.insert("encoding-type".to_string(), "url".to_string());
url_values.insert("prefix".to_string(), object_prefix.to_string());
url_values.insert("delimiter".to_string(), delimiter.to_string());
if continuation_token != "" {
url_values.insert("continuation-token".to_string(), continuation_token.to_string());
}
if fetch_owner {
url_values.insert("fetch-owner".to_string(), "true".to_string());
}
if max_keys > 0 {
url_values.insert("max-keys".to_string(), max_keys.to_string());
}
let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: "".to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers,
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
}
//let mut list_bucket_result = ListBucketV2Result::default();
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
Ok(result) => result,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
//println!("list_bucket_result: {:?}", list_bucket_result);
if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
return Err(std::io::Error::other(credentials::ErrorResponse {
sts_error: credentials::STSError {
r#type: "".to_string(),
code: "NotImplemented".to_string(),
message: "Truncated response should have continuation token set".to_string(),
},
request_id: "".to_string(),
}));
}
for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
//list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
}
for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
}
Ok(list_bucket_result)
}
pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result<ListVersionsResult, std::io::Error> {
/*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListVersionsResult{}, err
}
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
return ListVersionsResult{}, err
}
urlValues := make(url.Values)
urlValues.Set("versions", "")
urlValues.Set("prefix", opts.Prefix)
urlValues.Set("delimiter", delimiter)
if keyMarker != "" {
urlValues.Set("key-marker", keyMarker)
}
if opts.max_keys > 0 {
urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
}
if versionIDMarker != "" {
urlValues.Set("version-id-marker", versionIDMarker)
}
if opts.WithMetadata {
urlValues.Set("metadata", "true")
}
urlValues.Set("encoding-type", "url")
let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
customHeader: opts.headers,
}).await?;
defer closeResponse(resp)
if err != nil {
return ListVersionsResult{}, err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
}
}
listObjectVersionsOutput := ListVersionsResult{}
err = xml_decoder(resp.Body, &listObjectVersionsOutput)
if err != nil {
return ListVersionsResult{}, err
}
for i, obj := range listObjectVersionsOutput.Versions {
listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
if err != nil {
return listObjectVersionsOutput, err
}
}
for i, obj := range listObjectVersionsOutput.CommonPrefixes {
listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
if err != nil {
return listObjectVersionsOutput, err
}
}
if listObjectVersionsOutput.NextKeyMarker != "" {
listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
if err != nil {
return listObjectVersionsOutput, err
}
}
Ok(listObjectVersionsOutput)*/
todo!();
}
pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketResult, std::io::Error> {
todo!();
}
pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> {
todo!();
}
pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
todo!();
}
pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
todo!();
}
pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> {
todo!();
}
}
pub struct ListObjectsOptions {
reverse_versions: bool,
with_versions: bool,
with_metadata: bool,
prefix: String,
recursive: bool,
max_keys: i64,
start_after: String,
use_v1: bool,
headers: HeaderMap,
}
impl ListObjectsOptions {
pub fn set(&mut self, key: &str, value: &str) {
todo!();
}
}
fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
match encoding_type {
"url" => {
//return url::QueryUnescape(name);
return Ok(name.to_string());
}
_ => {
return Ok(name.to_string());
}
}
}

382
ecstore/src/client/api_put_object.rs Normal file
View File

@@ -0,0 +1,382 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use time::{OffsetDateTime, macros::format_description, Duration};
use http::{HeaderMap, HeaderName, HeaderValue};
use tracing::{error, info, warn};
use s3s::dto::{
ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
ReplicationStatus,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, X_AMZ_REPLICATION_STATUS,
};
use reader::hasher::Hasher;
//use crate::disk::{BufferReader, Reader};
use rustfs_utils::{
crypto::base64_encode,
net::{is_amz_header, is_standard_header, is_storageclass_header, is_rustfs_header, is_minio_header},
};
use crate::checksum::ChecksumMode;
use crate::client::{
api_s3_datatypes::{CompletePart, ObjectPart, CompleteMultipartUpload},
api_put_object_common::optimal_part_info,
transition_api::{TransitionClient, UploadInfo, ReaderImpl},
api_error_response::{err_invalid_argument, err_entity_too_large},
api_put_object_multipart::UploadPartParams,
credentials::SignatureType,
constants::{MAX_MULTIPART_PUT_OBJECT_SIZE, TOTAL_WORKERS, MIN_PART_SIZE, ISO8601_DATEFORMAT,},
};
#[derive(Debug, Clone)]
pub struct AdvancedPutOptions {
pub source_version_id: String,
pub source_etag: String,
pub replication_status: ReplicationStatus,
pub source_mtime: OffsetDateTime,
pub replication_request: bool,
pub retention_timestamp: OffsetDateTime,
pub tagging_timestamp: OffsetDateTime,
pub legalhold_timestamp: OffsetDateTime,
pub replication_validity_check: bool,
}
impl Default for AdvancedPutOptions {
fn default() -> Self {
Self {
source_version_id: "".to_string(),
source_etag: "".to_string(),
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
source_mtime: OffsetDateTime::now_utc(),
replication_request: false,
retention_timestamp: OffsetDateTime::now_utc(),
tagging_timestamp: OffsetDateTime::now_utc(),
legalhold_timestamp: OffsetDateTime::now_utc(),
replication_validity_check: false,
}
}
}
#[derive(Clone)]
pub struct PutObjectOptions {
pub user_metadata: HashMap<String, String>,
pub user_tags: HashMap<String, String>,
//pub progress: ReaderImpl,
pub content_type: String,
pub content_encoding: String,
pub content_disposition: String,
pub content_language: String,
pub cache_control: String,
pub expires: OffsetDateTime,
pub mode: ObjectLockRetentionMode,
pub retain_until_date: OffsetDateTime,
//pub server_side_encryption: encrypt.ServerSide,
pub num_threads: u64,
pub storage_class: String,
pub website_redirect_location: String,
pub part_size: u64,
pub legalhold: ObjectLockLegalHoldStatus,
pub send_content_md5: bool,
pub disable_content_sha256: bool,
pub disable_multipart: bool,
pub auto_checksum: ChecksumMode,
pub checksum: ChecksumMode,
pub concurrent_stream_parts: bool,
pub internal: AdvancedPutOptions,
pub custom_header: HeaderMap,
}
impl Default for PutObjectOptions {
fn default() -> Self {
Self {
user_metadata: HashMap::new(),
user_tags: HashMap::new(),
//progress: ReaderImpl::Body(Bytes::new()),
content_type: "".to_string(),
content_encoding: "".to_string(),
content_disposition: "".to_string(),
content_language: "".to_string(),
cache_control: "".to_string(),
expires: OffsetDateTime::UNIX_EPOCH,
mode: ObjectLockRetentionMode::from_static(""),
retain_until_date: OffsetDateTime::UNIX_EPOCH,
//server_side_encryption: encrypt.ServerSide::default(),
num_threads: 0,
storage_class: "".to_string(),
website_redirect_location: "".to_string(),
part_size: 0,
legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
send_content_md5: false,
disable_content_sha256: false,
disable_multipart: false,
auto_checksum: ChecksumMode::ChecksumNone,
checksum: ChecksumMode::ChecksumNone,
concurrent_stream_parts: false,
internal: AdvancedPutOptions::default(),
custom_header: HeaderMap::new(),
}
}
}
impl PutObjectOptions {
fn set_matche_tag(&mut self, etag: &str) {
if etag == "*" {
self.custom_header.insert("If-Match", HeaderValue::from_str("*").expect("err"));
} else {
self.custom_header.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
}
}
fn set_matche_tag_except(&mut self, etag: &str) {
if etag == "*" {
self.custom_header.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
} else {
self.custom_header.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
}
}
pub fn header(&self) -> HeaderMap {
let mut header = HeaderMap::new();
let mut content_type = self.content_type.clone();
if content_type == "" {
content_type = "application/octet-stream".to_string();
}
header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
if self.content_encoding != "" {
header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
}
if self.content_disposition != "" {
header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
}
if self.content_language != "" {
header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
}
if self.cache_control != "" {
header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
}
if self.expires.unix_timestamp() != 0 {
header.insert("Expires", HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); //rustfs invalid heade
}
if self.mode.as_str() != "" {
header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
}
if self.retain_until_date.unix_timestamp() != 0 {
header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"));
}
if self.legalhold.as_str() != "" {
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
}
if self.storage_class != "" {
header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
}
if self.website_redirect_location != "" {
header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, HeaderValue::from_str(&self.website_redirect_location).expect("err"));
}
if !self.internal.replication_status.as_str().is_empty() {
header.insert(X_AMZ_REPLICATION_STATUS, HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"));
}
for (k, v) in &self.user_metadata {
if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) {
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
header.insert(header_name, HeaderValue::from_str(&v).unwrap());
}
} else {
if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
header.insert(header_name, HeaderValue::from_str(&v).unwrap());
}
}
}
for (k, v) in self.custom_header.iter() {
header.insert(k.clone(), v.clone());
}
header
}
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
//if self.checksum.is_set() {
/*if !self.trailing_header_support {
return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
}*/
/*else if self.override_signer_type == SignatureType::SignatureV2 {
return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
}*/
//}
Ok(())
}
}
impl TransitionClient {
pub async fn put_object(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64,
opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
if object_size < 0 && opts.disable_multipart {
return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
}
self.put_object_common(bucket_name, object_name, reader, object_size, opts).await
}
pub async fn put_object_common(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name)));
}
let mut opts = opts.clone();
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
let mut part_size = opts.part_size as i64;
if opts.part_size == 0 {
part_size = MIN_PART_SIZE;
}
if SignatureType::SignatureV2 == self.override_signer_type {
if size >= 0 && size < part_size || opts.disable_multipart {
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
}
return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
}
if size < 0 {
if opts.disable_multipart {
return Err(std::io::Error::other("no length provided and multipart disabled"));
}
if opts.concurrent_stream_parts && opts.num_threads > 1 {
return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await;
}
return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await;
}
if size <= part_size || opts.disable_multipart {
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
}
self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await
}
pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default();
let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
let mut opts = opts.clone();
if opts.checksum.is_set() {
opts.send_content_md5 = false;
opts.auto_checksum = opts.checksum.clone();
}
if !opts.send_content_md5 {
//add_auto_checksum_headers(&mut opts);
}
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
let mut part_number = 1;
let mut parts_info = HashMap::<i64, ObjectPart>::new();
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
let mut custom_header = HeaderMap::new();
while part_number <= total_parts_count {
buf = match &mut reader {
ReaderImpl::Body(content_body) => {
content_body.to_vec()
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
};
let length = buf.len();
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let mut hash = md5_hasher.as_mut().expect("err");
hash.write(&buf[..length]);
md5_base64 = base64_encode(hash.sum().as_bytes());
} else {
let csum;
{
let mut crc = opts.auto_checksum.hasher()?;
crc.reset();
crc.write(&buf[..length]);
csum = crc.sum();
}
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap());
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
}
//let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
let rd = ReaderImpl::Body(Bytes::from(buf));
let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
upload_id: upload_id.clone(),
reader: rd,
part_number,
md5_base64,
size: length as i64,
//sse: opts.server_side_encryption,
stream_sha256: !opts.disable_content_sha256,
custom_header: custom_header.clone(),
sha256_hex: Default::default(),
trailer: Default::default(),
};
let obj_part = self.upload_part(&mut p).await?;
parts_info.entry(part_number).or_insert(obj_part);
total_uploaded_size += length as i64;
part_number += 1;
}
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
for i in 1..part_number {
let part = parts_info[&i].clone();
all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart {
etag: part.etag,
part_num: part.part_num,
checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default()
});
}
compl_multipart_upload.parts.sort();
let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum,
..Default::default()
};
//apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
upload_info.size = total_uploaded_size;
Ok(upload_info)
}
}

76
ecstore/src/client/api_put_object_common.rs Normal file
View File

@@ -0,0 +1,76 @@
#![allow(clippy::map_entry)]
use crate::client::{
api_put_object::PutObjectOptions,
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE},
transition_api::TransitionClient,
transition_api::ReaderImpl,
api_error_response::{err_entity_too_large, err_invalid_argument},
};
const NULL_VERSION_ID: &str = "null";
pub fn is_object(reader: &ReaderImpl) -> bool {
todo!();
}
pub fn is_read_at(reader: ReaderImpl) -> bool {
todo!();
}
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
let unknown_size;
let mut object_size = object_size;
if object_size == -1 {
unknown_size = true;
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
} else {
unknown_size = false;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", "")));
}
let mut part_size_flt: f64;
if configured_part_size > 0 {
if configured_part_size as i64 > object_size {
return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", "")));
}
if !unknown_size {
if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize.")));
}
}
if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB.")));
}
if configured_part_size as i64 > MAX_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB.")));
}
part_size_flt = configured_part_size as f64;
if unknown_size {
object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
}
} else {
let mut configured_part_size = configured_part_size;
configured_part_size = MIN_PART_SIZE as u64;
part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
}
let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
let part_size = part_size_flt.ceil() as i64;
let last_part_size = object_size - (total_parts_count-1) * part_size;
Ok((total_parts_count, part_size, last_part_size))
}
impl TransitionClient {
pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<String, std::io::Error> {
let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
Ok(init_multipart_upload_result.upload_id)
}
}

348
ecstore/src/client/api_put_object_multipart.rs Normal file
View File

@@ -0,0 +1,348 @@
#![allow(clippy::map_entry)]
use std::io::Read;
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use s3s::S3ErrorCode;
use time::{format_description, OffsetDateTime};
use uuid::Uuid;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use url::form_urlencoded::Serializer;
use tokio_util::sync::CancellationToken;
use tracing::warn;
use tracing::{error, info};
use s3s::{dto::StreamingBlob, Body};
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use reader::hasher::Hasher;
//use crate::disk::{Reader, BufferReader};
use crate::client::{
transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,},
api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response},
api_put_object::PutObjectOptions,
api_put_object_common::optimal_part_info,
api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart},
constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, },
};
use rustfs_utils::{
path::trim_etag,
crypto::base64_encode,
};
use crate::{
disk::DiskAPI,
store_api::{
GetObjectReader, StorageAPI,
},
checksum::ChecksumMode,
};
impl TransitionClient {
pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64,
opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await;
if let Err(err) = &info {
let err_resp = to_error_response(err);
if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
if size > MAX_SINGLE_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name)));
}
return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
}
}
Ok(info?)
}
pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default();
let ret = optimal_part_info(-1, opts.part_size)?;
let (total_parts_count, part_size, _) = ret;
let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
let mut opts = opts.clone();
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
let mut part_number = 1;
let mut parts_info = HashMap::<i64, ObjectPart>::new();
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
let mut custom_header = HeaderMap::new();
while part_number <= total_parts_count {
match reader {
ReaderImpl::Body(content_body) => {
buf = content_body.to_vec();
}
ReaderImpl::ObjectBody(content_body) => {
buf = content_body.read_all().await?;
}
}
let length = buf.len();
for (k, v) in hash_algos.iter_mut() {
v.write(&buf[..length]);
hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap());
}
//let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
let rd = Bytes::from(buf.clone());
let mut md5_base64: String;
let mut sha256_hex: String;
//if hash_sums["md5"] != nil {
md5_base64 = base64_encode(&hash_sums["md5"]);
//}
//if hash_sums["sha256"] != nil {
sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
//}
if hash_sums.len() == 0 {
let csum;
{
let mut crc = opts.auto_checksum.hasher()?;
crc.reset();
crc.write(&buf[..length]);
csum = crc.sum();
}
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err"));
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
}
let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
upload_id: upload_id.clone(),
reader: ReaderImpl::Body(rd),
part_number,
md5_base64,
sha256_hex,
size: length as i64,
//sse: opts.server_side_encryption,
stream_sha256: !opts.disable_content_sha256,
custom_header: custom_header.clone(),
trailer: HeaderMap::new(),
};
let obj_part = self.upload_part(&mut p).await?;
parts_info.insert(part_number, obj_part);
total_uploaded_size += length as i64;
part_number += 1;
}
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
for i in 1..part_number {
let part = parts_info[&i].clone();
all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart {
etag: part.etag,
part_num: part.part_num,
checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default()
});
}
compl_multipart_upload.parts.sort();
let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum,
..Default::default()
};
//apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
upload_info.size = total_uploaded_size;
Ok(upload_info)
}
pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<InitiateMultipartUploadResult, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("uploads".to_string(), "".to_string());
if opts.internal.source_version_id != "" {
if !opts.internal.source_version_id.is_empty() {
if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
}
}
url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
}
let mut custom_header = opts.header();
let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
custom_header,
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
content_sha256_hex: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
};
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
//if resp.is_none() {
if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
}
//}
let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
Ok(initiate_multipart_upload_result)
}
pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
if p.size > MAX_PART_SIZE {
return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name)));
}
if p.size <= -1 {
return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
}
if p.part_number <= 0 {
return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero.")));
}
if p.upload_id == "" {
return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
}
let mut url_values = HashMap::new();
url_values.insert("partNumber".to_string(), p.part_number.to_string());
url_values.insert("uploadId".to_string(), p.upload_id.clone());
let buf = match &mut p.reader {
ReaderImpl::Body(content_body) => {
content_body.to_vec()
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
};
let mut req_metadata = RequestMetadata {
bucket_name: p.bucket_name.clone(),
object_name: p.object_name.clone(),
query_values: url_values,
custom_header: p.custom_header.clone(),
content_body: ReaderImpl::Body(Bytes::from(buf)),
content_length: p.size,
content_md5_base64: p.md5_base64.clone(),
content_sha256_hex: p.sha256_hex.clone(),
stream_sha256: p.stream_sha256,
trailer: p.trailer.clone(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
};
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
//defer closeResponse(resp)
//if resp.is_none() {
if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name)));
}
//}
let h = resp.headers();
let mut obj_part = ObjectPart {
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() },
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() },
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() },
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() },
..Default::default()
};
obj_part.size = p.size;
obj_part.part_num = p.part_number;
obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() };
Ok(obj_part)
}
pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str,
complete: CompleteMultipartUpload, opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string());
let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
let mut headers = opts.header();
let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
content_length: 100,//complete_multipart_upload_bytes.len(),
content_sha256_hex: "".to_string(),//hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
custom_header: headers,
stream_sha256: Default::default(),
trailer: Default::default(),
content_md5_base64: "".to_string(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
};
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
let b = resp.body().bytes().expect("err").to_vec();
let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
(
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string()
)
} else {
(OffsetDateTime::now_utc(), "".to_string())
};
let h = resp.headers();
Ok(UploadInfo {
bucket: complete_multipart_upload_result.bucket,
key: complete_multipart_upload_result.key,
etag: trim_etag(&complete_multipart_upload_result.etag),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() },
location: complete_multipart_upload_result.location,
expiration: exp_time,
expiration_rule_id: rule_id,
checksum_sha256: complete_multipart_upload_result.checksum_sha256,
checksum_sha1: complete_multipart_upload_result.checksum_sha1,
checksum_crc32: complete_multipart_upload_result.checksum_crc32,
checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
..Default::default()
})
}
}
pub struct UploadPartParams {
pub bucket_name: String,
pub object_name: String,
pub upload_id: String,
pub reader: ReaderImpl,
pub part_number: i64,
pub md5_base64: String,
pub sha256_hex: String,
pub size: i64,
//pub sse: encrypt.ServerSide,
pub stream_sha256: bool,
pub custom_header: HeaderMap,
pub trailer: HeaderMap,
}

447
ecstore/src/client/api_put_object_streaming.rs Normal file
View File

@@ -0,0 +1,447 @@
#![allow(clippy::map_entry)]
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use time::{format_description, OffsetDateTime};
use tokio::{select, sync::mpsc};
use tokio_util::sync::CancellationToken;
use tracing::warn;
use uuid::Uuid;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use reader::hasher::Hasher;
use crate::client::{
constants::ISO8601_DATEFORMAT,
api_put_object::PutObjectOptions,
api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl},
api_put_object_common::{is_object, optimal_part_info,},
api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof},
};
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode};
pub struct UploadedPartRes {
pub error: std::io::Error,
pub part_num: i64,
pub size: i64,
pub part: ObjectPart,
}
pub struct UploadPartReq {
pub part_num: i64,
pub part: ObjectPart,
}
impl TransitionClient {
pub async fn put_object_multipart_stream(self: Arc<Self>, bucket_name: &str, object_name: &str,
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let info: UploadInfo;
if opts.concurrent_stream_parts && opts.num_threads > 1 {
info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?;
} else if !is_object(&reader) && !opts.send_content_md5 {
info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?;
} else {
info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?;
}
Ok(info)
}
pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str,
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let ret = optimal_part_info(size, opts.part_size)?;
let (total_parts_count, part_size, lastpart_size) = ret;
let mut opts = opts.clone();
if opts.checksum.is_set() {
opts.auto_checksum = opts.checksum.clone();
}
let with_checksum = self.trailing_header_support;
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
todo!();
}
pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str,
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone();
if opts.checksum.is_set() {
opts.auto_checksum = opts.checksum.clone();
opts.send_content_md5 = false;
}
if !opts.send_content_md5 {
add_auto_checksum_headers(&mut opts);
}
let ret = optimal_part_info(size, opts.part_size)?;
let (total_parts_count, mut part_size, lastpart_size) = ret;
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
let mut custom_header = opts.header().clone();
let mut total_uploaded_size: i64 = 0;
let mut parts_info = HashMap::<i64, ObjectPart>::new();
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
let mut md5_base64: String = "".to_string();
for part_number in 1..=total_parts_count {
if part_number == total_parts_count {
part_size = lastpart_size;
}
match &mut reader {
ReaderImpl::Body(content_body) => {
buf = content_body.to_vec();
}
ReaderImpl::ObjectBody(content_body) => {
buf = content_body.read_all().await?;
}
}
let length = buf.len();
if opts.send_content_md5 {
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let mut md5_hash = md5_hasher.as_mut().expect("err");
md5_hash.reset();
md5_hash.write(&buf[..length]);
md5_base64 = base64_encode(md5_hash.sum().as_bytes());
} else {
let csum;
{
let mut crc = opts.auto_checksum.hasher()?;
crc.reset();
crc.write(&buf[..length]);
csum = crc.sum();
}
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) {
custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err"));
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized());
}
}
let hooked = ReaderImpl::Body(Bytes::from(buf));//newHook(BufferReader::new(buf), opts.progress);
let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
upload_id: upload_id.clone(),
reader: hooked,
part_number,
md5_base64: md5_base64.clone(),
size: part_size,
//sse: opts.server_side_encryption,
stream_sha256: !opts.disable_content_sha256,
custom_header: custom_header.clone(),
sha256_hex: "".to_string(),
trailer: HeaderMap::new(),
};
let obj_part = self.upload_part(&mut p).await?;
parts_info.entry(part_number).or_insert(obj_part);
total_uploaded_size += part_size as i64;
}
if size > 0 {
if total_uploaded_size != size {
return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name)));
}
}
let mut compl_multipart_upload = CompleteMultipartUpload::default();
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
let part_number = total_parts_count;
for i in 1..part_number {
let part = parts_info[&i].clone();
all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart {
etag: part.etag,
part_num: part.part_num,
checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme,
});
}
compl_multipart_upload.parts.sort();
let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum,
..Default::default()
};
apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
upload_info.size = total_uploaded_size;
Ok(upload_info)
}
pub async fn put_object_multipart_stream_parallel(self: Arc<Self>, bucket_name: &str, object_name: &str,
mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions
) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone();
if opts.checksum.is_set() {
opts.send_content_md5 = false;
opts.auto_checksum = opts.checksum.clone();
}
if !opts.send_content_md5 {
add_auto_checksum_headers(&mut opts);
}
let ret = optimal_part_info(-1, opts.part_size)?;
let (total_parts_count, part_size, _) = ret;
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
let mut total_uploaded_size: i64 = 0;
let mut parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
let n_buffers = opts.num_threads;
let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
//let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
for i in 0..n_buffers {
//bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
}
let mut futures = Vec::with_capacity(total_parts_count as usize);
let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
let cancel_token = CancellationToken::new();
//reader = newHook(reader, opts.progress);
for part_number in 1..=total_parts_count {
let mut buf = Vec::<u8>::new();
select! {
buf = bufs_rx.recv() => {}
err = err_rx.recv() => {
//cancel_token.cancel();
//wg.Wait()
return Err(err.expect("err"));
}
else => (),
}
if buf.len() != part_size as usize {
return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size)));
}
match &mut reader {
ReaderImpl::Body(content_body) => {
buf = content_body.to_vec();
}
ReaderImpl::ObjectBody(content_body) => {
buf = content_body.read_all().await?;
}
}
let length = buf.len();
let mut custom_header = HeaderMap::new();
if !opts.send_content_md5 {
let csum;
{
let mut crc = opts.auto_checksum.hasher()?;
crc.reset();
crc.write(&buf[..length]);
csum = crc.sum();
}
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) {
custom_header.insert(header_name, header_value);
}
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
}
let clone_bufs_tx = bufs_tx.clone();
let clone_parts_info = parts_info.clone();
let clone_upload_id = upload_id.clone();
let clone_self = self.clone();
futures.push(async move {
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
let mut md5_hash = md5_hasher.as_mut().expect("err");
md5_hash.write(&buf[..length]);
md5_base64 = base64_encode(md5_hash.sum().as_bytes());
}
//defer wg.Done()
let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
upload_id: clone_upload_id,
reader: ReaderImpl::Body(Bytes::from(buf.clone())),
part_number,
md5_base64,
size: length as i64,
//sse: opts.server_side_encryption,
stream_sha256: !opts.disable_content_sha256,
custom_header,
sha256_hex: "".to_string(),
trailer: HeaderMap::new(),
};
let obj_part = clone_self.upload_part(&mut p).await.expect("err");
let mut clone_parts_info = clone_parts_info.write().unwrap();
clone_parts_info.entry(part_number).or_insert(obj_part);
clone_bufs_tx.send(buf);
});
total_uploaded_size += length as i64;
}
let results = join_all(futures).await;
select! {
err = err_rx.recv() => {
return Err(err.expect("err"));
}
else => (),
}
let mut compl_multipart_upload = CompleteMultipartUpload::default();
let mut part_number: i64 = total_parts_count;
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.read().unwrap().len());
for i in 1..part_number {
let part = parts_info.read().unwrap()[&i].clone();
all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart {
etag: part.etag,
part_num: part.part_num,
checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default()
});
}
compl_multipart_upload.parts.sort();
let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum,
..Default::default()
};
apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?;
upload_info.size = total_uploaded_size;
Ok(upload_info)
}
pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone();
if opts.checksum.is_set() {
opts.send_content_md5 = false;
}
let mut md5_base64: String = "".to_string();
let progress_reader = reader;//newHook(reader, opts.progress);
self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await
}
pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let custom_header = opts.header();
let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
custom_header,
content_body: reader,
content_length: size,
content_md5_base64: md5_base64.to_string(),
content_sha256_hex: sha256_hex.to_string(),
stream_sha256: !opts.disable_content_sha256,
add_crc: Default::default(),
bucket_location: Default::default(),
pre_sign_url: Default::default(),
query_values: Default::default(),
extra_pre_sign_header: Default::default(),
expires: Default::default(),
trailer: Default::default(),
};
let mut add_crc = false;//self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
let mut opts = opts.clone();
if opts.checksum.is_set() {
req_metadata.add_crc = opts.checksum;
} else if add_crc {
for (k, _) in opts.user_metadata {
if k.to_lowercase().starts_with("x-amz-checksum-") {
add_crc = false;
}
}
if add_crc {
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
req_metadata.add_crc = opts.auto_checksum;
}
}
if opts.internal.source_version_id != "" {
if !opts.internal.source_version_id.is_empty() {
if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
}
}
let mut url_values = HashMap::new();
url_values.insert("versionId".to_string(), opts.internal.source_version_id);
req_metadata.query_values = url_values;
}
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
}
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
(
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string()
)
} else {
(OffsetDateTime::now_utc(), "".to_string())
};
let h = resp.headers();
Ok(UploadInfo {
bucket: bucket_name.to_string(),
key: object_name.to_string(),
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() },
size: size,
expiration: exp_time,
expiration_rule_id: rule_id,
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() },
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() },
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() },
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() },
..Default::default()
})
}
}

387
ecstore/src/client/api_remove.rs Normal file
View File

@@ -0,0 +1,387 @@
#![allow(clippy::map_entry)]
use std::fmt::Display;
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use s3s::S3ErrorCode;
use time::OffsetDateTime;
use s3s::dto::ReplicationStatus;
use tokio::sync::mpsc::{self, Receiver, Sender};
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use reader::hasher::{sum_sha256_hex, sum_md5_base64};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::{
disk::DiskAPI,
store_api::{
GetObjectReader, ObjectInfo, StorageAPI,
},
};
use crate::client::{
transition_api::{TransitionClient, RequestMetadata, ReaderImpl},
api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,},
};
struct RemoveBucketOptions {
forced_elete: bool,
}
#[derive(Debug)]
pub struct AdvancedRemoveOptions {
replication_delete_marker: bool,
replication_status: ReplicationStatus,
replication_mtime: OffsetDateTime,
replication_request: bool,
replication_validity_check: bool,
}
impl Default for AdvancedRemoveOptions {
fn default() -> Self {
Self {
replication_delete_marker: false,
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
replication_mtime: OffsetDateTime::now_utc(),
replication_request: false,
replication_validity_check: false,
}
}
}
#[derive(Debug, Default)]
pub struct RemoveObjectOptions {
pub force_delete: bool,
pub governance_bypass: bool,
pub version_id: String,
pub internal: AdvancedRemoveOptions,
}
impl TransitionClient {
pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
let mut headers = HeaderMap::new();
/*if opts.force_delete {
headers.insert(rustFSForceDelete, "true");
}*/
let resp = self.execute_method(Method::DELETE, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers,
object_name: "".to_string(),
query_values: Default::default(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.delete(bucket_name);
}
Ok(())
}
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: Default::default(),
object_name: "".to_string(),
query_values: Default::default(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.delete(bucket_name);
}
Ok(())
}
pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
res.err
}
pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result<RemoveObjectResult, std::io::Error> {
let mut url_values = HashMap::new();
if opts.version_id != "" {
url_values.insert("versionId".to_string(), opts.version_id.clone());
}
let mut headers = HeaderMap::new();
if opts.governance_bypass {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));//amzBypassGovernance
}
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
query_values: url_values,
custom_header: headers,
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
Ok(RemoveObjectResult {
object_name: object_name.to_string(),
object_version_id: opts.version_id,
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(),
..Default::default()
})
}
pub async fn remove_objects_with_result(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectResult> {
let (result_tx, mut result_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self);
let bucket_name_owned = bucket_name.to_string();
tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await;
});
result_rx
}
pub async fn remove_objects(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectError> {
let (error_tx, mut error_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self);
let bucket_name_owned = bucket_name.to_string();
let (result_tx, mut result_rx) = mpsc::channel(1);
tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await;
});
tokio::spawn(async move {
while let Some(res) = result_rx.recv().await {
if res.err.is_none() {
continue;
}
error_tx.send(RemoveObjectError {
object_name: res.object_name,
version_id: res.object_version_id,
err: res.err,
..Default::default()
}).await;
}
});
error_rx
}
pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver<ObjectInfo>, result_tx: &Sender<RemoveObjectResult>, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> {
let max_entries = 1000;
let mut finish = false;
let mut url_values = HashMap::new();
url_values.insert("delete".to_string(), "".to_string());
loop {
if finish {
break;
}
let mut count = 0;
let mut batch = Vec::<ObjectInfo>::new();
while let Some(object) = objects_rx.recv().await {
if has_invalid_xml_char(&object.name) {
let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions {
version_id: object.version_id.expect("err").to_string(),
governance_bypass: opts.governance_bypass,
..Default::default()
}).await?;
let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() {
match to_error_response(&remove_result.err.expect("err")).code {
S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
continue;
}
_ => (),
}
result_tx.send(remove_result_clone.clone()).await;
}
result_tx.send(remove_result_clone).await;
continue;
}
batch.push(object);
count += 1;
if count >= max_entries {
break;
}
}
if count == 0 {
break;
}
if count < max_entries {
finish = true;
}
let mut headers = HeaderMap::new();
if opts.governance_bypass {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
}
let remove_bytes = generate_remove_multi_objects_request(&batch);
let resp = self.execute_method(http::Method::POST, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values.clone(),
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
content_length: remove_bytes.len() as i64,
content_md5_base64: sum_md5_base64(&remove_bytes),
content_sha256_hex: sum_sha256_hex(&remove_bytes),
custom_header: headers,
object_name: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
}
Ok(())
}
pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
for upload_id in upload_ids {
self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
}
Ok(())
}
pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: HeaderMap::new(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
}).await?;
//if resp.is_some() {
if resp.status() != StatusCode::NO_CONTENT {
let error_response: ErrorResponse;
match resp.status() {
StatusCode::NOT_FOUND => {
error_response = ErrorResponse {
code: S3ErrorCode::NoSuchUpload,
message: "The specified multipart upload does not exist.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
request_id: resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(),
host_id: resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(),
region: resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(),
..Default::default()
};
}
_ => {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
}
}
return Err(std::io::Error::other(error_response));
}
//}
Ok(())
}
}
#[derive(Debug, Default)]
struct RemoveObjectError {
object_name: String,
version_id: String,
err: Option<std::io::Error>,
}
impl Display for RemoveObjectError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
if self.err.is_none() {
return write!(f, "unexpected remove object error result");
}
write!(f, "{}", self.err.as_ref().expect("err").to_string())
}
}
#[derive(Debug, Default)]
pub struct RemoveObjectResult {
pub object_name: String,
pub object_version_id: String,
pub delete_marker: bool,
pub delete_marker_version_id: String,
pub err: Option<std::io::Error>,
}
impl Clone for RemoveObjectResult {
fn clone(&self) -> Self {
Self {
object_name: self.object_name.clone(),
object_version_id: self.object_version_id.clone(),
delete_marker: self.delete_marker,
delete_marker_version_id: self.delete_marker_version_id.clone(),
err: None, //err
}
}
}
pub struct RemoveObjectsOptions {
pub governance_bypass: bool,
}
pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
todo!();
}
pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
todo!();
}
fn has_invalid_xml_char(str: &str) -> bool {
false
}

336
ecstore/src/client/api_s3_datatypes.rs Normal file
View File

@@ -0,0 +1,336 @@
#![allow(clippy::map_entry)]
use std::collections::HashMap;
use s3s::dto::Owner;
use serde::{Serialize, Deserialize};
use time::OffsetDateTime;
use rustfs_utils::crypto::base64_decode;
use crate::checksum::ChecksumMode;
use crate::client::transition_api::ObjectMultipartInfo;
use super::transition_api;
pub struct ListAllMyBucketsResult {
pub owner: Owner,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CommonPrefix {
pub prefix: String,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(default, rename_all = "PascalCase")]
pub struct ListBucketV2Result {
pub common_prefixes: Vec<CommonPrefix>,
pub contents: Vec<transition_api::ObjectInfo>,
pub delimiter: String,
pub encoding_type: String,
pub is_truncated: bool,
pub max_keys: i64,
pub name: String,
pub next_continuation_token: String,
pub continuation_token: String,
pub prefix: String,
pub fetch_owner: String,
pub start_after: String,
}
pub struct Version {
etag: String,
is_latest: bool,
key: String,
last_modified: OffsetDateTime,
owner: Owner,
size: i64,
storage_class: String,
version_id: String,
user_metadata: HashMap<String, String>,
user_tags: HashMap<String, String>,
is_delete_marker: bool,
}
pub struct ListVersionsResult {
versions: Vec<Version>,
common_prefixes: Vec<CommonPrefix>,
name: String,
prefix: String,
delimiter: String,
max_keys: i64,
encoding_type: String,
is_truncated: bool,
key_marker: String,
version_id_marker: String,
next_key_marker: String,
next_version_id_marker: String,
}
impl ListVersionsResult {
fn unmarshal_xml() -> Result<(), std::io::Error> {
todo!();
}
}
pub struct ListBucketResult {
common_prefixes: Vec<CommonPrefix>,
contents: Vec<transition_api::ObjectInfo>,
delimiter: String,
encoding_type: String,
is_truncated: bool,
marker: String,
max_keys: i64,
name: String,
next_marker: String,
prefix: String,
}
pub struct ListMultipartUploadsResult {
bucket: String,
key_marker: String,
upload_id_marker: String,
next_key_marker: String,
next_upload_id_marker: String,
encoding_type: String,
max_uploads: i64,
is_truncated: bool,
uploads: Vec<ObjectMultipartInfo>,
prefix: String,
delimiter: String,
common_prefixes: Vec<CommonPrefix>,
}
pub struct Initiator {
id: String,
display_name: String,
}
pub struct CopyObjectResult {
pub etag: String,
pub last_modified: OffsetDateTime,
}
#[derive(Debug, Clone)]
pub struct ObjectPart {
pub etag: String,
pub part_num: i64,
pub last_modified: OffsetDateTime,
pub size: i64,
pub checksum_crc32: String,
pub checksum_crc32c: String,
pub checksum_sha1: String,
pub checksum_sha256: String,
pub checksum_crc64nvme: String,
}
impl Default for ObjectPart {
fn default() -> Self {
ObjectPart {
etag: Default::default(),
part_num: 0,
last_modified: OffsetDateTime::now_utc(),
size: 0,
checksum_crc32: Default::default(),
checksum_crc32c: Default::default(),
checksum_sha1: Default::default(),
checksum_sha256: Default::default(),
checksum_crc64nvme: Default::default(),
}
}
}
impl ObjectPart {
fn checksum(&self, t: &ChecksumMode) -> String {
match t {
ChecksumMode::ChecksumCRC32C => {
return self.checksum_crc32c.clone();
}
ChecksumMode::ChecksumCRC32 => {
return self.checksum_crc32.clone();
}
ChecksumMode::ChecksumSHA1 => {
return self.checksum_sha1.clone();
}
ChecksumMode::ChecksumSHA256 => {
return self.checksum_sha256.clone();
}
ChecksumMode::ChecksumCRC64NVME => {
return self.checksum_crc64nvme.clone();
}
_ => {
return "".to_string();
}
}
}
pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
let b = self.checksum(t);
if b == "" {
return Err(std::io::Error::other("no checksum set"));
}
let decoded = match base64_decode(b.as_bytes()) {
Ok(b) => b,
Err(e) => return Err(std::io::Error::other(e)),
};
if decoded.len() != t.raw_byte_len() as usize {
return Err(std::io::Error::other("checksum length mismatch"));
}
Ok(decoded)
}
}
pub struct ListObjectPartsResult {
pub bucket: String,
pub key: String,
pub upload_id: String,
pub initiator: Initiator,
pub owner: Owner,
pub storage_class: String,
pub part_number_marker: i32,
pub next_part_number_marker: i32,
pub max_parts: i32,
pub checksum_algorithm: String,
pub checksum_type: String,
pub is_truncated: bool,
pub object_parts: Vec<ObjectPart>,
pub encoding_type: String,
}
#[derive(Debug, Default)]
pub struct InitiateMultipartUploadResult {
pub bucket: String,
pub key: String,
pub upload_id: String,
}
#[derive(Debug, Default)]
pub struct CompleteMultipartUploadResult {
pub location: String,
pub bucket: String,
pub key: String,
pub etag: String,
pub checksum_crc32: String,
pub checksum_crc32c: String,
pub checksum_sha1: String,
pub checksum_sha256: String,
pub checksum_crc64nvme: String,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
#[derive(serde::Serialize)]
pub struct CompletePart { //api has
pub etag: String,
pub part_num: i64,
pub checksum_crc32: String,
pub checksum_crc32c: String,
pub checksum_sha1: String,
pub checksum_sha256: String,
pub checksum_crc64nvme: String,
}
impl CompletePart {
fn checksum(&self, t: &ChecksumMode) -> String {
match t {
ChecksumMode::ChecksumCRC32C => {
return self.checksum_crc32c.clone();
}
ChecksumMode::ChecksumCRC32 => {
return self.checksum_crc32.clone();
}
ChecksumMode::ChecksumSHA1 => {
return self.checksum_sha1.clone();
}
ChecksumMode::ChecksumSHA256 => {
return self.checksum_sha256.clone();
}
ChecksumMode::ChecksumCRC64NVME => {
return self.checksum_crc64nvme.clone();
}
_ => {
return "".to_string();
}
}
}
}
pub struct CopyObjectPartResult {
pub etag: String,
pub last_modified: OffsetDateTime,
}
#[derive(Debug, Default)]
#[derive(serde::Serialize)]
pub struct CompleteMultipartUpload {
pub parts: Vec<CompletePart>,
}
impl CompleteMultipartUpload {
pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
//let buf = serde_json::to_string(self)?;
let buf = match serde_xml_rs::to_string(self) {
Ok(buf) => buf,
Err(e) => {
return Err(std::io::Error::other(e));
}
};
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
todo!();
}
}
pub struct CreateBucketConfiguration {
pub location: String,
}
#[derive(serde::Serialize)]
pub struct DeleteObject { //api has
pub key: String,
pub version_id: String,
}
pub struct DeletedObject { //s3s has
pub key: String,
pub version_id: String,
pub deletemarker: bool,
pub deletemarker_version_id: String,
}
pub struct NonDeletedObject {
pub key: String,
pub code: String,
pub message: String,
pub version_id: String,
}
#[derive(serde::Serialize)]
pub struct DeleteMultiObjects {
pub quiet: bool,
pub objects: Vec<DeleteObject>,
}
impl DeleteMultiObjects {
pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
//let buf = serde_json::to_string(self)?;
let buf = match serde_xml_rs::to_string(self) {
Ok(buf) => buf,
Err(e) => {
return Err(std::io::Error::other(e));
}
};
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
todo!();
}
}
pub struct DeleteMultiObjectsResult {
pub deleted_objects: Vec<DeletedObject>,
pub undeleted_objects: Vec<NonDeletedObject>,
}

224
ecstore/src/client/bucket_cache.rs Normal file
View File

@@ -0,0 +1,224 @@
#![allow(clippy::map_entry)]
use http::Request;
use hyper::body::Incoming;
use std::{collections::HashMap, sync::Arc};
use tracing::warn;
use tracing::{error, info, debug};
use hyper::StatusCode;
use reader::hasher::{Hasher, Sha256};
use s3s::S3ErrorCode;
use s3s::Body;
use crate::client::{
api_error_response::{http_resp_to_error_response, to_error_response},
transition_api::{TransitionClient, Document},
};
use crate::signer;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use super::constants::UNSIGNED_PAYLOAD;
use super::credentials::SignatureType;
pub struct BucketLocationCache {
items: HashMap<String, String>,
}
impl BucketLocationCache {
pub fn new() -> BucketLocationCache {
BucketLocationCache{
items: HashMap::new(),
}
}
pub fn get(&self, bucket_name: &str) -> Option<String> {
self.items.get(bucket_name).map(|s| s.clone())
}
pub fn set(&mut self, bucket_name: &str, location: &str) {
self.items.insert(bucket_name.to_string(), location.to_string());
}
pub fn delete(&mut self, bucket_name: &str) {
self.items.remove(bucket_name);
}
}
impl TransitionClient {
pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
Ok(self.get_bucket_location_inner(bucket_name).await?)
}
async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
if self.region != "" {
return Ok(self.region.clone())
}
let mut location;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
let ret = bucket_loc_cache.get(bucket_name);
if let Some(location) = ret {
return Ok(location);
}
//location = ret?;
}
let req = self.get_bucket_location_request(bucket_name)?;
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(resp, bucket_name).await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.set(bucket_name, &location);
}
Ok(location)
}
fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("location".to_string(), "".to_string());
let mut target_url = self.endpoint_url.clone();
let scheme = self.endpoint_url.scheme();
let h = target_url.host().expect("host is none.");
let default_port = if scheme == "https" {
443
} else {
80
};
let p = target_url.port().unwrap_or(default_port);
let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
let mut url_str: String = "".to_string();
if is_virtual_style {
url_str = scheme.to_string();
url_str.push_str("://");
url_str.push_str(bucket_name);
url_str.push_str(".");
url_str.push_str(target_url.host_str().expect("err"));
url_str.push_str("/?location");
} else {
let mut path = bucket_name.to_string();
path.push_str("/");
target_url.set_path(&path);
{
let mut q = target_url.query_pairs_mut();
for (k, v) in url_values {
q.append_pair(&k, &urlencoding::encode(&v));
}
}
url_str = target_url.to_string();
}
let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str);
self.set_user_agent(&mut req_builder);
let value;
{
let mut creds_provider = self.creds_provider.lock().unwrap();
value = match creds_provider.get_with_context(Some(self.cred_context())) {
Ok(v) => v,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
}
let mut signer_type = value.signer_type.clone();
let mut access_key_id = value.access_key_id;
let mut secret_access_key = value.secret_access_key;
let mut session_token = value.session_token;
if self.override_signer_type != SignatureType::SignatureDefault {
signer_type = self.override_signer_type.clone();
}
if value.signer_type == SignatureType::SignatureAnonymous {
signer_type = SignatureType::SignatureAnonymous
}
if signer_type == SignatureType::SignatureAnonymous {
let req = match req_builder.body(Body::empty()) {
Ok(req) => return Ok(req),
Err(err) => {
return Err(std::io::Error::other(err));
}
};
}
if signer_type == SignatureType::SignatureV2 {
let req_builder = signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style);
let req = match req_builder.body(Body::empty()) {
Ok(req) => return Ok(req),
Err(err) => {
return Err(std::io::Error::other(err));
}
};
}
let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
if self.secure {
content_sha256 = UNSIGNED_PAYLOAD.to_string();
}
req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
let req = match req_builder.body(Body::empty()) {
Ok(req) => return Ok(req),
Err(err) => {
return Err(std::io::Error::other(err));
}
};
}
}
async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
//if resp != nil {
if resp.status() != StatusCode::OK {
let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
match err_resp.code {
S3ErrorCode::NotImplemented => {
match err_resp.server.as_str() {
"AmazonSnowball" => {
return Ok("snowball".to_string());
}
"cloudflare" => {
return Ok("us-east-1".to_string());
}
_ => {
return Err(std::io::Error::other(err_resp));
}
}
}
S3ErrorCode::AuthorizationHeaderMalformed |
//S3ErrorCode::InvalidRegion |
S3ErrorCode::AccessDenied => {
if err_resp.region == "" {
return Ok("us-east-1".to_string());
}
return Ok(err_resp.region);
}
_ => {
return Err(std::io::Error::other(err_resp));
}
}
}
//}
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
let mut location = location_constraint;
if location == "" {
location = "us-east-1".to_string();
}
if location == "EU" {
location = "eu-west-1".to_string();
}
Ok(location)
}

24
ecstore/src/client/constants.rs Normal file
View File

@@ -0,0 +1,24 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use time::{macros::format_description, format_description::FormatItem};
use lazy_static::lazy_static;
pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
pub const MAX_PARTS_COUNT: i64 = 10000;
pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
pub const TOTAL_WORKERS: i64 = 4;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
const GetObjectAttributesMaxParts: i64 = 1000;
const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFs-Source-Mtime";

166
ecstore/src/client/credentials.rs Normal file
View File

@@ -0,0 +1,166 @@
use std::fmt::{Display, Formatter};
use time::OffsetDateTime;
#[derive(Debug, Default, Clone, Eq, PartialEq)]
pub enum SignatureType {
#[default]
SignatureDefault,
SignatureV4,
SignatureV2,
SignatureV4Streaming,
SignatureAnonymous,
}
#[derive(Debug, Clone, Default)]
pub struct Credentials<P: Provider + Default> {
creds: Value,
force_refresh: bool,
provider: P,
}
impl<P: Provider + Default> Credentials<P>
{
pub fn new(provider: P) -> Self {
Self {
provider: provider,
force_refresh: true,
..Default::default()
}
}
pub fn get(&mut self) -> Result<Value, std::io::Error> {
self.get_with_context(None)
}
pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
if self.is_expired() {
let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
self.creds = creds;
self.force_refresh = false;
}
Ok(self.creds.clone())
}
fn expire(&mut self) {
self.force_refresh = true;
}
pub fn is_expired(&self) -> bool {
self.force_refresh || self.provider.is_expired()
}
}
#[derive(Debug, Clone)]
pub struct Value {
pub access_key_id: String,
pub secret_access_key: String,
pub session_token: String,
pub expiration: OffsetDateTime,
pub signer_type: SignatureType,
}
impl Default for Value {
fn default() -> Self {
Self {
access_key_id: "".to_string(),
secret_access_key: "".to_string(),
session_token: "".to_string(),
expiration: OffsetDateTime::now_utc(),
signer_type: SignatureType::SignatureDefault,
}
}
}
pub struct CredContext {
//pub client: SendRequest,
pub endpoint: String,
}
trait Provider {
fn retrieve(&self) -> Value;
fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
fn is_expired(&self) -> bool;
}
#[derive(Debug, Clone, Default)]
pub struct Static(pub Value);
impl Provider for Static {
fn retrieve(&self) -> Value {
if self.0.access_key_id == "" || self.0.secret_access_key == "" {
return Value {
signer_type: SignatureType::SignatureAnonymous,
..Default::default()
};
}
self.0.clone()
}
fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
self.retrieve()
}
fn is_expired(&self) -> bool {
false
}
}
#[derive(Debug, Clone, Default)]
pub struct STSError {
pub r#type: String,
pub code: String,
pub message: String,
}
#[derive(Debug, Clone, thiserror::Error)]
pub struct ErrorResponse {
pub sts_error: STSError,
pub request_id: String,
}
impl Display for ErrorResponse {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error())
}
}
impl ErrorResponse {
fn error(&self) -> String {
if self.sts_error.message == "" {
return format!("Error response code {}.", self.sts_error.code);
}
return self.sts_error.message.clone();
}
}
struct Error {
code: String,
message: String,
bucket_name: String,
key: String,
resource: String,
request_id: String,
host_id: String,
region: String,
server: String,
status_code: i64,
}
impl Display for Error {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.message == "" {
return write!(f, "{}", format!("Error response code {}.", self.code));
}
write!(f, "{}", self.message)
}
}
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
todo!();
}
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
todo!();
}

45
ecstore/src/client/hook_reader.rs Normal file
View File

@@ -0,0 +1,45 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use crate::{
disk::{
error::{is_unformatted_disk, DiskError},
format::{DistributionAlgoVersion, FormatV3},
new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
},
store_api::{
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult,
ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
},
credentials::{Credentials, SignatureType,},
api_put_object_multipart::UploadPartParams,
};
use http::HeaderMap;
use tokio_util::sync::CancellationToken;
use tracing::warn;
use tracing::{error, info};
use url::Url;
struct HookReader {
source: GetObjectReader,
hook: GetObjectReader,
}
impl HookReader {
pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader {
HookReader {
source,
hook,
}
}
fn seek(&self, offset: i64, whence: i64) -> Result<i64> {
todo!();
}
fn read(&self, b: &[u8]) -> Result<i64> {
todo!();
}
}

18
ecstore/src/client/mod.rs Normal file
View File

@@ -0,0 +1,18 @@
pub mod constants;
pub mod transition_api;
pub mod api_list;
pub mod api_error_response;
pub mod api_s3_datatypes;
pub mod api_bucket_policy;
pub mod api_put_object_common;
pub mod api_get_options;
pub mod api_get_object;
pub mod api_put_object;
pub mod api_put_object_streaming;
pub mod api_put_object_multipart;
pub mod api_remove;
pub mod object_api_utils;
pub mod object_handlers_common;
pub mod admin_handler_utils;
pub mod credentials;
pub mod bucket_cache;

130
ecstore/src/client/object_api_utils.rs Normal file
View File

@@ -0,0 +1,130 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor;
use s3s::S3ErrorCode;
use crate::store_api::{
GetObjectReader, HTTPRangeSpec,
ObjectInfo, ObjectOptions,
};
use rustfs_filemeta::fileinfo::ObjectPartInfo;
use rustfs_rio::HashReader;
use crate::error::ErrorResponse;
//#[derive(Clone)]
pub struct PutObjReader {
pub reader: HashReader,
pub raw_reader: HashReader,
//pub sealMD5Fn: SealMD5CurrFn,
}
impl PutObjReader {
pub fn new(raw_reader: HashReader) -> Self {
todo!();
}
fn size(&self) -> usize {
//self.reader.size()
todo!();
}
fn md5_current_hex_string(&self) -> String {
todo!();
}
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
self.reader = enc_reader;
Ok(())
}
}
pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
if oi.size == 0 || oi.parts.len() == 0 {
return None;
}
let mut start: i64 = 0;
let mut end: i64 = -1;
let mut i = 0;
while i < oi.parts.len() && i < part_number {
start = end + 1;
end = start + oi.parts[i].actual_size as i64 - 1;
i += 1;
}
Some(HTTPRangeSpec {start: start as usize, end: Some(end as usize), is_suffix_length: false})
}
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
let mut skip_length: i64 = 0;
let mut cumulative_actual_size: i64 = 0;
let mut first_part_idx: i64 = 0;
let mut compressed_offset: i64 = 0;
let mut part_skip: i64 = 0;
let mut decrypt_skip: i64 = 0;
let mut seq_num: u64 = 0;
for (i, part) in oi.parts.iter().enumerate() {
cumulative_actual_size += part.actual_size as i64;
if cumulative_actual_size <= offset {
compressed_offset += part.size as i64;
} else {
first_part_idx = i as i64;
skip_length = cumulative_actual_size - part.actual_size as i64;
break;
}
}
skip_length = offset - skip_length;
let parts: &[ObjectPartInfo] = &oi.parts;
if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 {
todo!();
}
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
}
pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
//let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
let mut is_encrypted = false;
let is_compressed = false;//oi.is_compressed_ok();
let mut get_fn: ObjReaderFn;
let (off, length) = match rs.get_offset_length(oi.size) {
Ok(x) => x,
Err(err) => return Err(ErrorResponse {
code: S3ErrorCode::InvalidRange,
message: err.to_string(),
key: None,
bucket_name: None,
region: None,
request_id: None,
host_id: "".to_string(),
}),
};
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
//Box::pin({
/*let r = GetObjectReader {
object_info: oi.clone(),
stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
};
r*/
todo!();
//})
});
Ok((get_fn, off as i64, length as i64))
}
pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
if let Some(etag) = metadata.get("etag") {
etag.clone()
} else {
metadata["md5Sum"].clone()
}
}

27
ecstore/src/client/object_handlers_common.rs Normal file
View File

@@ -0,0 +1,27 @@
use lock::local_locker::MAX_DELETE_LIST;
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore;
use crate::store_api::{ObjectOptions, ObjectToDelete,};
use crate::StorageAPI;
use crate::bucket::lifecycle::lifecycle;
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) {
let mut remaining = to_del;
loop {
if remaining.len() <= 0 {break};
let mut to_del = remaining;
if to_del.len() > MAX_DELETE_LIST {
remaining = &to_del[MAX_DELETE_LIST..];
to_del = &to_del[..MAX_DELETE_LIST];
} else {
remaining = &[];
}
let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions {
//prefix_enabled_fn: vc.prefix_enabled(""),
version_suspended: vc.suspended(),
..Default::default()
});
}
}

888
ecstore/src/client/transition_api.rs Normal file
View File

@@ -0,0 +1,888 @@
#![allow(clippy::map_entry)]
use std::pin::Pin;
use bytes::Bytes;
use futures::Future;
use http::{HeaderMap, HeaderName};
use serde::{Serialize, Deserialize};
use uuid::Uuid;
use rand::Rng;
use std::{collections::HashMap, sync::{Arc, Mutex}};
use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll};
use time::Duration;
use time::OffsetDateTime;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector};
use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response};
use tracing::{error, debug};
use url::{form_urlencoded, Url};
use tokio::io::BufReader;
use std::io::Cursor;
use s3s::S3ErrorCode;
use s3s::{dto::Owner, Body};
use s3s::dto::ReplicationStatus;
use crate::client::bucket_cache::BucketLocationCache;
use reader::hasher::{Sha256, MD5,};
use rustfs_rio::HashReader;
use rustfs_utils::{
net::get_endpoint_url,
retry::{new_retry_timer, MAX_RETRY},
};
use crate::{
store_api::GetObjectReader,
checksum::ChecksumMode,
};
use crate::signer;
use crate::client::{
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{Credentials, SignatureType, CredContext, Static,},
api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument},
api_put_object_multipart::UploadPartParams,
api_put_object::PutObjectOptions,
api_get_options::GetObjectOptions,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart},
};
const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)";
const C_USER_AGENT: &str = "RustFS (linux; x86)";
const SUCCESS_STATUS: [StatusCode; 3] = [
StatusCode::OK,
StatusCode::NO_CONTENT,
StatusCode::PARTIAL_CONTENT,
];
const C_UNKNOWN: i32 = -1;
const C_OFFLINE: i32 = 0;
const C_ONLINE: i32 = 1;
//pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
pub enum ReaderImpl {
Body(Bytes),
ObjectBody(GetObjectReader),
}
pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
pub struct TransitionClient {
pub endpoint_url: Url,
pub creds_provider: Arc<Mutex<Credentials<Static>>>,
pub override_signer_type: SignatureType,
/*app_info: TODO*/
pub secure: bool,
pub http_client: Client<HttpsConnector<HttpConnector>, Body>,
//pub http_trace: Httptrace.ClientTrace,
pub bucket_loc_cache: Arc<Mutex<BucketLocationCache>>,
pub is_trace_enabled: Arc<Mutex<bool>>,
pub trace_errors_only: Arc<Mutex<bool>>,
//pub trace_output: io.Writer,
pub s3_accelerate_endpoint: Arc<Mutex<String>>,
pub s3_dual_stack_enabled: Arc<Mutex<bool>>,
pub region: String,
pub random: u64,
pub lookup: BucketLookupType,
//pub lookupFn: func(u url.URL, bucketName string) BucketLookupType,
pub md5_hasher: Arc<Mutex<Option<MD5>>>,
pub sha256_hasher: Option<Sha256>,
pub health_status: AtomicI32,
pub trailing_header_support: bool,
pub max_retries: i64,
}
#[derive(Debug, Default)]
pub struct Options {
pub creds: Credentials<Static>,
pub secure: bool,
//pub transport: http.RoundTripper,
//pub trace: *httptrace.ClientTrace,
pub region: String,
pub bucket_lookup: BucketLookupType,
//pub custom_region_via_url: func(u url.URL) string,
//pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType,
pub trailing_headers: bool,
pub custom_md5: Option<MD5>,
pub custom_sha256: Option<Sha256>,
pub max_retries: i64,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
enum BucketLookupType {
#[default]
BucketLookupAuto,
BucketLookupDNS,
BucketLookupPath,
}
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
let clnt = Self::private_new(endpoint, opts).await?;
Ok(clnt)
}
async fn private_new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
//let jar = cookiejar.New(cookiejar.Options{PublicSuffixList: publicsuffix.List})?;
//#[cfg(feature = "ring")]
//let _ = rustls::crypto::ring::default_provider().install_default();
//#[cfg(feature = "aws-lc-rs")]
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let scheme = endpoint_url.scheme();
let client;
//if scheme == "https" {
// client = Client::builder(TokioExecutor::new()).build_http();
//} else {
let tls = rustls::ClientConfig::builder()
.with_native_roots()?
.with_no_client_auth();
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
.https_or_http()
.enable_http1()
.build();
client = Client::builder(TokioExecutor::new()).build(https);
//}
let mut clnt = TransitionClient {
endpoint_url,
creds_provider: Arc::new(Mutex::new(opts.creds)),
override_signer_type: SignatureType::SignatureDefault,
secure: opts.secure,
http_client: client,
bucket_loc_cache: Arc::new(Mutex::new(BucketLocationCache::new())),
is_trace_enabled: Arc::new(Mutex::new(false)),
trace_errors_only: Arc::new(Mutex::new(false)),
s3_accelerate_endpoint: Arc::new(Mutex::new("".to_string())),
s3_dual_stack_enabled: Arc::new(Mutex::new(false)),
region: opts.region,
random: rand::rng().random_range(10..=50),
lookup: opts.bucket_lookup,
md5_hasher: Arc::new(Mutex::new(opts.custom_md5)),
sha256_hasher: opts.custom_sha256,
health_status: AtomicI32::new(C_UNKNOWN),
trailing_header_support: opts.trailing_headers,
max_retries: opts.max_retries,
};
{
let mut md5_hasher = clnt.md5_hasher.lock().unwrap();
if md5_hasher.is_none() {
*md5_hasher = Some(MD5::new());
}
}
if clnt.sha256_hasher.is_none() {
clnt.sha256_hasher = Some(Sha256::new());
}
clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4;
if opts.max_retries > 0 {
clnt.max_retries = opts.max_retries;
}
Ok(clnt)
}
fn endpoint_url(&self) -> Url {
self.endpoint_url.clone()
}
fn set_appinfo(&self, app_name: &str, app_version: &str) {
/*if app_name != "" && app_version != "" {
self.appInfo.app_name = app_name
self.appInfo.app_version = app_version
}*/
}
fn trace_errors_only_off(&self) {
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
*trace_errors_only = false;
}
fn trace_off(&self) {
let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
*is_trace_enabled = false;
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
*trace_errors_only = false;
}
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
todo!();
}
fn set_s3_enable_dual_stack(&self, enabled: bool) {
todo!();
}
pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) {
todo!();
}
fn is_online(&self) -> bool {
!self.is_offline()
}
fn mark_offline(&self) {
self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
}
fn is_offline(&self) -> bool {
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
}
fn health_check(hc_duration: Duration) {
todo!();
}
fn dump_http(&self, req: &http::Request<Body>, resp: &http::Response<Body>) -> Result<(), std::io::Error> {
let mut resp_trace: Vec<u8>;
//info!("{}{}", self.trace_output, "---------END-HTTP---------");
Ok(())
}
pub async fn doit(&self, req: http::Request<Body>) -> Result<http::Response<Body>, std::io::Error> {
let req_method;
let req_uri;
let req_headers;
let resp;
let http_client = self.http_client.clone();
{
//let mut http_client = http_client.lock().unwrap();
req_method = req.method().clone();
req_uri = req.uri().clone();
req_headers = req.headers().clone();
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req);
}
let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from));
debug!("http_client url: {} {}", req_method, req_uri);
debug!("http_client headers: {:?}", req_headers);
if let Err(err) = resp {
error!("http_client call error: {:?}", err);
return Err(std::io::Error::other(err));
}
let mut resp = resp.unwrap();
debug!("http_resp: {:?}", resp);
//if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
if resp.status() != StatusCode::OK {
//self.dump_http(&cloned_req, &resp)?;
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
debug!("err_body: {}", String::from_utf8(b).unwrap());
}
Ok(resp)
}
pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Response<Body>, std::io::Error> {
if self.is_offline() {
let mut s = self.endpoint_url.to_string();
s.push_str(" is offline.");
return Err(std::io::Error::other(s));
}
let mut retryable: bool;
//let mut body_seeker: BufferReader;
let mut req_retry = self.max_retries;
let mut resp: http::Response<Body>;
//if metadata.content_body != nil {
//body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
retryable = true;
if !retryable {
req_retry = 1;
}
//}
//let mut retry_timer = RetryTimer::new();
//while let Some(v) = retry_timer.next().await {
for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ {
let req = self.new_request(method, metadata).await?;
resp = self.doit(req).await?;
for http_status in SUCCESS_STATUS {
if http_status == resp.status() {
return Ok(resp);
}
}
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let err_response = http_resp_to_error_response(resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
if self.region == "" {
match err_response.code {
S3ErrorCode::AuthorizationHeaderMalformed | S3ErrorCode::InvalidArgument /*S3ErrorCode::InvalidRegion*/ => {
//break;
return Err(std::io::Error::other(err_response));
}
S3ErrorCode::AccessDenied => {
if err_response.region == "" {
return Err(std::io::Error::other(err_response));
}
if metadata.bucket_name != "" {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
let location = bucket_loc_cache.get(&metadata.bucket_name);
if location.is_some() && location.unwrap() != err_response.region {
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
//continue;
}
} else {
if err_response.region != metadata.bucket_location {
metadata.bucket_location = err_response.region.clone();
//continue;
}
}
return Err(std::io::Error::other(err_response));
}
_ => {
return Err(std::io::Error::other(err_response));
}
}
}
break;
}
Err(std::io::Error::other("resp err"))
}
async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Request<Body>, std::io::Error> {
let location = metadata.bucket_location.clone();
if location == "" {
if metadata.bucket_name != "" {
let location = self.get_bucket_location(&metadata.bucket_name).await?;
}
}
let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0;
let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket;
let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location,
is_virtual_host, &metadata.query_values)?;
let mut req_builder = Request::builder().method(method).uri(target_url.to_string());
let value;
{
let mut creds_provider = self.creds_provider.lock().unwrap();
value = creds_provider.get_with_context(Some(self.cred_context()))?;
}
let mut signer_type = value.signer_type.clone();
let access_key_id = value.access_key_id;
let secret_access_key = value.secret_access_key;
let session_token = value.session_token;
if self.override_signer_type != SignatureType::SignatureDefault {
signer_type = self.override_signer_type.clone();
}
if value.signer_type == SignatureType::SignatureAnonymous {
signer_type = SignatureType::SignatureAnonymous;
}
if metadata.expires != 0 && metadata.pre_sign_url {
if signer_type == SignatureType::SignatureAnonymous {
return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials.")));
}
if metadata.extra_pre_sign_header.is_some() {
if signer_type == SignatureType::SignatureV2 {
return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported.")));
}
for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
req_builder = req_builder.header(k, v);
}
}
if signer_type == SignatureType::SignatureV2 {
req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
} else if signer_type == SignatureType::SignatureV4 {
req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc());
}
let req = match req_builder.body(Body::empty()) {
Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); }
};
return Ok(req);
}
self.set_user_agent(&mut req_builder);
for (k, v) in metadata.custom_header.clone() {
req_builder.headers_mut().expect("err").insert(k.expect("err"), v);
}
//req.content_length = metadata.content_length;
if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value);
}
if metadata.content_md5_base64.len() > 0 {
let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
req_builder.headers_mut().expect("err").insert("Content-Md5", md5_value);
}
if signer_type == SignatureType::SignatureAnonymous {
let req = match req_builder.body(Body::empty()) {
Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); }
};
return Ok(req);
}
if signer_type == SignatureType::SignatureV2 {
req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
}
else if metadata.stream_sha256 && !self.secure {
if metadata.trailer.len() > 0 {
//req.Trailer = metadata.trailer;
for (_, v) in &metadata.trailer {
req_builder = req_builder.header(http::header::TRAILER, v.clone());
}
}
//req_builder = signer::streaming_sign_v4(req_builder, &access_key_id,
// &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher());
}
else {
let mut sha_header = UNSIGNED_PAYLOAD.to_string();
if metadata.content_sha256_hex != "" {
sha_header = metadata.content_sha256_hex.clone();
if metadata.trailer.len() > 0 {
return Err(std::io::Error::other("internal error: content_sha256_hex with trailer not supported"));
}
} else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
}
req_builder = req_builder.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err"));
req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone());
}
let req;
if metadata.content_length == 0 {
req = req_builder.body(Body::empty());
} else {
match &mut metadata.content_body {
ReaderImpl::Body(content_body) => {
req = req_builder.body(Body::from(content_body.clone()));
}
ReaderImpl::ObjectBody(content_body) => {
req = req_builder.body(Body::from(content_body.read_all().await?));
}
}
//req = req_builder.body(s3s::Body::from(metadata.content_body.read_all().await?));
}
match req {
Ok(req) => Ok(req),
Err(err) => {
Err(std::io::Error::other(err))
}
}
}
pub fn set_user_agent(&self, req: &mut Builder) {
let mut headers = req.headers_mut().expect("err");
headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
/*if self.app_info.app_name != "" && self.app_info.app_version != "" {
headers.insert("User-Agent", C_USER_AGENT+" "+self.app_info.app_name+"/"+self.app_info.app_version);
}*/
}
fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap<String, String>) -> Result<Url, std::io::Error> {
let scheme = self.endpoint_url.scheme();
let host = self.endpoint_url.host().unwrap();
let default_port = if scheme == "https" {
443
} else {
80
};
let port = self.endpoint_url.port().unwrap_or(default_port);
let mut url_str = format!("{scheme}://{host}:{port}/");
if bucket_name != "" {
if is_virtual_host_style {
url_str = format!("{scheme}://{bucket_name}.{host}:{port}/");
if object_name != "" {
url_str.push_str(object_name);
}
} else {
url_str.push_str(bucket_name);
url_str.push_str("/");
if object_name != "" {
url_str.push_str(object_name);
}
}
}
if query_values.len() > 0 {
let mut encoded = form_urlencoded::Serializer::new(String::new());
for (k, v) in query_values {
encoded.append_pair(&k, &v);
}
url_str.push_str("?");
url_str.push_str(&encoded.finish());
}
Url::parse(&url_str).map_err(|e| std::io::Error::other(e.to_string()))
}
pub fn is_virtual_host_style_request(&self, url: &Url, bucket_name: &str) -> bool {
if bucket_name == "" {
return false;
}
if self.lookup == BucketLookupType::BucketLookupDNS {
return true;
}
if self.lookup == BucketLookupType::BucketLookupPath {
return false;
}
false
}
pub fn cred_context(&self) -> CredContext {
CredContext {
//client: http_client,
endpoint: self.endpoint_url.to_string(),
}
}
}
struct LockedRandSource {
src: u64,//rand.Source,
}
impl LockedRandSource {
fn int63(&self) -> i64 {
/*let n = self.src.int63();
n*/
todo!();
}
fn seed(&self, seed: i64) {
//self.src.seed(seed);
todo!();
}
}
pub struct RequestMetadata {
pub pre_sign_url: bool,
pub bucket_name: String,
pub object_name: String,
pub query_values: HashMap<String, String>,
pub custom_header: HeaderMap,
pub extra_pre_sign_header: Option<HeaderMap>,
pub expires: i64,
pub bucket_location: String,
pub content_body: ReaderImpl,
pub content_length: i64,
pub content_md5_base64: String,
pub content_sha256_hex: String,
pub stream_sha256: bool,
pub add_crc: ChecksumMode,
pub trailer: HeaderMap,
}
pub struct TransitionCore(pub Arc<TransitionClient>);
impl TransitionCore {
pub async fn new(endpoint: &str, opts: Options) -> Result<Self, std::io::Error> {
let client = TransitionClient::new(endpoint, opts).await?;
Ok(Self(Arc::new(client)))
}
pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketResult, std::io::Error> {
let client = self.0.clone();
client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new())
}
pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketV2Result, std::io::Error> {
let client = self.0.clone();
client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await
}
/*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> {
self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts)
}*/
pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str,
part_id: i32, start_offset: i32, length: i64, metadata: HashMap<String, String>,
) -> Result<CompletePart, std::io::Error> {
//self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
// part_id, start_offset, length, metadata)
todo!();
}
pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let hook_reader = data;//newHook(data, opts.progress);
let client = self.0.clone();
client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await
}
pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result<String, std::io::Error> {
let client = self.0.clone();
let result = client.initiate_multipart_upload(bucket, object, &opts).await?;
Ok(result.upload_id)
}
pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> {
let client = self.0.clone();
client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads)
}
pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64,
data: ReaderImpl, size: i64, opts: PutObjectPartOptions
) -> Result<ObjectPart, std::io::Error> {
let mut p = UploadPartParams {
bucket_name: bucket.to_string(),
object_name: object.to_string(),
upload_id: upload_id.to_string(),
reader: data,
part_number: part_id,
md5_base64: opts.md5_base64,
sha256_hex: opts.sha256_hex,
size: size,
//sse: opts.sse,
stream_sha256: !opts.disable_content_sha256,
custom_header: opts.custom_header,
trailer: opts.trailer,
};
let client = self.0.clone();
client.upload_part(&mut p).await
}
pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> {
let client = self.0.clone();
client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await
}
pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result<UploadInfo, std::io::Error> {
let client = self.0.clone();
let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload {
parts: parts.to_vec(),
}, &opts).await?;
Ok(res)
}
pub async fn abort_multipart_upload(&self, bucket_name: &str, object: &str, upload_id: &str) -> Result<(), std::io::Error> {
let client = self.0.clone();
client.abort_multipart_upload(bucket_name, object, upload_id).await
}
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let client = self.0.clone();
client.get_bucket_policy(bucket_name).await
}
pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> {
let client = self.0.clone();
client.put_bucket_policy(bucket_name, bucket_policy).await
}
pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let client = self.0.clone();
client.get_object_inner(bucket_name, object_name, opts).await
}
}
pub struct PutObjectPartOptions {
pub md5_base64: String,
pub sha256_hex: String,
//pub sse: encrypt.ServerSide,
pub custom_header: HeaderMap,
pub trailer: HeaderMap,
pub disable_content_sha256: bool,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ObjectInfo {
pub etag: String,
pub name: String,
pub mod_time: OffsetDateTime,
pub size: usize,
pub content_type: Option<String>,
#[serde(skip)]
pub metadata: HeaderMap,
pub user_metadata: HashMap<String, String>,
pub user_tags: String,
pub user_tag_count: i64,
#[serde(skip)]
pub owner: Owner,
//pub grant: Vec<Grant>,
pub storage_class: String,
pub is_latest: bool,
pub is_delete_marker: bool,
pub version_id: Uuid,
#[serde(skip, default = "replication_status_default")]
pub replication_status: ReplicationStatus,
pub replication_ready: bool,
pub expiration: OffsetDateTime,
pub expiration_rule_id: String,
pub num_versions: usize,
pub restore: RestoreInfo,
pub checksum_crc32: String,
pub checksum_crc32c: String,
pub checksum_sha1: String,
pub checksum_sha256: String,
pub checksum_crc64nvme: String,
pub checksum_mode: String,
}
fn replication_status_default() -> ReplicationStatus {
ReplicationStatus::from_static(ReplicationStatus::PENDING)
}
impl Default for ObjectInfo {
fn default() -> Self {
Self {
etag: "".to_string(),
name: "".to_string(),
mod_time: OffsetDateTime::now_utc(),
size: 0,
content_type: None,
metadata: HeaderMap::new(),
user_metadata: HashMap::new(),
user_tags: "".to_string(),
user_tag_count: 0,
owner: Owner::default(),
storage_class: "".to_string(),
is_latest: false,
is_delete_marker: false,
version_id: Uuid::nil(),
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
replication_ready: false,
expiration: OffsetDateTime::now_utc(),
expiration_rule_id: "".to_string(),
num_versions: 0,
restore: RestoreInfo::default(),
checksum_crc32: "".to_string(),
checksum_crc32c: "".to_string(),
checksum_sha1: "".to_string(),
checksum_sha256: "".to_string(),
checksum_crc64nvme: "".to_string(),
checksum_mode: "".to_string(),
}
}
}
#[derive(Serialize, Deserialize)]
#[derive(Debug, Clone)]
pub struct RestoreInfo {
ongoing_restore: bool,
expiry_time: OffsetDateTime,
}
impl Default for RestoreInfo {
fn default() -> Self {
Self {
ongoing_restore: false,
expiry_time: OffsetDateTime::now_utc(),
}
}
}
pub struct ObjectMultipartInfo {
pub initiated: OffsetDateTime,
//pub initiator: initiator,
//pub owner: owner,
pub storage_class: String,
pub key: String,
pub size: i64,
pub upload_id: String,
//pub err error,
}
pub struct UploadInfo {
pub bucket: String,
pub key: String,
pub etag: String,
pub size: i64,
pub last_modified: OffsetDateTime,
pub location: String,
pub version_id: String,
pub expiration: OffsetDateTime,
pub expiration_rule_id: String,
pub checksum_crc32: String,
pub checksum_crc32c: String,
pub checksum_sha1: String,
pub checksum_sha256: String,
pub checksum_crc64nvme: String,
pub checksum_mode: String,
}
impl Default for UploadInfo {
fn default() -> Self {
Self {
bucket: "".to_string(),
key: "".to_string(),
etag: "".to_string(),
size: 0,
last_modified: OffsetDateTime::now_utc(),
location: "".to_string(),
version_id: "".to_string(),
expiration: OffsetDateTime::now_utc(),
expiration_rule_id: "".to_string(),
checksum_crc32: "".to_string(),
checksum_crc32c: "".to_string(),
checksum_sha1: "".to_string(),
checksum_sha256: "".to_string(),
checksum_crc64nvme: "".to_string(),
checksum_mode: "".to_string(),
}
}
}
pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Result<ObjectInfo, std::io::Error> {
todo!()
}
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
//#[derive(Clone)]
pub struct SendRequest {
inner: hyper::client::conn::http1::SendRequest<Body>,
}
impl From<hyper::client::conn::http1::SendRequest<Body>> for SendRequest {
fn from(inner: hyper::client::conn::http1::SendRequest<Body>) -> Self {
Self { inner }
}
}
impl tower::Service<Request<Body>> for SendRequest {
type Response = Response<Body>;
type Error = std::io::Error;
type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx).map_err(std::io::Error::other)
}
fn call(&mut self, req: Request<Body>) -> Self::Future {
//let req = hyper::Request::builder().uri("/").body(http_body_util::Empty::<Bytes>::new()).unwrap();
//let req = hyper::Request::builder().uri("/").body(Body::empty()).unwrap();
let fut = self.inner.send_request(req);
Box::pin(async move { fut.await.map_err(std::io::Error::other).map(|res| res.map(Body::from)) })
}
}
#[derive(Serialize, Deserialize)]
pub struct Document(pub String);

2
ecstore/src/cmd/bucket_replication.rs
View File

@@ -56,6 +56,7 @@ use tracing::{debug, error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh3::xxh3_64;
// use bucket_targets::{self, GLOBAL_Bucket_Target_Sys};
use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
#[derive(Serialize, Deserialize, Debug)]
struct MRFReplicateEntry {
@@ -2026,6 +2027,7 @@ impl ReplicateObjectInfo {
data_blocks: 0,
version_id: Uuid::try_parse(&self.version_id).ok(),
delete_marker: self.delete_marker,
transitioned_object: TransitionedObject::default(),
user_tags: self.user_tags.clone(),
parts: Vec::new(),
is_latest: true,

6
ecstore/src/disk/local.rs
View File

@@ -2125,7 +2125,7 @@ impl DiskAPI for LocalDisk {
// Check if the current bucket has a configured lifecycle policy
if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await {
if lc_has_active_rules(&lc, "") {
cache.info.life_cycle = Some(lc);
cache.info.lifecycle = Some(lc);
}
}
@@ -2234,9 +2234,9 @@ impl DiskAPI for LocalDisk {
}
}
for frer_version in fivs.free_versions.iter() {
for free_version in fivs.free_versions.iter() {
let _obj_info = ObjectInfo::from_file_info(
frer_version,
free_version,
&item.bucket,
&item.object_path().to_string_lossy(),
versioned,

244
ecstore/src/error.rs
View File

@@ -1,3 +1,5 @@
use s3s::{S3ErrorCode, S3Error};
use rustfs_utils::path::decode_dir_object;
use crate::disk::error::DiskError;
@@ -724,6 +726,248 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
}
}
pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool {
err.contains("Connection closed by foreign host") ||
err.contains("TLS handshake timeout") ||
err.contains("i/o timeout") ||
err.contains("connection timed out") ||
err.contains("connection reset by peer") ||
err.contains("broken pipe") ||
err.to_lowercase().contains("503 service unavailable") ||
err.contains("use of closed network connection") ||
err.contains("An existing connection was forcibly closed by the remote host") ||
err.contains("client error (Connect)")
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct GenericError {
pub bucket: String,
pub object: String,
pub version_id: String,
//pub err: Error,
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum ObjectApiError {
#[error("Operation timed out")]
OperationTimedOut,
#[error("etag of the object has changed")]
InvalidETag,
#[error("BackendDown")]
BackendDown(String),
#[error("Unsupported headers in Metadata")]
UnsupportedMetadata,
#[error("Method not allowed: {}/{}", .0.bucket, .0.object)]
MethodNotAllowed(GenericError),
#[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)]
InvalidObjectState(GenericError),
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
#[error("{}", .message)]
pub struct ErrorResponse {
pub code: S3ErrorCode,
pub message: String,
pub key: Option<String>,
pub bucket_name: Option<String>,
pub region: Option<String>,
pub request_id: Option<String>,
pub host_id: String,
}
pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error {
let mut bucket = "";
let mut object = "";
let mut version_id = "";
if params.len() >= 1 {
bucket = params[0];
}
if params.len() >= 2 {
object = params[1];
}
if params.len() >= 3 {
version_id = params[2];
}
if is_network_or_host_down(&err.to_string(), false) {
return std::io::Error::other(ObjectApiError::BackendDown(format!("{}", err)));
}
let mut err_ = std::io::Error::other(err.to_string());
let r_err = err;
let mut err = err_;
let bucket = bucket.to_string();
let object = object.to_string();
let version_id = version_id.to_string();
match r_err.code {
/*S3ErrorCode::SlowDownWrite => {
err = StorageError::InsufficientWriteQuorum;//{bucket: bucket, object: object};
}*/
/*S3ErrorCode::SlowDown/* SlowDownRead */ => {
err = std::io::Error::other(StorageError::InsufficientReadQuorum.to_string());
}*/
/*S3ErrorCode::PreconditionFailed => {
err = Error::from(StorageError::PreConditionFailed);
}*/
/*S3ErrorCode::InvalidRange => {
err = Error::from(StorageError::InvalidRange);
}*/
/*S3ErrorCode::BucketAlreadyOwnedByYou => {
err = Error::from(StorageError::BucketAlreadyOwnedByYou);
}*/
S3ErrorCode::BucketNotEmpty => {
err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string());
}
/*S3ErrorCode::NoSuchBucketPolicy => {
err = Error::from(StorageError::BucketPolicyNotFound);
}*/
/*S3ErrorCode::NoSuchLifecycleConfiguration => {
err = Error::from(StorageError::BucketLifecycleNotFound);
}*/
S3ErrorCode::InvalidBucketName => {
err = std::io::Error::other(StorageError::BucketNameInvalid(bucket));
}
S3ErrorCode::InvalidPart => {
err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object/* , version_id */));
}
S3ErrorCode::NoSuchBucket => {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
S3ErrorCode::NoSuchKey => {
if object != "" {
err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));
} else {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
}
S3ErrorCode::NoSuchVersion => {
if object != "" {
err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));//, version_id);
} else {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
}
/*S3ErrorCode::XRustFsInvalidObjectName => {
err = Error::from(StorageError::ObjectNameInvalid(bucket, object));
}*/
S3ErrorCode::AccessDenied => {
err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object));
}
/*S3ErrorCode::XAmzContentSHA256Mismatch => {
err = hash.SHA256Mismatch{};
}*/
S3ErrorCode::NoSuchUpload => {
err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id));
}
/*S3ErrorCode::EntityTooSmall => {
err = std::io::Error::other(StorageError::PartTooSmall);
}*/
/*S3ErrorCode::ReplicationPermissionCheck => {
err = std::io::Error::other(StorageError::ReplicationPermissionCheck);
}*/
_ => {
err = std::io::Error::other("err");
}
}
err
}
pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
let storage_err = &err;
let mut bucket: String = "".to_string();
let mut object: String = "".to_string();
if params.len() >= 1 {
bucket = params[0].to_string();
}
if params.len() >= 2 {
object = decode_dir_object(params[1]);
}
return match storage_err {
/*StorageError::NotImplemented => s3_error!(NotImplemented),
StorageError::InvalidArgument(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
}*/
StorageError::MethodNotAllowed => {
S3Error::with_message(S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError {bucket: bucket, object: object, ..Default::default()}).to_string())
}
/*StorageError::BucketNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket)
}
StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
StorageError::ObjectNameInvalid(bucket, object) => {
s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
}
StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
StorageError::PrefixAccessDenied(bucket, object) => {
s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
}
StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object)
}
StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
StorageError::ObjectNameTooLong(bucket, object) => {
s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
}
StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
}
StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
StorageError::VersionNotFound(bucket, object, version_id) => {
s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
}
StorageError::InvalidUploadID(bucket, object, version_id) => {
s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id)
}
StorageError::InvalidVersionID(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
}
// extended
StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
InvalidArgument,
"invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
bucket,
object,
version_id
),
// extended
StorageError::ObjectExistsAsDirectory(bucket, object) => {
s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
}
StorageError::InsufficientReadQuorum => {
s3_error!(SlowDown, "Storage resources are insufficient for the read operation")
}
StorageError::InsufficientWriteQuorum => {
s3_error!(SlowDown, "Storage resources are insufficient for the write operation")
}
StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"),
StorageError::VolumeNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket)
}
StorageError::InvalidPart(bucket, object, version_id) => {
s3_error!(
InvalidPart,
"Specified part could not be found. PartNumber {}, Expected {}, got {}",
bucket,
object,
version_id
)
}
StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/
_ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()),
};
}
#[cfg(test)]
mod tests {
use super::*;

3
ecstore/src/event/mod.rs Normal file
View File

@@ -0,0 +1,3 @@
pub mod name;
pub mod targetid;
pub mod targetlist;

225
ecstore/src/event/name.rs Normal file
View File

@@ -0,0 +1,225 @@
#[derive(Default)]
pub enum EventName {
ObjectAccessedGet,
ObjectAccessedGetRetention,
ObjectAccessedGetLegalHold,
ObjectAccessedHead,
ObjectAccessedAttributes,
ObjectCreatedCompleteMultipartUpload,
ObjectCreatedCopy,
ObjectCreatedPost,
ObjectCreatedPut,
ObjectCreatedPutRetention,
ObjectCreatedPutLegalHold,
ObjectCreatedPutTagging,
ObjectCreatedDeleteTagging,
ObjectRemovedDelete,
ObjectRemovedDeleteMarkerCreated,
ObjectRemovedDeleteAllVersions,
ObjectRemovedNoOP,
BucketCreated,
BucketRemoved,
ObjectReplicationFailed,
ObjectReplicationComplete,
ObjectReplicationMissedThreshold,
ObjectReplicationReplicatedAfterThreshold,
ObjectReplicationNotTracked,
ObjectRestorePost,
ObjectRestoreCompleted,
ObjectTransitionFailed,
ObjectTransitionComplete,
ObjectManyVersions,
ObjectLargeVersions,
PrefixManyFolders,
ILMDelMarkerExpirationDelete,
objectSingleTypesEnd,
ObjectAccessedAll,
ObjectCreatedAll,
ObjectRemovedAll,
ObjectReplicationAll,
ObjectRestoreAll,
ObjectTransitionAll,
ObjectScannerAll,
#[default]
Everything,
}
impl EventName {
fn expand(&self) -> Vec<EventName> {
todo!();
}
fn mask(&self) -> u64 {
todo!();
}
}
impl AsRef<str> for EventName {
fn as_ref(&self) -> &str {
match self {
EventName::BucketCreated => "s3:BucketCreated:*",
EventName::BucketRemoved => "s3:BucketRemoved:*",
EventName::ObjectAccessedAll => "s3:ObjectAccessed:*",
EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get",
EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention",
EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold",
EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head",
EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
EventName::ObjectCreatedAll => "s3:ObjectCreated:*",
EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
EventName::ObjectCreatedPost => "s3:ObjectCreated:Post",
EventName::ObjectCreatedPut => "s3:ObjectCreated:Put",
EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging",
EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging",
EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention",
EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold",
EventName::ObjectRemovedAll => "s3:ObjectRemoved:*",
EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete",
EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
EventName::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
EventName::ObjectReplicationAll => "s3:Replication:*",
EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
EventName::ObjectRestoreAll => "s3:ObjectRestore:*",
EventName::ObjectRestorePost => "s3:ObjectRestore:Post",
EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
EventName::ObjectTransitionAll => "s3:ObjectTransition:*",
EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed",
EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete",
EventName::ObjectManyVersions => "s3:Scanner:ManyVersions",
EventName::ObjectLargeVersions => "s3:Scanner:LargeVersions",
EventName::PrefixManyFolders => "s3:Scanner:BigPrefix",
_ => "",
}
}
}
impl From<&str> for EventName {
fn from(s: &str) -> Self {
match s {
"s3:BucketCreated:*" => {
EventName::BucketCreated
}
"s3:BucketRemoved:*" => {
EventName::BucketRemoved
}
"s3:ObjectAccessed:*" => {
EventName::ObjectAccessedAll
}
"s3:ObjectAccessed:Get" => {
EventName::ObjectAccessedGet
}
"s3:ObjectAccessed:GetRetention" => {
EventName::ObjectAccessedGetRetention
}
"s3:ObjectAccessed:GetLegalHold" => {
EventName::ObjectAccessedGetLegalHold
}
"s3:ObjectAccessed:Head" => {
EventName::ObjectAccessedHead
}
"s3:ObjectAccessed:Attributes" => {
EventName::ObjectAccessedAttributes
}
"s3:ObjectCreated:*" => {
EventName::ObjectCreatedAll
}
"s3:ObjectCreated:CompleteMultipartUpload" => {
EventName::ObjectCreatedCompleteMultipartUpload
}
"s3:ObjectCreated:Copy" => {
EventName::ObjectCreatedCopy
}
"s3:ObjectCreated:Post" => {
EventName::ObjectCreatedPost
}
"s3:ObjectCreated:Put" => {
EventName::ObjectCreatedPut
}
"s3:ObjectCreated:PutRetention" => {
EventName::ObjectCreatedPutRetention
}
"s3:ObjectCreated:PutLegalHold" => {
EventName::ObjectCreatedPutLegalHold
}
"s3:ObjectCreated:PutTagging" => {
EventName::ObjectCreatedPutTagging
}
"s3:ObjectCreated:DeleteTagging" => {
EventName::ObjectCreatedDeleteTagging
}
"s3:ObjectRemoved:*" => {
EventName::ObjectRemovedAll
}
"s3:ObjectRemoved:Delete" => {
EventName::ObjectRemovedDelete
}
"s3:ObjectRemoved:DeleteMarkerCreated" => {
EventName::ObjectRemovedDeleteMarkerCreated
}
"s3:ObjectRemoved:NoOP" => {
EventName::ObjectRemovedNoOP
}
"s3:ObjectRemoved:DeleteAllVersions" => {
EventName::ObjectRemovedDeleteAllVersions
}
"s3:LifecycleDelMarkerExpiration:Delete" => {
EventName::ILMDelMarkerExpirationDelete
}
"s3:Replication:*" => {
EventName::ObjectReplicationAll
}
"s3:Replication:OperationFailedReplication" => {
EventName::ObjectReplicationFailed
}
"s3:Replication:OperationCompletedReplication" => {
EventName::ObjectReplicationComplete
}
"s3:Replication:OperationMissedThreshold" => {
EventName::ObjectReplicationMissedThreshold
}
"s3:Replication:OperationReplicatedAfterThreshold" => {
EventName::ObjectReplicationReplicatedAfterThreshold
}
"s3:Replication:OperationNotTracked" => {
EventName::ObjectReplicationNotTracked
}
"s3:ObjectRestore:*" => {
EventName::ObjectRestoreAll
}
"s3:ObjectRestore:Post" => {
EventName::ObjectRestorePost
}
"s3:ObjectRestore:Completed" => {
EventName::ObjectRestoreCompleted
}
"s3:ObjectTransition:Failed" => {
EventName::ObjectTransitionFailed
}
"s3:ObjectTransition:Complete" => {
EventName::ObjectTransitionComplete
}
"s3:ObjectTransition:*" => {
EventName::ObjectTransitionAll
}
"s3:Scanner:ManyVersions" => {
EventName::ObjectManyVersions
}
"s3:Scanner:LargeVersions" => {
EventName::ObjectLargeVersions
}
"s3:Scanner:BigPrefix" => {
EventName::PrefixManyFolders
}
_ => {
EventName::Everything
}
}
}
}

10
ecstore/src/event/targetid.rs Normal file
View File

@@ -0,0 +1,10 @@
pub struct TargetID {
id: String,
name: String,
}
impl TargetID {
fn to_string(&self) -> String {
format!("{}:{}", self.id, self.name)
}
}

31
ecstore/src/event/targetlist.rs Normal file
View File

@@ -0,0 +1,31 @@
use std::sync::atomic::AtomicI64;
use super::targetid::TargetID;
#[derive(Default)]
pub struct TargetList {
pub current_send_calls: AtomicI64,
pub total_events: AtomicI64,
pub events_skipped: AtomicI64,
pub events_errors_total: AtomicI64,
//pub targets: HashMap<TargetID, Target>,
//pub queue: AsyncEvent,
//pub targetStats: HashMap<TargetID, TargetStat>,
}
impl TargetList {
pub fn new() -> TargetList {
TargetList::default()
}
}
struct TargetStat {
current_send_calls: i64,
total_events: i64,
failed_events: i64,
}
struct TargetIDResult {
id: TargetID,
err: std::io::Error,
}

61
ecstore/src/event_notification.rs Normal file
View File

@@ -0,0 +1,61 @@
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use crate::store_api::ObjectInfo;
use crate::event::name::EventName;
use crate::event::targetlist::TargetList;
use crate::store::ECStore;
use crate::bucket::metadata::BucketMetadata;
pub struct EventNotifier {
target_list: TargetList,
//bucket_rules_map: HashMap<String , HashMap<EventName, Rules>>,
}
impl EventNotifier {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
target_list: TargetList::new(),
//bucket_rules_map: HashMap::new(),
}))
}
fn get_arn_list(&self) -> Vec<String> {
todo!();
}
fn set(&self, bucket: &str, meta: BucketMetadata) {
todo!();
}
fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> {
/*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil {
return err
}
self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
nil*/
todo!();
}
fn send(&self, args: EventArgs) {
todo!();
}
}
#[derive(Debug, Default)]
pub struct EventArgs {
pub event_name: String,
pub bucket_name: String,
pub object: ObjectInfo,
pub req_params: HashMap<String, String>,
pub resp_elements: HashMap<String, String>,
pub host: String,
pub user_agent: String,
}
impl EventArgs {
}
pub fn send_event(args: EventArgs) {
}

13
ecstore/src/global.rs
View File

@@ -12,6 +12,9 @@ use crate::{
disk::DiskStore,
endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
tier::tier::TierConfigMgr,
event_notification::EventNotifier,
store::ECStore,
};
@@ -20,6 +23,8 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
pub const DEFAULT_PORT: u16 = 9000;
lazy_static! {
static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
@@ -33,11 +38,17 @@ lazy_static! {
pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
//pub static ref GLOBAL_RemoteTargetTransport
pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
}
pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();}
/// Get the global rustfs port
pub fn global_rustfs_port() -> u16 {

302
ecstore/src/heal/data_scanner.rs
View File

@@ -6,19 +6,45 @@ use std::{
path::{Path, PathBuf},
pin::Pin,
sync::{
Arc,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
Arc,
},
time::{Duration, SystemTime},
};
use time::{self, OffsetDateTime};
use common::defer;
use rustfs_utils::path::encode_dir_object;
use super::{
data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics},
data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend},
data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode},
};
use crate::{bucket::metadata_sys, cmd::bucket_replication::queue_replication_heal};
use crate::bucket::{
object_lock::objectlock_sys::{
enforce_retention_for_deletion,
BucketObjectLockSys,
}, utils::is_meta_bucketname,
};
use crate::{
bucket::{
lifecycle::{
bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc},
lifecycle::{self, Lifecycle, ExpirationOptions},
bucket_lifecycle_ops::{
self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps,
expire_transitioned_object,
},
},
metadata_sys
},
event_notification::{send_event, EventArgs},
global::GLOBAL_LocalNodeName, heal::{data_scanner},
store_api::{ObjectOptions, ObjectToDelete, StorageAPI},
};
use crate::cmd::bucket_replication::queue_replication_heal;
use crate::{
bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
cmd::bucket_replication::ReplicationStatusType,
@@ -44,6 +70,7 @@ use crate::{
peer::is_reserved_or_invalid_bucket,
store::ECStore,
};
use crate::event::name::EventName;
use crate::{disk::DiskAPI, store_api::ObjectInfo};
use crate::{
disk::error::DiskError,
@@ -56,7 +83,7 @@ use rand::Rng;
use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
use serde::{Deserialize, Serialize};
use tokio::{
sync::{
@@ -515,8 +542,55 @@ impl ScannerItem {
path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())])
}
pub async fn apply_versions_actions(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let obj_infos = self.apply_newer_noncurrent_version_limit(fives).await?;
async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) {
let mut size = oi.get_actual_size().expect("err!");
if self.debug {
info!("apply_lifecycle debug");
}
if self.lifecycle.is_none() {
return (lifecycle::IlmAction::NoneAction, size as i64);
}
let version_id = oi.version_id;
let mut vc = None;
let mut lr = None;
let mut rcfg = None;
if !is_meta_bucketname(&self.bucket) {
vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap());
lr = BucketObjectLockSys::get(&self.bucket).await;
rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await {
Some(replication_config)
} else { None };
}
let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await;
if self.debug {
if !version_id.is_none() {
info!("lifecycle: {} (version-id={}), Initial scan: {}", self.object_path().to_string_lossy().to_string(), version_id.expect("err"), lc_evt.action);
} else {
info!("lifecycle: {} Initial scan: {}", self.object_path().to_string_lossy().to_string(), lc_evt.action);
}
}
match lc_evt.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
size = 0;
}
lifecycle::IlmAction::DeleteAction => {
if !vc.unwrap().prefix_enabled(&oi.name) {
size = 0
}
}
_ => ()
}
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
(lc_evt.action, size as i64)
}
pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize {
// todo
}
@@ -533,16 +607,22 @@ impl ScannerItem {
Ok(obj_infos)
}
pub async fn apply_newer_noncurrent_version_limit(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
// let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await {
rcfg.mode.is_some()
} else { false };
let vcfg = BucketVersioningSys::get(&self.bucket).await?;
let versioned = match BucketVersioningSys::get(&self.bucket).await {
Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()),
Err(_) => false,
};
let mut object_infos = Vec::with_capacity(fives.len());
let mut object_infos = Vec::with_capacity(fivs.len());
if self.lifecycle.is_none() {
for info in fives.iter() {
for info in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(
info,
&self.bucket,
@@ -553,6 +633,54 @@ impl ScannerItem {
return Ok(object_infos);
}
let event = self.lifecycle.as_ref().expect("lifecycle err.").noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts {
name: self.object_path().to_string_lossy().to_string(),
..Default::default()
}).await;
let lim = event.newer_noncurrent_versions;
if lim == 0 || fivs.len() <= lim+1 {
for fi in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned));
}
return Ok(object_infos);
}
let overflow_versions = &fivs[lim+1..];
for fi in fivs[..lim+1].iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned));
}
let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len());
for fi in overflow_versions.iter() {
let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned);
if lock_enabled && enforce_retention_for_deletion(&obj) {
//if enforce_retention_for_deletion(&obj) {
if self.debug {
if obj.version_id.is_some() {
info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err"));
} else {
info!("lifecycle: {} is locked, not deleting\n", obj.name);
}
}
object_infos.push(obj);
continue;
}
if OffsetDateTime::now_utc().unix_timestamp() < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32).unix_timestamp() {
object_infos.push(obj);
continue;
}
to_del.push(ObjectToDelete {
object_name: obj.name,
version_id: obj.version_id,
});
}
if to_del.len() > 0 {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await;
}
// done().await;
Ok(object_infos)
@@ -560,7 +688,9 @@ impl ScannerItem {
pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, usize) {
let done = ScannerMetrics::time(ScannerMetric::Ilm);
//todo: lifecycle
let (action, size) = self.apply_lifecycle(oi).await;
info!(
"apply_actions {} {} {:?} {:?}",
oi.bucket.clone(),
@@ -581,6 +711,10 @@ impl ScannerItem {
self.heal_replication(&oi, _size_s).await;
done();
if action.delete_all() {
return (true, 0);
}
(false, oi.size)
}
@@ -757,9 +891,9 @@ impl FolderScanner {
let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name);
// Todo: lifeCycle
let active_life_cycle = if let Some(lc) = self.old_cache.info.life_cycle.as_ref() {
let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() {
if lc_has_active_rules(lc, &prefix) {
self.old_cache.info.life_cycle.clone()
self.old_cache.info.lifecycle.clone()
} else {
None
}
@@ -1401,7 +1535,151 @@ pub async fn scan_data_folder(
Ok(s.new_cache)
}
// pub fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: &ObjectLockConfiguration, rcfg: &ReplicationConfiguration， obj: &ObjectInfo)
pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: Option<DefaultRetention>, rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, oi: &ObjectInfo) -> lifecycle::Event {
let event = lc.eval(&oi.to_lifecycle_opts()).await;
//if serverDebugLog {
info!("lifecycle: Secondary scan: {}", event.action);
//}
let lock_enabled = if let Some(lr) = lr {
lr.mode.is_some()
} else { false };
match event.action {
lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
if lock_enabled {
return lifecycle::Event::default();
}
}
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => {
if oi.version_id.is_none() {
return lifecycle::Event::default();
}
if lock_enabled && enforce_retention_for_deletion(oi) {
//if serverDebugLog {
if !oi.version_id.is_none() {
info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err"));
} else {
info!("lifecycle: {} is locked, not deleting", oi.name);
}
//}
return lifecycle::Event::default();
}
if let Some(rcfg) = rcfg {
if rep_has_active_rules(&rcfg.0, &oi.name, true) {
return lifecycle::Event::default();
}
}
}
_ => ()
}
event
}
async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
if oi.delete_marker || oi.is_dir {
return false;
}
GLOBAL_TransitionState.queue_transition_task(oi, event, src).await;
true
}
pub async fn apply_expiry_on_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool {
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
return false;
}
time_ilm(1);
true
}
pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool {
let mut opts = ObjectOptions {
expiration: ExpirationOptions {expire: true},
..Default::default()
};
if lc_event.action.delete_versioned() {
opts.version_id = Some(oi.version_id.expect("err").to_string());
}
opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await;
opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await;
if lc_event.action.delete_all() {
opts.delete_prefix = true;
opts.delete_prefix_object = true;
}
let dobj: ObjectInfo;
//let err: Error;
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
let mut dobj = api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await.unwrap();
if dobj.name == "" {
dobj = oi.clone();
}
//let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
//tags["version-id"] = dobj.version_id;
let mut event_name = EventName::ObjectRemovedDelete;
if oi.delete_marker {
event_name = EventName::ObjectRemovedDeleteMarkerCreated;
}
match lc_event.action {
lifecycle::IlmAction::DeleteAllVersionsAction => {
event_name = EventName::ObjectRemovedDeleteAllVersions
}
lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
event_name = EventName::ILMDelMarkerExpirationDelete
}
_ => ()
}
send_event(EventArgs {
event_name: event_name.as_ref().to_string(),
bucket_name: dobj.bucket.clone(),
object: dobj,
user_agent: "Internal: [ILM-Expiry]".to_string(),
host: GLOBAL_LocalNodeName.to_string(),
..Default::default()
});
if lc_event.action != lifecycle::IlmAction::NoneAction {
let mut num_versions = 1_u64;
if lc_event.action.delete_all() {
num_versions = oi.num_versions as u64;
}
time_ilm(num_versions);
}
true
}
async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_by_days(oi, event, src).await;
true
}
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut success = false;
match event.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAction
| lifecycle::IlmAction::DeleteRestoredAction | lifecycle::IlmAction::DeleteRestoredVersionAction
| lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
success = apply_expiry_rule(event, src, oi).await;
}
lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
success = apply_transition_rule(event, src, oi).await;
}
_ => ()
}
success
}
#[cfg(test)]
mod tests {

76
ecstore/src/heal/data_scanner_metric.rs
View File

@@ -15,14 +15,13 @@ use tokio::sync::{Mutex, RwLock};
use tracing::debug;
use super::data_scanner::CurrentScannerCycle;
use crate::bucket::lifecycle::lifecycle;
lazy_static! {
pub static ref globalScannerMetrics: Arc<ScannerMetrics> = Arc::new(ScannerMetrics::new());
}
/// Scanner metric types, matching the Go version exactly
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[repr(u8)]
#[derive(Clone, Debug, PartialEq, PartialOrd)]
pub enum ScannerMetric {
// START Realtime metrics, that only records
// last minute latencies and total operation count.
@@ -196,7 +195,8 @@ pub struct ScannerMetrics {
// All fields must be accessed atomically and aligned.
operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>,
actions_latency: Vec<LockedLastMinuteLatency>,
// Current paths contains disk -> tracker mappings
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
@@ -215,6 +215,8 @@ impl ScannerMetrics {
Self {
operations,
latency,
actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
current_paths: Arc::new(RwLock::new(HashMap::new())),
cycle_info: Arc::new(RwLock::new(None)),
}
@@ -222,16 +224,17 @@ impl ScannerMetrics {
/// Log scanner action with custom metadata - compatible with existing usage
pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap<String, String>) {
let metric = metric as usize;
let start_time = SystemTime::now();
move |_custom: &HashMap<String, String>| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task for this)
if (metric as usize) < ScannerMetric::LastRealtime as usize {
let metric_index = metric as usize;
if (metric) < ScannerMetric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
globalScannerMetrics.latency[metric_index].add(duration).await;
});
@@ -239,23 +242,24 @@ impl ScannerMetrics {
// Log trace metrics
if metric as u8 > ScannerMetric::StartTrace as u8 {
debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
//debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
}
}
}
/// Time scanner action with size - returns function that takes size
pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) {
let metric = metric as usize;
let start_time = SystemTime::now();
move |size: u64| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics with size (spawn async task)
if (metric as usize) < ScannerMetric::LastRealtime as usize {
let metric_index = metric as usize;
if (metric) < ScannerMetric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
globalScannerMetrics.latency[metric_index].add_size(duration, size).await;
});
@@ -265,16 +269,17 @@ impl ScannerMetrics {
/// Time a scanner action - returns a closure to call when done
pub fn time(metric: ScannerMetric) -> impl Fn() {
let metric = metric as usize;
let start_time = SystemTime::now();
move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task)
if (metric as usize) < ScannerMetric::LastRealtime as usize {
let metric_index = metric as usize;
if (metric) < ScannerMetric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
globalScannerMetrics.latency[metric_index].add(duration).await;
});
@@ -284,17 +289,18 @@ impl ScannerMetrics {
/// Time N scanner actions - returns function that takes count, then returns completion function
pub fn time_n(metric: ScannerMetric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let metric = metric as usize;
let start_time = SystemTime::now();
Box::new(move |count: usize| {
Box::new(move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
globalScannerMetrics.operations[metric as usize].fetch_add(count as u64, Ordering::Relaxed);
globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task)
if (metric as usize) < ScannerMetric::LastRealtime as usize {
let metric_index = metric as usize;
if (metric) < ScannerMetric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
globalScannerMetrics.latency[metric_index].add(duration).await;
});
@@ -303,31 +309,53 @@ impl ScannerMetrics {
})
}
pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let a_clone = a as usize;
if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
return Box::new(move |_: u64| {
Box::new(move || {})
});
}
let start = SystemTime::now();
Box::new(move |versions: u64| {
Box::new(move || {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
tokio::spawn(async move {
globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed);
globalScannerMetrics.actions_latency[a_clone].add(duration).await;
});
})
})
}
/// Increment time with specific duration
pub async fn inc_time(metric: ScannerMetric, duration: Duration) {
let metric = metric as usize;
// Update operation count
globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics
if (metric as usize) < ScannerMetric::LastRealtime as usize {
globalScannerMetrics.latency[metric as usize].add(duration).await;
if (metric) < ScannerMetric::LastRealtime as usize {
globalScannerMetrics.latency[metric].add(duration).await;
}
}
/// Get lifetime operation count for a metric
pub fn lifetime(&self, metric: ScannerMetric) -> u64 {
if (metric as usize) >= ScannerMetric::Last as usize {
let metric = metric as usize;
if (metric) >= ScannerMetric::Last as usize {
return 0;
}
self.operations[metric as usize].load(Ordering::Relaxed)
self.operations[metric].load(Ordering::Relaxed)
}
/// Get last minute statistics for a metric
pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem {
if (metric as usize) >= ScannerMetric::LastRealtime as usize {
let metric = metric as usize;
if (metric) >= ScannerMetric::LastRealtime as usize {
return AccElem::default();
}
self.latency[metric as usize].total().await
self.latency[metric].total().await
}
/// Set current cycle information

55
ecstore/src/heal/data_usage_cache.rs
View File

@@ -129,6 +129,58 @@ const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERS
},
];
#[derive(Clone, Copy, Default)]
pub struct TierStats {
pub total_size: u64,
pub num_versions: i32,
pub num_objects: i32,
}
impl TierStats {
pub fn add(&self, u: &TierStats) -> TierStats {
TierStats {
total_size: self.total_size + u.total_size,
num_versions: self.num_versions + u.num_versions,
num_objects: self.num_objects + u.num_objects,
}
}
}
#[derive(Clone)]
struct AllTierStats {
tiers: HashMap<String, TierStats>,
}
impl AllTierStats {
pub fn new() -> Self {
Self {
tiers: HashMap::new(),
}
}
fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
for (tier, st) in tiers {
self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
}
}
fn merge(&mut self, other: AllTierStats) {
for (tier, st) in other.tiers {
self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
}
}
fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
for (tier, st) in &self.tiers {
stats.insert(tier.clone(), TierStats {
total_size: st.total_size.clone(),
num_versions: st.num_versions.clone(),
num_objects: st.num_objects.clone(),
});
}
}
}
// sizeHistogram is a size histogram.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SizeHistogram(Vec<u64>);
@@ -346,8 +398,7 @@ pub struct DataUsageCacheInfo {
pub last_update: Option<SystemTime>,
pub skip_healing: bool,
#[serde(skip)]
pub life_cycle: Option<BucketLifecycleConfiguration>,
// pub life_cycle:
pub lifecycle: Option<BucketLifecycleConfiguration>,
#[serde(skip)]
pub updates: Option<Sender<DataUsageEntry>>,
#[serde(skip)]

7
ecstore/src/lib.rs
View File

@@ -26,6 +26,13 @@ mod store_init;
pub mod store_list_objects;
mod store_utils;
pub mod checksum;
pub mod event;
pub mod event_notification;
pub mod client;
pub mod tier;
pub mod signer;
pub use global::new_object_layer_fn;
pub use global::set_global_endpoints;
pub use global::update_erasure_type;

282
ecstore/src/set_disk.rs
View File

@@ -1,3 +1,5 @@
use s3s::header::X_AMZ_RESTORE;
use crate::error::ObjectApiError;
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
use crate::disk::{
@@ -20,6 +22,8 @@ use crate::{
UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
},
error::{StorageError, to_object_err},
event::name::EventName, event_notification::{send_event, EventArgs},
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, put_restore_opts, get_transitioned_object_reader,},
global::{
GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id,
is_dist_erasure,
@@ -40,11 +44,16 @@ use crate::{
},
store_init::load_format_erasure,
};
use crate::client::{
object_api_utils::extract_etag,
transition_api::ReaderImpl,
};
use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation};
use crate::{
heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig},
store_api::ListObjectVersionsInfo,
};
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use bytesize::ByteSize;
use chrono::Utc;
use futures::future::join_all;
@@ -91,6 +100,7 @@ use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
use workers::workers::Workers;
use crate::global::{GLOBAL_TierConfigMgr, GLOBAL_LocalNodeName};
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024;
@@ -3666,6 +3676,29 @@ impl SetDisks {
Ok(())
}
pub async fn update_restore_metadata(&self, bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> {
let mut oi = obj_info.clone();
oi.metadata_only = true;
if let Some(user_defined) = &mut oi.user_defined {
user_defined.remove(X_AMZ_RESTORE.as_str());
}
let version_id = oi.version_id.clone().map(|v| v.to_string());
let obj = self.copy_object(bucket, object, bucket, object, &mut oi, &ObjectOptions {
version_id: version_id.clone(),
..Default::default()
}, &ObjectOptions {
version_id: version_id,
..Default::default()
}).await;
if let Err(err) = obj {
//storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
return Err(err);
}
Ok(())
}
}
#[async_trait::async_trait]
@@ -4125,6 +4158,45 @@ impl StorageAPI for SetDisks {
src_opts.versioned || src_opts.version_suspended,
))
}
#[tracing::instrument(skip(self))]
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
let disks = self.get_disks(0, 0).await?;
let write_quorum = disks.len() / 2 + 1;
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for disk in disks.iter() {
futures.push(async move {
if let Some(disk) = disk {
match disk.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()).await {
Ok(r) => Ok(r),
Err(e) => Err(e),
}
} else {
Err(DiskError::DiskNotFound)
}
});
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err.into());
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_objects(
&self,
@@ -4318,6 +4390,21 @@ impl StorageAPI for SetDisks {
Ok(oi)
}
#[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
GLOBAL_MRFState.add_partial(PartialOperation {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: Some(version_id.to_string()),
queued: Utc::now(),
set_index: self.set_index,
pool_index: self.pool_index,
..Default::default()
})
.await;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
// TODO: nslock
@@ -4404,6 +4491,201 @@ impl StorageAPI for SetDisks {
Ok(oi.user_tags)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await {
Ok(client) => client,
Err(err) => {
return Err(Error::other(err.to_string()));
}
};
/*if !opts.no_lock {
let lk = self.new_ns_lock(bucket, object);
let lkctx = lk.get_lock(globalDeleteOperationTimeout)?;
//ctx = lkctx.Context()
//defer lk.Unlock(lkctx)
}*/
let (mut fi, mut meta_arr, mut online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true).await?;
/*if err != nil {
return Err(to_object_err(err, vec![bucket, object]));
}*/
/*if fi.deleted {
if opts.version_id.is_none() {
return Err(to_object_err(DiskError::FileNotFound, vec![bucket, object]));
}
return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object]));
}*/
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() || !(opts.transition.etag == extract_etag(&fi.metadata)) {
return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object]));
}
if fi.transition_status == TRANSITION_COMPLETE {
return Ok(());
}
/*if fi.xlv1 {
if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) {
return err.expect("err");
}
(fi, meta_arr, online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true);
if err != nil {
return to_object_err(err, vec![bucket, object]);
}
}*/
//let traceFn = GLOBAL_LifecycleSys.trace(fi.to_object_info(bucket, object, opts.Versioned || opts.VersionSuspended));
let dest_obj = gen_transition_objname(bucket);
if let Err(err) = dest_obj {
//traceFn(ILMTransition, nil, err)
return Err(to_object_err(err, vec![]));
}
let dest_obj = dest_obj.unwrap();
let oi = ObjectInfo::from_file_info(&fi, &bucket, &object, opts.versioned || opts.version_suspended);
let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size);
//let h = HeaderMap::new();
//let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi});
let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: Box::new(pr), object_info: oi});
let cloned_bucket = bucket.to_string();
let cloned_object = object.to_string();
let cloned_fi = fi.clone();
let set_index = self.set_index;
let pool_index = self.pool_index;
tokio::spawn(async move {
if let Err(e) = Self::get_object_with_fileinfo(
&cloned_bucket, &cloned_object, 0, cloned_fi.size, &mut pw, cloned_fi, meta_arr, &online_disks, set_index, pool_index
)
.await
{
error!("get_object_with_fileinfo err {:?}", e);
};
});
let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size as i64, {
let mut m = HashMap::<String, String>::new();
m.insert("name".to_string(), object.to_string());
m
}).await;
//pr.CloseWithError(err);
if let Err(err) = rv {
//traceFn(ILMTransition, nil, err)
return Err(StorageError::Io(err));
}
let rv = rv.unwrap();
fi.transition_status = TRANSITION_COMPLETE.to_string();
fi.transitioned_objname = dest_obj;
fi.transition_tier = opts.transition.tier.clone();
fi.transition_version_id = if rv=="" { None } else { Some(Uuid::parse_str(&rv)?) };
let mut event_name = EventName::ObjectTransitionComplete.as_ref();
let disks = self.get_disks(0, 0).await?;
if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await {
event_name = EventName::ObjectTransitionFailed.as_ref();
}
for disk in disks.iter() {
if let Some(disk) = disk {
if disk.is_online().await {
continue;
}
}
self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")).await;
break;
}
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
send_event(EventArgs {
event_name: event_name.to_string(),
bucket_name: bucket.to_string(),
object: obj_info,
user_agent: "Internal: [ILM-Transition]".to_string(),
host: GLOBAL_LocalNodeName.to_string(),
..Default::default()
});
//let tags = opts.lifecycle_audit_event.tags();
//auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option<Error>| -> Result<()> {
if rerr.is_none() {
return Ok(());
}
self.update_restore_metadata(bucket, object, oi, opts).await?;
Err(rerr.unwrap())
};
let mut oi = ObjectInfo::default();
let fi = self.get_object_fileinfo(&bucket, &object, &opts, true).await;
if let Err(err) = fi {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
}
let (mut actual_fi, _, _) = fi.unwrap();
oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended);
let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi);
/*if oi.parts.len() == 1 {
let mut rs: HTTPRangeSpec;
let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts);
//if err != nil {
// return set_restore_header_fn(&mut oi, Some(toObjectErr(err, bucket, object)));
//}
//defer gr.Close()
let hash_reader = HashReader::new(gr, gr.obj_info.size, "", "", gr.obj_info.size);
let p_reader = PutObjReader::new(StreamingBlob::from(Box::pin(hash_reader)), hash_reader.size());
if let Err(err) = self.put_object(bucket, object, &mut p_reader, &ropts).await {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object])));
} else {
return Ok(());
}
}
let res = self.new_multipart_upload(bucket, object, &ropts).await?;
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
//}
let mut uploaded_parts: Vec<CompletePart> = vec![];
let mut rs: HTTPRangeSpec;
let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts).await?;
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
//}
for part_info in oi.parts {
//let hr = HashReader::new(LimitReader(gr, part_info.size), part_info.size, "", "", part_info.size);
let hr = HashReader::new(gr, part_info.size as i64, part_info.size as i64, None, false);
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
//}
let mut p_reader = PutObjReader::new(hr, hr.size());
let p_info = self.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()).await?;
//if let Err(err) = p_info {
// return set_restore_header_fn(&mut oi, err);
//}
if p_info.size != part_info.size {
return set_restore_header_fn(&mut oi, Some(Error::from(ObjectApiError::InvalidObjectState(GenericError{bucket: bucket.to_string(), object: object.to_string(), ..Default::default()}))));
}
uploaded_parts.push(CompletePart {
part_num: p_info.part_num,
etag: p_info.etag,
});
}
if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
mod_time: oi.mod_time,
..Default::default()
}).await {
set_restore_header_fn(&mut oi, Some(err));
}*/
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let (mut fi, _, disks) = self.get_object_fileinfo(bucket, object, opts, false).await?;

24
ecstore/src/sets.rs
View File

@@ -1,6 +1,7 @@
#![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use rustfs_filemeta::FileInfo;
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::{
@@ -459,6 +460,11 @@ impl StorageAPI for Sets {
))
}
#[tracing::instrument(skip(self))]
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
if opts.delete_prefix && !opts.delete_prefix_object {
@@ -572,6 +578,22 @@ impl StorageAPI for Sets {
self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object).transition_object(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
self.get_disks_by_key(object).add_partial(bucket, object, version_id).await;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object).restore_transitioned_object(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn copy_object_part(
&self,
@@ -666,6 +688,8 @@ impl StorageAPI for Sets {
.await
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await

13
ecstore/src/signer/mod.rs Normal file
View File

@@ -0,0 +1,13 @@
pub mod utils;
pub mod request_signature_v2;
pub mod request_signature_v4;
pub mod request_signature_streaming;
pub mod request_signature_streaming_unsigned_trailer;
pub mod ordered_qs;
pub use request_signature_v2::sign_v2;
pub use request_signature_v2::pre_sign_v2;
pub use request_signature_v4::sign_v4;
pub use request_signature_v4::pre_sign_v4;
pub use request_signature_v4::sign_v4_trailer;
pub use request_signature_streaming::streaming_sign_v4;

109
ecstore/src/signer/ordered_qs.rs Normal file
View File

@@ -0,0 +1,109 @@
//! Ordered query strings
use crate::signer::utils::stable_sort_by_first;
/// Immutable query string container
#[derive(Debug, Default, Clone)]
pub struct OrderedQs {
/// Ascending query strings
qs: Vec<(String, String)>,
}
/// [`OrderedQs`]
#[derive(Debug, thiserror::Error)]
#[error("ParseOrderedQsError: {inner}")]
pub struct ParseOrderedQsError {
/// url decode error
inner: serde_urlencoded::de::Error,
}
impl OrderedQs {
/// Constructs [`OrderedQs`] from vec
///
/// + strings must be url-decoded
#[cfg(test)]
#[must_use]
pub fn from_vec_unchecked(mut v: Vec<(String, String)>) -> Self {
stable_sort_by_first(&mut v);
Self { qs: v }
}
/// Parses [`OrderedQs`] from query
///
/// # Errors
/// Returns [`ParseOrderedQsError`] if query cannot be decoded
pub fn parse(query: &str) -> Result<Self, ParseOrderedQsError> {
let result = serde_urlencoded::from_str::<Vec<(String, String)>>(query);
let mut v = result.map_err(|e| ParseOrderedQsError { inner: e })?;
stable_sort_by_first(&mut v);
Ok(Self { qs: v })
}
#[must_use]
pub fn has(&self, name: &str) -> bool {
self.qs.binary_search_by_key(&name, |x| x.0.as_str()).is_ok()
}
/// Gets query values by name. Time `O(logn)`
pub fn get_all(&self, name: &str) -> impl Iterator<Item = &str> + use<'_> {
let qs = self.qs.as_slice();
let lower_bound = qs.partition_point(|x| x.0.as_str() < name);
let upper_bound = qs.partition_point(|x| x.0.as_str() <= name);
qs[lower_bound..upper_bound].iter().map(|x| x.1.as_str())
}
pub fn get_unique(&self, name: &str) -> Option<&str> {
let qs = self.qs.as_slice();
let lower_bound = qs.partition_point(|x| x.0.as_str() < name);
let mut iter = qs[lower_bound..].iter();
let pair = iter.next()?;
if let Some(following) = iter.next() {
if following.0 == name {
return None;
}
}
(pair.0.as_str() == name).then_some(pair.1.as_str())
}
}
impl AsRef<[(String, String)]> for OrderedQs {
fn as_ref(&self) -> &[(String, String)] {
self.qs.as_ref()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tag() {
{
let query = "tagging";
let qs = OrderedQs::parse(query).unwrap();
assert_eq!(qs.as_ref(), &[("tagging".to_owned(), String::new())]);
assert_eq!(qs.get_unique("taggin"), None);
assert_eq!(qs.get_unique("tagging"), Some(""));
assert_eq!(qs.get_unique("taggingg"), None);
}
{
let query = "tagging&tagging";
let qs = OrderedQs::parse(query).unwrap();
assert_eq!(
qs.as_ref(),
&[("tagging".to_owned(), String::new()), ("tagging".to_owned(), String::new())]
);
assert_eq!(qs.get_unique("taggin"), None);
assert_eq!(qs.get_unique("tagging"), None);
assert_eq!(qs.get_unique("taggingg"), None);
}
}
}

71
ecstore/src/signer/request_signature_streaming.rs Normal file
View File

@@ -0,0 +1,71 @@
use std::pin::Pin;
use std::sync::Mutex;
use futures::prelude::*;
use futures::task;
use bytes::{Bytes, BytesMut};
use http::header::TRAILER;
use http::Uri;
use lazy_static::lazy_static;
use std::collections::HashMap;
use stdx::str::StrExt;
use std::fmt::Write;
use tracing::{error, info, warn, debug};
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description};
use http::request::{self, Request};
use http::HeaderMap;
use hyper::Method;
use rustfs_utils::{
crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256},
hash::EMPTY_STRING_SHA256_HASH
};
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::request_signature_v4::{get_scope, get_signing_key, get_signature, SERVICE_TYPE_S3};
const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER";
const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024;
const CHUNK_SIGCONST_LEN: i64 = 17;
const SIGNATURESTR_LEN: i64 = 64;
const CRLF_LEN: i64 = 2;
const TRAILER_KV_SEPARATOR: &str = ":";
const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature";
lazy_static! {
static ref ignored_streaming_headers: HashMap<String, bool> = {
let mut m = <HashMap<String, bool>>::new();
m.insert("authorization".to_string(), true);
m.insert("user-agent".to_string(), true);
m.insert("content-type".to_string(), true);
m
};
}
fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &str, chunk_check_sum: &str) -> String {
let mut string_to_sign_parts = <Vec<String>>::new();
string_to_sign_parts.push(STREAMING_PAYLOAD_HDR.to_string());
let format = format_description!("[year][month][day]T[hour][minute][second]Z");
string_to_sign_parts.push(t.format(&format).unwrap());
string_to_sign_parts.push(get_scope(region, t, SERVICE_TYPE_S3));
string_to_sign_parts.push(previous_sig.to_string());
string_to_sign_parts.push(EMPTY_STRING_SHA256_HASH.to_string());
string_to_sign_parts.push(chunk_check_sum.to_string());
string_to_sign_parts.join("\n")
}
fn build_chunk_signature(chunk_check_sum: &str, req_time: OffsetDateTime, region: &str,
previous_signature: &str, secret_access_key: &str
) -> String {
let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region,
previous_signature, chunk_check_sum);
let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3);
get_signature(signing_key, &chunk_string_to_sign)
}
pub fn streaming_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str,
region: &str, data_len: i64, req_time: OffsetDateTime/*, sh256: md5simd.Hasher*/
) -> request::Builder {
todo!();
}

6
ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs Normal file
View File

@@ -0,0 +1,6 @@
use http::request;
use time::OffsetDateTime;
pub fn streaming_unsigned_v4(mut req: request::Builder, session_token: &str, data_len: i64, req_time: OffsetDateTime) -> request::Builder{
todo!();
}

202
ecstore/src/signer/request_signature_v2.rs Normal file
View File

@@ -0,0 +1,202 @@
use std::collections::HashMap;
use http::request;
use hyper::Uri;
use bytes::{Bytes, BytesMut};
use std::fmt::Write;
use time::{OffsetDateTime, macros::format_description, format_description};
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
use super::utils::get_host_addr;
const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
const SIGN_V2_ALGORITHM: &str = "AWS";
fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String {
let mut path = "".to_string();
//path = serde_urlencoded::to_string(req.uri_ref().unwrap().path().unwrap()).unwrap();
path = req.uri_ref().unwrap().path().to_string();
path
}
pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, expires: i64, virtual_host: bool) -> request::Builder {
if access_key_id == "" || secret_access_key == "" {
return req;
}
let d = OffsetDateTime::now_utc();
let d = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
let epoch_expires = d.unix_timestamp() + expires;
let mut headers = req.headers_mut().expect("err");
let expires_str = headers.get("Expires");
if expires_str.is_none() {
headers.insert("Expires", format!("{:010}", epoch_expires).parse().unwrap());
}
let string_to_sign = pre_string_to_sign_v2(&req, virtual_host);
let signature = hex(hmac_sha1(secret_access_key, string_to_sign));
let result = serde_urlencoded::from_str::<HashMap<String, String>>(req.uri_ref().unwrap().query().unwrap());
let mut query = result.unwrap_or_default();
if get_host_addr(&req).contains(".storage.googleapis.com") {
query.insert("GoogleAccessId".to_string(), access_key_id.to_string());
} else {
query.insert("AWSAccessKeyId".to_string(), access_key_id.to_string());
}
query.insert("Expires".to_string(), format!("{:010}", epoch_expires));
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());
req
}
fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> String {
let signature = hex(hmac_sha1(secret_access_key, policy_base64));
signature
}
pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, virtual_host: bool) -> request::Builder {
if access_key_id == "" || secret_access_key == "" {
return req;
}
let d = OffsetDateTime::now_utc();
let d2 = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
let string_to_sign = string_to_sign_v2(&req, virtual_host);
let mut headers = req.headers_mut().expect("err");
let date = headers.get("Date").unwrap();
if date.to_str().unwrap() == "" {
headers.insert("Date", d2.format(&format_description::well_known::Rfc2822).unwrap().to_string().parse().unwrap());
}
let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id);
let auth_header = format!("{}{}", auth_header, base64_encode(&hmac_sha1(secret_access_key, string_to_sign)));
headers.insert("Authorization", auth_header.parse().unwrap());
req
}
fn pre_string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String {
let mut buf = BytesMut::new();
write_pre_sign_v2_headers(&mut buf, &req);
write_canonicalized_headers(&mut buf, &req);
write_canonicalized_resource(&mut buf, &req, virtual_host);
String::from_utf8(buf.to_vec()).unwrap()
}
fn write_pre_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) {
buf.write_str(req.method_ref().unwrap().as_str());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Expires").unwrap().to_str().unwrap());
buf.write_char('\n');
}
fn string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String {
let mut buf = BytesMut::new();
write_sign_v2_headers(&mut buf, &req);
write_canonicalized_headers(&mut buf, &req);
write_canonicalized_resource(&mut buf, &req, virtual_host);
String::from_utf8(buf.to_vec()).unwrap()
}
fn write_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) {
buf.write_str(req.method_ref().unwrap().as_str());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap());
buf.write_char('\n');
buf.write_str(req.headers_ref().unwrap().get("Date").unwrap().to_str().unwrap());
buf.write_char('\n');
}
fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) {
let mut proto_headers = Vec::<String>::new();
let mut vals = HashMap::<String, Vec<String>>::new();
for k in req.headers_ref().expect("err").keys() {
let lk = k.as_str().to_lowercase();
if lk.starts_with("x-amz") {
proto_headers.push(lk.clone());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect();
vals.insert(lk, vv);
}
}
proto_headers.sort();
for k in proto_headers {
buf.write_str(&k);
buf.write_char(':');
for (idx, v) in vals[&k].iter().enumerate() {
if idx > 0 {
buf.write_char(',');
}
buf.write_str(v);
}
buf.write_char('\n');
}
}
const INCLUDED_QUERY: &[&str] = &[
"acl",
"delete",
"lifecycle",
"location",
"logging",
"notification",
"partNumber",
"policy",
"requestPayment",
"response-cache-control",
"response-content-disposition",
"response-content-encoding",
"response-content-language",
"response-content-type",
"response-expires",
"uploadId",
"uploads",
"versionId",
"versioning",
"versions",
"website",
];
fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virtual_host: bool) {
let request_url = req.uri_ref().unwrap();
buf.write_str(&encode_url2path(req, virtual_host));
if request_url.query().unwrap() != "" {
let mut n: i64 = 0;
let result = serde_urlencoded::from_str::<HashMap<String, Vec<String>>>(req.uri_ref().unwrap().query().unwrap());
let mut vals = result.unwrap_or_default();
for resource in INCLUDED_QUERY {
let vv = &vals[*resource];
if vv.len() > 0 {
n += 1;
match n {
1 => {
buf.write_char('?');
}
_ => {
buf.write_char('&');
buf.write_str(resource);
if vv[0].len() > 0 {
buf.write_char('=');
buf.write_str(&vv[0]);
}
}
}
}
}
}
}

607
ecstore/src/signer/request_signature_v4.rs Normal file
View File

@@ -0,0 +1,607 @@
use bytes::{Bytes, BytesMut};
use http::header::TRAILER;
use http::Uri;
use lazy_static::lazy_static;
use std::collections::HashMap;
use std::fmt::Write;
use tracing::{error, info, warn, debug};
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description};
use http::request::{self, Request};
use http::HeaderMap;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::ordered_qs::OrderedQs;
use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
use super::utils::stable_sort_by_first;
use super::utils::{get_host_addr, sign_v4_trim_all};
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const SERVICE_TYPE_S3: &str = "s3";
pub const SERVICE_TYPE_STS: &str = "sts";
lazy_static! {
static ref v4_ignored_headers: HashMap<String, bool> = {
let mut m = <HashMap<String, bool>>::new();
m.insert("accept-encoding".to_string(), true);
m.insert("authorization".to_string(), true);
m.insert("user-agent".to_string(), true);
m
};
}
pub fn get_signing_key(secret: &str, loc: &str, t: OffsetDateTime, service_type: &str) -> [u8; 32] {
let mut s = "AWS4".to_string();
s.push_str(secret);
let format = format_description!("[year][month][day]");
let date = hmac_sha256(s.into_bytes(), t.format(&format).unwrap().into_bytes());
let location = hmac_sha256(date, loc);
let service = hmac_sha256(location, service_type);
let signing_key = hmac_sha256(service, "aws4_request");
signing_key
}
pub fn get_signature(signing_key: [u8; 32], string_to_sign: &str) -> String {
hex(hmac_sha256(signing_key, string_to_sign))
}
pub fn get_scope(location: &str, t: OffsetDateTime, service_type: &str) -> String {
let format = format_description!("[year][month][day]");
let mut ans = String::from("");
ans.push_str(&t.format(&format).unwrap().to_string());
ans.push('/');
ans.push_str(location); // TODO: use a `Region` type
ans.push('/');
ans.push_str(service_type);
ans.push_str("/aws4_request");
ans
}
fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, service_type: &str) -> String {
let scope = get_scope(location, t, service_type);
let mut s = access_key_id.to_string();
s.push_str("/");
s.push_str(&scope);
s
}
fn get_hashed_payload(req: &request::Builder) -> String {
let headers = req.headers_ref().unwrap();
let mut hashed_payload = "";
if let Some(payload)= headers.get("X-Amz-Content-Sha256") {
hashed_payload = payload.to_str().unwrap();
}
if hashed_payload == "" {
hashed_payload = UNSIGNED_PAYLOAD;
}
hashed_payload.to_string()
}
fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap<String, bool>) -> String {
let mut headers = Vec::<String>::new();
let mut vals = HashMap::<String, Vec<String>>::new();
for k in req.headers_ref().expect("err").keys() {
if ignored_headers.get(&k.to_string()).is_some() {
continue;
}
headers.push(k.as_str().to_lowercase());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect();
vals.insert(k.as_str().to_lowercase(), vv);
}
if !header_exists("host", &headers) {
headers.push("host".to_string());
}
headers.sort();
debug!("get_canonical_headers vals: {:?}", vals);
debug!("get_canonical_headers headers: {:?}", headers);
let mut buf = BytesMut::new();
for k in headers {
buf.write_str(&k);
buf.write_char(':');
let k: &str = &k;
match k {
"host" => {
buf.write_str(&get_host_addr(&req));
buf.write_char('\n');
}
_ => {
for (idx, v) in vals[k].iter().enumerate() {
if idx > 0 {
buf.write_char(',');
}
buf.write_str(&sign_v4_trim_all(v));
}
buf.write_char('\n');
}
}
}
String::from_utf8(buf.to_vec()).unwrap()
}
fn header_exists(key: &str, headers: &[String]) -> bool {
for k in headers {
if k == key {
return true;
}
}
false
}
fn get_signed_headers(req: &request::Builder, ignored_headers: &HashMap<String, bool>) -> String {
let mut headers = Vec::<String>::new();
let headers_ref = req.headers_ref().expect("err");
debug!("get_signed_headers headers: {:?}", headers_ref);
for (k, _) in headers_ref {
if ignored_headers.get(&k.to_string()).is_some() {
continue;
}
headers.push(k.as_str().to_lowercase());
}
if !header_exists("host", &headers) {
headers.push("host".to_string());
}
headers.sort();
headers.join(";")
}
fn get_canonical_request(req: &request::Builder, ignored_headers: &HashMap<String, bool>, hashed_payload: &str) -> String {
let mut canonical_query_string = "".to_string();
if let Some(q) = req.uri_ref().unwrap().query() {
let mut query = q.split('&').map(|h| h.to_string()).collect::<Vec<String>>();
query.sort();
canonical_query_string = query.join("&");
canonical_query_string = canonical_query_string.replace("+", "%20");
}
let mut canonical_request = <Vec<String>>::new();
canonical_request.push(req.method_ref().unwrap().to_string());
canonical_request.push(req.uri_ref().unwrap().path().to_string());
canonical_request.push(canonical_query_string);
canonical_request.push(get_canonical_headers(&req, ignored_headers));
canonical_request.push(get_signed_headers(&req, ignored_headers));
canonical_request.push(hashed_payload.to_string());
canonical_request.join("\n")
}
fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &str, service_type: &str) -> String {
let mut string_to_sign = SIGN_V4_ALGORITHM.to_string();
string_to_sign.push_str("\n");
let format = format_description!("[year][month][day]T[hour][minute][second]Z");
string_to_sign.push_str(&t.format(&format).unwrap());
string_to_sign.push_str("\n");
string_to_sign.push_str(&get_scope(location, t, service_type));
string_to_sign.push_str("\n");
string_to_sign.push_str(&hex_sha256(canonical_request.as_bytes(), |s| s.to_string()));
string_to_sign
}
pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, expires: i64, t: OffsetDateTime) -> request::Builder {
if access_key_id == "" || secret_access_key == "" {
return req;
}
//let t = OffsetDateTime::now_utc();
//let date = AmzDate::parse(timestamp).unwrap();
let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
//let credential = get_scope(location, t, SERVICE_TYPE_S3);
let credential = get_credential(access_key_id, location, t, SERVICE_TYPE_S3);
let signed_headers = get_signed_headers(&req, &v4_ignored_headers);
let mut query = <Vec<(String, String)>>::new();
if let Some(q) = req.uri_ref().unwrap().query() {
let result = serde_urlencoded::from_str::<Vec<(String, String)>>(q);
query = result.unwrap_or_default();
}
query.push(("X-Amz-Algorithm".to_string(), SIGN_V4_ALGORITHM.to_string()));
let format = format_description!("[year][month][day]T[hour][minute][second]Z");
query.push(("X-Amz-Date".to_string(), t.format(&format).unwrap().to_string()));
query.push(("X-Amz-Expires".to_string(), format!("{:010}", expires)));
query.push(("X-Amz-SignedHeaders".to_string(), signed_headers));
query.push(("X-Amz-Credential".to_string(), credential));
if session_token != "" {
query.push(("X-Amz-Security-Token".to_string(), session_token.to_string()));
}
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3);
//println!("canonical_request: \n{}\n", canonical_request);
//println!("string_to_sign: \n{}\n", string_to_sign);
let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3);
let signature = get_signature(signing_key, &string_to_sign);
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&X-Amz-Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());
req
}
fn post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_access_key: &str, location: &str) -> String {
let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3);
let signature = get_signature(signing_key, policy_base64);
signature
}
fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Builder {
sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
}
fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, service_type: &str, trailer: HeaderMap) -> request::Builder {
if access_key_id == "" || secret_access_key == "" {
return req;
}
let t = OffsetDateTime::now_utc();
let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
let mut headers = req.headers_mut().expect("err");
let format = format_description!("[year][month][day]T[hour][minute][second]Z");
headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap());
if session_token != "" {
headers.insert("X-Amz-Security-Token", session_token.parse().unwrap());
}
if trailer.len() > 0{
for (k, _) in &trailer {
headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
}
headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap());
}
if service_type == SERVICE_TYPE_STS {
headers.remove("X-Amz-Content-Sha256");
}
let hashed_payload = get_hashed_payload(&req);
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &hashed_payload);
let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, service_type);
let signing_key = get_signing_key(secret_access_key, location, t, service_type);
let credential = get_credential(access_key_id, location, t2, service_type);
let signed_headers = get_signed_headers(&req, &v4_ignored_headers);
let signature = get_signature(signing_key, &string_to_sign);
//debug!("\n\ncanonical_request: \n{}\nstring_to_sign: \n{}\nsignature: \n{}\n\n", &canonical_request, &string_to_sign, &signature);
let mut headers = req.headers_mut().expect("err");
let auth = format!("{} Credential={}, SignedHeaders={}, Signature={}", SIGN_V4_ALGORITHM, credential, signed_headers, signature);
headers.insert("Authorization", auth.parse().unwrap());
if trailer.len() > 0 {
//req.Trailer = trailer;
for (_, v) in &trailer {
headers.append(http::header::TRAILER, v.clone());
}
return streaming_unsigned_v4(req, &session_token, content_len, t);
}
req
}
fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: HeaderMap) {
if trailer.len() > 0 {
return;
}
let t = OffsetDateTime::now_utc();
let t = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
let mut headers = req.headers_mut().expect("err");
let format = format_description!("[year][month][day]T[hour][minute][second]Z");
headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap());
for (k, _) in &trailer {
headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
}
headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap());
if trailer.len() > 0 {
for (_, v) in &trailer {
headers.append(http::header::TRAILER, v.clone());
}
}
streaming_unsigned_v4(req, "", content_len, t);
}
pub fn sign_v4(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str) -> request::Builder {
sign_v4_inner(req, content_len, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, HeaderMap::new())
}
pub fn sign_v4_trailer(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, trailer: HeaderMap) -> request::Builder {
sign_v4_inner(req, 0, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, trailer)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn example_list_objects() {
// let access_key_id = "AKIAIOSFODNN7EXAMPLE";
let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
let timestamp = "20130524T000000Z";
let t = datetime!(2013-05-24 0:00 UTC);
// let bucket = "examplebucket";
let region = "us-east-1";
let service = "s3";
let path = "/";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/?");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap());
headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query = <Vec<(String, String)>>::new();
query.push(("max-keys".to_string(), "2".to_string()));
query.push(("prefix".to_string(), "J".to_string()));
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/\n",
"max-keys=2&prefix=J\n",
"host:examplebucket.s3.amazonaws.com\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20130524T000000Z", "\n",
"\n",
"host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
)
);
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
assert_eq!(
string_to_sign,
concat!(
"AWS4-HMAC-SHA256\n",
"20130524T000000Z", "\n",
"20130524/us-east-1/s3/aws4_request\n",
"df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7",
)
);
let signing_key = get_signing_key(secret_access_key, region, t, service);
let signature = get_signature(signing_key, &string_to_sign);
assert_eq!(signature, "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7");
}
#[test]
fn example_signature() {
// let access_key_id = "rustfsadmin";
let secret_access_key = "rustfsadmin";
let timestamp = "20250505T011054Z";
let t = datetime!(2025-05-05 01:10:54 UTC);
// let bucket = "mblock2";
let region = "us-east-1";
let service = "s3";
let path = "/mblock2/";
let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap());
headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query: Vec<(String, String)> = Vec::new();
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
println!("canonical_request: \n{}\n", canonical_request);
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/mblock2/\n",
"\n",
"host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250505T011054Z", "\n",
"\n",
"host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
)
);
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
println!("string_to_sign: \n{}\n", string_to_sign);
assert_eq!(
string_to_sign,
concat!(
"AWS4-HMAC-SHA256\n",
"20250505T011054Z", "\n",
"20250505/us-east-1/s3/aws4_request\n",
"c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0",
)
);
let signing_key = get_signing_key(secret_access_key, region, t, service);
let signature = get_signature(signing_key, &string_to_sign);
println!("signature: \n{}\n", signature);
assert_eq!(signature, "df4116595e27b0dfd1103358947d9199378cc6386c4657abd8c5f0b11ebb4931");
}
#[test]
fn example_signature2() {
// let access_key_id = "rustfsadmin";
let secret_access_key = "rustfsadmin";
let timestamp = "20250507T051030Z";
let t = datetime!(2025-05-07 05:10:30 UTC);
// let bucket = "mblock2";
let region = "us-east-1";
let service = "s3";
let path = "/mblock2/";
let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?list-type=2&encoding-type=url&prefix=mypre&delimiter=%2F&fetch-owner=true&max-keys=1");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap());
headers.insert("x-amz-date", timestamp.parse().unwrap());
/*let uri = req.uri_ref().unwrap().clone();
println!("{:?}", uri);
let mut canonical_query_string = "".to_string();
if let Some(q) = uri.query() {
let result = serde_urlencoded::from_str::<Vec<String>>(q);
let mut query = result.unwrap_or_default();
query.sort();
canonical_query_string = query.join("&");
canonical_query_string.replace("+", "%20");
}
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());*/
println!("{:?}", req.uri_ref().unwrap().query());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
println!("canonical_request: \n{}\n", canonical_request);
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/mblock2/\n",
"delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n",
"host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250507T051030Z", "\n",
"\n",
"host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
)
);
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
println!("string_to_sign: \n{}\n", string_to_sign);
assert_eq!(
string_to_sign,
concat!(
"AWS4-HMAC-SHA256\n",
"20250507T051030Z", "\n",
"20250507/us-east-1/s3/aws4_request\n",
"e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999",
)
);
let signing_key = get_signing_key(secret_access_key, region, t, service);
let signature = get_signature(signing_key, &string_to_sign);
println!("signature: \n{}\n", signature);
assert_eq!(signature, "760278c9a77d5c245ac83d85917bddc3e3b14343091e8f4ad8edbbf73107d685");
}
#[test]
fn example_presigned_url() {
use hyper::Uri;
let access_key_id = "AKIAIOSFODNN7EXAMPLE";
let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
let timestamp = "20130524T000000Z";
let t = datetime!(2013-05-24 0:00 UTC);
// let bucket = "mblock2";
let region = "us-east-1";
let service = "s3";
let path = "/";
let session_token = "";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/test.txt");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", &region, 86400, t);
let mut canonical_request = req.method_ref().unwrap().as_str().to_string();
canonical_request.push_str("\n");
canonical_request.push_str(req.uri_ref().unwrap().path());
canonical_request.push_str("\n");
canonical_request.push_str(req.uri_ref().unwrap().query().unwrap());
canonical_request.push_str("\n");
canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
canonical_request.push_str("\n");
canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
canonical_request.push_str("\n");
canonical_request.push_str(&get_hashed_payload(&req));
//println!("canonical_request: \n{}\n", canonical_request);
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/test.txt\n",
"X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404\n",
"host:examplebucket.s3.amazonaws.com\n",
"\n",
"host\n",
"UNSIGNED-PAYLOAD",
)
);
}
#[test]
fn example_presigned_url2() {
use hyper::Uri;
let access_key_id = "rustfsadmin";
let secret_access_key = "rustfsadmin";
let timestamp = "20130524T000000Z";
let t = datetime!(2013-05-24 0:00 UTC);
// let bucket = "mblock2";
let region = "us-east-1";
let service = "s3";
let path = "/mblock2/";
let session_token = "";
let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/test.txt?delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap());
req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", &region, 86400, t);
let mut canonical_request = req.method_ref().unwrap().as_str().to_string();
canonical_request.push_str("\n");
canonical_request.push_str(req.uri_ref().unwrap().path());
canonical_request.push_str("\n");
canonical_request.push_str(req.uri_ref().unwrap().query().unwrap());
canonical_request.push_str("\n");
canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
canonical_request.push_str("\n");
canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
canonical_request.push_str("\n");
canonical_request.push_str(&get_hashed_payload(&req));
//println!("canonical_request: \n{}\n", canonical_request);
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/mblock2/test.txt\n",
"delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=rustfsadmin%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=e4af975e4a7e2c0449451740c7e9a425123681d2a8830bfb188789ea19618b20\n",
"host:192.168.1.11:9020\n",
"\n",
"host\n",
"UNSIGNED-PAYLOAD",
)
);
}
}

31
ecstore/src/signer/utils.rs Normal file
View File

@@ -0,0 +1,31 @@
use http::request;
pub fn get_host_addr(req: &request::Builder) -> String {
let host = req.headers_ref().expect("err").get("host");
let uri = req.uri_ref().unwrap();
let req_host;
if let Some(port) = uri.port() {
req_host = format!("{}:{}", uri.host().unwrap(), port);
} else {
req_host = uri.host().unwrap().to_string();
}
if host.is_some() && req_host != host.unwrap().to_str().unwrap().to_string() {
return host.unwrap().to_str().unwrap().to_string();
}
/*if req.uri_ref().unwrap().host().is_some() {
return req.uri_ref().unwrap().host().unwrap();
}*/
req_host
}
pub fn sign_v4_trim_all(input: &str) -> String {
let ss = input.split_whitespace().collect::<Vec<_>>();
ss.join(" ")
}
pub fn stable_sort_by_first<T>(v: &mut [(T, T)])
where
T: Ord,
{
v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0));
}

72
ecstore/src/store.rs
View File

@@ -1,5 +1,6 @@
#![allow(clippy::map_entry)]
use rustfs_filemeta::FileInfo;
use crate::bucket::metadata_sys::{self, set_bucket_metadata};
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::config::GLOBAL_StorageClass;
@@ -13,7 +14,7 @@ use crate::error::{
use crate::global::{
DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd,
set_global_deployment_id, set_object_layer,
set_global_deployment_id, set_object_layer, GLOBAL_TierConfigMgr,
};
use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo};
use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo};
@@ -26,7 +27,10 @@ use crate::rebalance::RebalanceMeta;
use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::{
bucket::metadata::BucketMetadata,
bucket::{
metadata::BucketMetadata,
lifecycle::bucket_lifecycle_ops::TransitionState
},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools,
peer::S3PeerSys,
@@ -40,7 +44,7 @@ use crate::{
};
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
use crate::error::{Error, Result};
use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
use futures::future::join_all;
@@ -324,6 +328,14 @@ impl ECStore {
}
}
init_background_expiry(self.clone()).await;
TransitionState::init(self.clone()).await;
if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await {
info!("TierConfigMgr init error: {}", err);
}
Ok(())
}
@@ -1836,7 +1848,6 @@ impl StorageAPI for ECStore {
return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
}
}
let idx = self.get_pool_idx(bucket, object, -1).await?;
if opts.data_movement && idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
@@ -1849,6 +1860,47 @@ impl StorageAPI for ECStore {
self.pools[idx].new_multipart_upload(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
let object = encode_dir_object(object);
if self.single_pool() {
self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
}
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()).await?;
self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].transition_object(bucket, &object, opts).await;
}
//opts.skip_decommissioned = true;
//opts.no_lock = true;
let idx = self.get_pool_idx_existing_with_opts(bucket, &object, opts).await?;
self.pools[idx].transition_object(bucket, &object, opts).await
}
#[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].restore_transitioned_object(bucket, &object, opts).await;
}
//opts.skip_decommissioned = true;
//opts.nolock = true;
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].restore_transitioned_object(bucket, &object, opts).await
}
#[tracing::instrument(skip(self))]
async fn copy_object_part(
&self,
@@ -2075,6 +2127,18 @@ impl StorageAPI for ECStore {
self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
}
#[tracing::instrument(skip(self))]
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
check_del_obj_args(bucket, object)?;
let object = rustfs_utils::path::encode_dir_object(object);
if self.single_pool() {
return self.pools[0].delete_object_version(bucket, object.as_str(), fi, force_del_marker).await;
}
Ok(())
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);

33
ecstore/src/store_api.rs
View File

@@ -2,14 +2,22 @@ use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi as _;
use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::error::{Error, Result};
use rustfs_utils::path::decode_dir_object;
use crate::heal::heal_ops::HealSequence;
use crate::store_utils::clean_metadata;
use crate::{disk::DiskStore, heal::heal_commands::HealOpts};
use crate::{disk::DiskStore, heal::heal_commands::HealOpts,};
use crate::{
bucket::lifecycle::{
lifecycle::TransitionOptions,
bucket_lifecycle_ops::TransitionedObject,
},
bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
bucket::lifecycle::lifecycle::ExpirationOptions,
};
use http::{HeaderMap, HeaderValue};
use madmin::heal_commands::HealResultItem;
use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING};
use rustfs_rio::{HashReader, Reader};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::Debug;
@@ -236,6 +244,10 @@ pub struct ObjectOptions {
pub replication_request: bool,
pub delete_marker: bool,
pub transition: TransitionOptions,
pub expiration: ExpirationOptions,
pub lifecycle_audit_event: LcAuditEvent,
pub eval_metadata: Option<HashMap<String, String>>,
}
@@ -307,6 +319,7 @@ pub struct ObjectInfo {
pub data_blocks: usize,
pub version_id: Option<Uuid>,
pub delete_marker: bool,
pub transitioned_object: TransitionedObject,
pub user_tags: String,
pub parts: Vec<ObjectPartInfo>,
pub is_latest: bool,
@@ -340,6 +353,7 @@ impl Clone for ObjectInfo {
data_blocks: self.data_blocks,
version_id: self.version_id,
delete_marker: self.delete_marker,
transitioned_object: self.transitioned_object.clone(),
user_tags: self.user_tags.clone(),
parts: self.parts.clone(),
is_latest: self.is_latest,
@@ -430,7 +444,15 @@ impl ObjectInfo {
// TODO:expires
// TODO:ReplicationState
// TODO:TransitionedObject
let transitioned_object = TransitionedObject {
name: fi.transitioned_objname.clone(),
version_id: if let Some(transition_version_id) = fi.transition_version_id { transition_version_id.to_string() } else { "".to_string() },
status: fi.transition_status.clone(),
free_version: fi.tier_free_version(),
tier: fi.transition_tier.clone(),
};
let metadata = {
let mut v = fi.metadata.clone();
@@ -473,6 +495,7 @@ impl ObjectInfo {
etag,
inlined,
user_defined: metadata,
transitioned_object,
..Default::default()
}
}
@@ -746,6 +769,7 @@ pub trait StorageAPI: ObjectIO {
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<ObjectInfo>;
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>;
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
async fn delete_objects(
&self,
@@ -820,6 +844,9 @@ pub trait StorageAPI: ObjectIO {
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
// DecomTieredObject
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String>;
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>;
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;

9
ecstore/src/tier/mod.rs Normal file
View File

@@ -0,0 +1,9 @@
pub mod warm_backend_s3;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend;
pub mod tier_admin;
pub mod tier_config;
pub mod tier;
pub mod tier_gen;
pub mod tier_handlers;

443
ecstore/src/tier/tier.rs Normal file
View File

@@ -0,0 +1,443 @@
use std::{
collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,}
};
use bytes::Bytes;
use serde::{Serialize, Deserialize};
use time::OffsetDateTime;
use tokio::{select, sync::RwLock, time::interval};
use rand::Rng;
use tracing::{info, debug, warn, error};
use http::status::StatusCode;
use lazy_static::lazy_static;
use tokio::io::BufReader;
use s3s::S3ErrorCode;
use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::{path_join, SLASH_SEPARATOR};
use crate::{
config::com::{read_config, CONFIG_PREFIX},
disk::RUSTFS_META_BUCKET,
store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI
};
use crate::client::admin_handler_utils::AdminError;
use crate::tier::{
warm_backend::{check_warm_backend, new_warm_backend},
tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND,
},
tier_admin::TierCreds,
tier_config::{TierType, TierConfig,},
};
use crate::new_object_layer_fn;
use rustfs_rio::HashReader;
use super::{
tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND},
warm_backend::WarmBackendImpl,
};
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 1;
lazy_static! {
//pub static ref TIER_CONFIG_PATH: PathBuf = path_join(&[PathBuf::from(RUSTFS_CONFIG_PREFIX), PathBuf::from(TIER_CONFIG_FILE)]);
}
const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierMissingCredentials",
message: "Specified remote credentials are empty",
status_code: StatusCode::FORBIDDEN,
};
pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
code: "XRustFSAdminTierBackendInUse",
message: "Specified remote tier is already in use",
status_code: StatusCode::CONFLICT,
};
pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
code: "XRustFSAdminTierTypeUnsupported",
message: "Specified tier type is unsupported",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
code: "XRustFSAdminTierBackendNotEmpty",
message: "Specified remote backend is not empty",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
code: "XRustFSAdminTierInvalidConfig",
message: "Unable to setup remote tier, check tier configuration",
status_code: StatusCode::BAD_REQUEST,
};
#[derive(Serialize, Deserialize)]
pub struct TierConfigMgr {
#[serde(skip)]
pub driver_cache: HashMap<String, WarmBackendImpl>,
pub tiers: HashMap<String, TierConfig>,
pub last_refreshed_at: OffsetDateTime,
}
impl TierConfigMgr {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
}))
}
pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
let cfg: TierConfigMgr = serde_json::from_slice(data)?;
//let mut cfg = TierConfigMgr(m);
//let mut cfg = m;
Ok(cfg)
}
pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
let data = serde_json::to_vec(&self)?;
//let mut data = Vec<u8>::with_capacity(self.msg_size()+4);
let mut data = Bytes::from(data);
//LittleEndian::write_u16(&mut data[0..2], TIER_CONFIG_FORMAT);
//LittleEndian::write_u16(&mut data[2..4], TIER_CONFIG_VERSION);
Ok(data)
}
pub fn refreshed_at(&self) -> OffsetDateTime {
self.last_refreshed_at
}
pub fn is_tier_valid(&self, tier_name: &str) -> bool {
let (_, valid) = self.is_tier_name_in_use(tier_name);
valid
}
pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
if let Some(t) = self.tiers.get(tier_name) {
return (t.tier_type.clone(), true);
}
(TierType::Unsupported, false)
}
pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
let tier_name = &tier.name;
if tier_name != tier_name.to_uppercase().as_str() {
return Err(ERR_TIER_NAME_NOT_UPPERCASE);
}
let (_, b) = self.is_tier_name_in_use(tier_name);
if b {
return Err(ERR_TIER_ALREADY_EXISTS);
}
let d = new_warm_backend(&tier, true).await?;
if !force {
let in_use = d.in_use().await;
match in_use {
Ok(b) => {
if b {
return Err(ERR_TIER_BACKEND_IN_USE);
}
}
Err(err) => {
warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR);
} else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS);
} else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND);
}
return Err(ERR_TIER_PERM_ERR);
}
}
}
self.driver_cache.insert(tier_name.to_string(), d);
self.tiers.insert(tier_name.to_string(), tier);
Ok(())
}
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
let d = self.get_driver(tier_name).await;
if let Err(err) = d {
match err {
ERR_TIER_NOT_FOUND => {
return Ok(());
}
_ => {
return Err(err);
}
}
}
if !force {
let inuse = d.expect("err").in_use().await;
if let Err(err) = inuse {
return Err(ERR_TIER_PERM_ERR);
} else if inuse.expect("err") {
return Err(ERR_TIER_BACKEND_NOT_EMPTY);
}
}
self.tiers.remove(tier_name);
self.driver_cache.remove(tier_name);
Ok(())
}
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
let d = match self.get_driver(tier_name).await {
Ok(d) => d,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
if let Err(err) = check_warm_backend(Some(d)).await {
return Err(std::io::Error::other(err));
} else {
return Ok(());
}
}
pub fn empty(&self) -> bool {
self.list_tiers().len() == 0
}
pub fn tier_type(&self, tier_name: &str) -> String {
let cfg = self.tiers.get(tier_name);
if cfg.is_none() {
return "internal".to_string();
}
cfg.expect("err").tier_type.to_string()
}
pub fn list_tiers(&self) -> Vec<TierConfig> {
let mut tier_cfgs = Vec::<TierConfig>::new();
for (_, tier) in self.tiers.iter() {
let tier = tier.clone();
tier_cfgs.push(tier);
}
tier_cfgs
}
pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
for (tier_name2, tier) in self.tiers.iter() {
if tier_name == tier_name2 {
return Some(tier.clone());
}
}
None
}
pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
if !exists {
return Err(ERR_TIER_NOT_FOUND);
}
let mut cfg = self.tiers[tier_name].clone();
match tier_type {
TierType::S3 => {
let mut s3 = cfg.s3.as_mut().expect("err");
if creds.aws_role {
s3.aws_role = true
}
if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
s3.aws_role_arn = creds.aws_role_arn;
s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
}
if creds.access_key != "" && creds.secret_key != "" {
s3.access_key = creds.access_key;
s3.secret_key = creds.secret_key;
}
}
TierType::RustFS => {
let mut rustfs = cfg.rustfs.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS);
}
rustfs.access_key = creds.access_key;
rustfs.secret_key = creds.secret_key;
}
TierType::MinIO => {
let mut minio = cfg.minio.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS);
}
minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key;
}
_ => ()
}
let d = new_warm_backend(&cfg, true).await?;
self.tiers.insert(tier_name.to_string(), cfg);
self.driver_cache.insert(tier_name.to_string(), d);
Ok(())
}
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
Ok(match self.driver_cache.entry(tier_name.to_string()) {
Entry::Occupied(e) => {
e.into_mut()
}
Entry::Vacant(e) => {
let t = self.tiers.get(tier_name);
if t.is_none() {
return Err(ERR_TIER_NOT_FOUND);
}
let d = new_warm_backend(t.expect("err"), false).await?;
e.insert(d)
}
})
}
pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
//let Some(api) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) };
let new_config = load_tier_config(api).await;
match &new_config {
Ok(_c) => {}
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
}
self.driver_cache.clear();
self.tiers.clear();
let new_config = new_config.expect("err");
for (tier, cfg) in new_config.tiers {
self.tiers.insert(tier, cfg);
}
self.last_refreshed_at = OffsetDateTime::now_utc();
Ok(())
}
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) };
//let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
pub async fn save_config<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts(
api,
file,
data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
}
pub async fn save_config_with_opts<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> {
debug!("save tier config:{}", file);
let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
.await?;
Ok(())
}
pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
//let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let mut rng = rand::rng();
let r = rng.random_range(0.0..1.0);
let rand_interval = || {
Duration::from_secs((r * 60_f64).round() as u64)
};
let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop {
select! {
_ = t.tick() => {
if let Err(err) = self.reload(api.clone()).await {
info!("{}", err);
}
}
else => ()
}
t.reset();
}
}
pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
self.reload(api).await?;
//if globalIsDistErasure {
// self.refresh_tier_config(api).await;
//}
Ok(())
}
}
async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
//lookup_configs(&mut cfg, api.clone()).await;
cfg.save_tiering_config(api).await?;
Ok(cfg)
}
#[tracing::instrument(level = "debug")]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data {
if is_err_config_not_found(&err) {
warn!("config not found, start to init");
let cfg = new_and_save_tiering_config(api).await?;
return Ok(cfg);
} else {
error!("read config err {:?}", &err);
return Err(std::io::Error::other(err));
}
}
let cfg;
let version = 1;//LittleEndian::read_u16(&data[2..4]);
match version {
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
Ok(cfg) => cfg,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
}
_ => {
return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version)));
}
}
Ok(cfg)
}
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _))
}

29
ecstore/src/tier/tier_admin.rs Normal file
View File

@@ -0,0 +1,29 @@
use std::{
time::{Duration, SystemTime, UNIX_EPOCH},
};
use rand::Rng;
use tracing::warn;
use http::status::StatusCode;
use serde::{Serialize, Deserialize};
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[serde(default)]
pub struct TierCreds {
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
#[serde(rename = "awsRole")]
pub aws_role: bool,
#[serde(rename = "awsRoleWebIdentityTokenFile")]
pub aws_role_web_identity_token_file: String,
#[serde(rename = "awsRoleArn")]
pub aws_role_arn: String,
//azsp: ServicePrincipalAuth,
//#[serde(rename = "credsJson")]
pub creds_json: Vec<u8>,
}

353
ecstore/src/tier/tier_config.rs Normal file
View File

@@ -0,0 +1,353 @@
use std::fmt::Display;
use serde::{Serialize, Deserialize};
use tracing::info;
const C_TierConfigVer: &str = "v1";
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config";
const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version";
const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type";
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
pub enum TierType {
#[default]
Unsupported,
#[serde(rename = "s3")]
S3,
#[serde(rename = "azure")]
Azure,
#[serde(rename = "gcs")]
GCS,
#[serde(rename = "rustfs")]
RustFS,
#[serde(rename = "minio")]
MinIO,
}
impl Display for TierType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TierType::S3 => {
write!(f, "S3")
}
TierType::RustFS => {
write!(f, "RustFS")
}
TierType::MinIO => {
write!(f, "MinIO")
}
_ => {
write!(f, "Unsupported")
}
}
}
}
impl TierType {
pub fn new(sc_type: &str) -> Self {
match sc_type {
"S3" => {
TierType::S3
}
"RustFS" => {
TierType::RustFS
}
"MinIO" => {
TierType::MinIO
}
_ => {
TierType::Unsupported
}
}
}
pub fn to_string(&self) -> String {
match self {
TierType::S3 => {
"s3".to_string()
}
TierType::RustFS => {
"rustfs".to_string()
}
TierType::MinIO => {
"minio".to_string()
}
_ => {
"unsupported".to_string()
}
}
}
}
#[derive(Default, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct TierConfig {
#[serde(skip)]
pub version: String,
#[serde(rename = "type")]
pub tier_type: TierType,
#[serde(skip)]
pub name: String,
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
pub s3: Option<TierS3>,
//TODO: azure: Option<TierAzure>,
//TODO: gcs: Option<TierGCS>,
#[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
pub rustfs: Option<TierRustFS>,
#[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
pub minio: Option<TierMinIO>,
}
impl Clone for TierConfig {
fn clone(&self) -> TierConfig {
let mut s3 = None;
//az TierAzure
//gcs TierGCS
let mut r = None;
let mut m = None;
match self.tier_type {
TierType::S3 => {
let mut s3_ = self.s3.as_ref().expect("err").clone();
s3_.secret_key = "REDACTED".to_string();
s3 = Some(s3_);
}
TierType::RustFS => {
let mut r_ = self.rustfs.as_ref().expect("err").clone();
r_.secret_key = "REDACTED".to_string();
r = Some(r_);
}
TierType::MinIO => {
let mut m_ = self.minio.as_ref().expect("err").clone();
m_.secret_key = "REDACTED".to_string();
m = Some(m_);
}
_ => ()
}
TierConfig {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
s3: s3,
//azure: az,
//gcs: gcs,
rustfs: r,
minio: m,
}
}
}
impl TierConfig {
pub fn unmarshal(data: &[u8]) -> Result<TierConfig, std::io::Error> {
/*let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
let mut cfg = TierConfig(m);
cfg.set_defaults();
Ok(cfg)*/
todo!();
}
pub fn marshal(&self) -> Result<Vec<u8>, std::io::Error> {
let data = serde_json::to_vec(&self)?;
Ok(data)
}
fn endpoint(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").endpoint.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").endpoint.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").endpoint.clone()
}
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
}
}
}
fn bucket(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").bucket.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").bucket.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").bucket.clone()
}
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
}
}
}
fn prefix(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").prefix.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").prefix.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").prefix.clone()
}
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
}
}
}
fn region(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").region.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").region.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").region.clone()
}
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
}
}
}
}
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[serde(default)]
pub struct TierS3 {
pub name: String,
pub endpoint: String,
#[serde(rename = "accesskey")]
pub access_key: String,
#[serde(rename = "secretkey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageclass")]
pub storage_class: String,
#[serde(skip)]
pub aws_role: bool,
#[serde(skip)]
pub aws_role_web_identity_token_file: String,
#[serde(skip)]
pub aws_role_arn: String,
#[serde(skip)]
pub aws_role_session_name: String,
#[serde(skip)]
pub aws_role_duration_seconds: i32,
}
impl TierS3 {
fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
where
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static
{
if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
}
let sc = TierS3 {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
endpoint: "https://s3.amazonaws.com".to_string(),
region: "".to_string(),
storage_class: "".to_string(),
..Default::default()
};
for option in options {
let option = option(sc.clone());
let option = *option;
option?;
}
Ok(TierConfig {
version: C_TierConfigVer.to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(sc),
..Default::default()
})
}
}
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[serde(default)]
pub struct TierRustFS {
pub name: String,
pub endpoint: String,
#[serde(rename = "accesskey")]
pub access_key: String,
#[serde(rename = "secretkey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageclass")]
pub storage_class: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[serde(default)]
pub struct TierMinIO {
pub name: String,
pub endpoint: String,
#[serde(rename = "accesskey")]
pub access_key: String,
#[serde(rename = "secretkey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
impl TierMinIO {
fn new<F>(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
where
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static
{
if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
}
let m = TierMinIO {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
endpoint: endpoint.to_string(),
..Default::default()
};
for option in options {
let option = option(m.clone());
let option = *option;
option?;
}
Ok(TierConfig {
version: C_TierConfigVer.to_string(),
tier_type: TierType::MinIO,
name: name.to_string(),
minio: Some(m),
..Default::default()
})
}
}

48
ecstore/src/tier/tier_config_gen.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::{fmt::Display, pin::Pin, sync::Arc};
use tracing::info;
use common::error::{Error, Result};
use crate::bucket::tier_config::{TierType, TierConfig,};
impl TierType {
fn decode_msg(&self/*, dc *msgp.Reader*/) -> Result<()> {
todo!();
}
fn encode_msg(&self/*, en *msgp.Writer*/) -> Result<()> {
todo!();
}
pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>> {
todo!();
}
pub fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>> {
todo!();
}
pub fn msg_size() -> usize {
todo!();
}
}
impl TierConfig {
fn decode_msg(&self, dc *msgp.Reader) -> Result<()> {
todo!();
}
pub fn encode_msg(&self, en *msgp.Writer) -> Result<()> {
todo!();
}
pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>> {
todo!();
}
pub fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>> {
todo!();
}
fn msg_size(&self) -> usize {
todo!();
}
}

23
ecstore/src/tier/tier_gen.rs Normal file
View File

@@ -0,0 +1,23 @@
use crate::tier::tier::TierConfigMgr;
impl TierConfigMgr {
fn decode_msg(/*dc *msgp.Reader*/) -> Result<(), std::io::Error> {
todo!();
}
fn encode_msg(/*en *msgp.Writer*/) -> Result<(), std::io::Error> {
todo!();
}
pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>, std::io::Error> {
todo!();
}
pub fn unmarshal_msg(buf: &[u8]) -> Result<Self, std::io::Error> {
todo!();
}
pub fn msg_size(&self) -> usize {
100
}
}

51
ecstore/src/tier/tier_handlers.rs Normal file
View File

@@ -0,0 +1,51 @@
use crate::client::admin_handler_utils::AdminError;
use tracing::warn;
use http::status::StatusCode;
pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
code: "XRustFSAdminTierAlreadyExists",
message: "Specified remote tier already exists",
status_code: StatusCode::CONFLICT,
};
pub const ERR_TIER_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierNotFound",
message: "Specified remote tier was not found",
status_code: StatusCode::NOT_FOUND,
};
pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
code: "XRustFSAdminTierNameNotUpperCase",
message: "Tier name must be in uppercase",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierBucketNotFound",
message: "Remote tier bucket not found",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierInvalidCredentials",
message: "Invalid remote tier credentials",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError {
code: "XRustFSAdminTierReserved",
message: "Cannot use reserved tier name",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_PERM_ERR: AdminError = AdminError {
code: "TierPermErr",
message: "Tier Perm Err",
status_code: StatusCode::OK,
};
pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError {
code: "TierConnectErr",
message: "Tier Connect Err",
status_code: StatusCode::OK,
};

97
ecstore/src/tier/warm_backend.rs Normal file
View File

@@ -0,0 +1,97 @@
use std::collections::HashMap;
use bytes::Bytes;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{ReadCloser, ReaderImpl,},
};
use crate::error::is_err_bucket_not_found;
use tracing::{info, warn};
use crate::tier::{
tier_config::{TierType, TierConfig},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,},
warm_backend_s3::WarmBackendS3,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_minio::WarmBackendMinIO,
};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
const PROBE_OBJECT: &str = "probeobject";
#[derive(Default)]
pub struct WarmBackendGetOpts {
pub start_offset: i64,
pub length: i64,
}
#[async_trait::async_trait]
pub trait WarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error>;
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
async fn in_use(&self) -> Result<bool, std::io::Error>;
}
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.expect("err");
let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await;
if let Err(err) = remote_version_id {
return Err(ERR_TIER_PERM_ERR);
}
let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
//xhttp.DrainBody(r);
if let Err(err) = r {
//if is_err_bucket_not_found(&err) {
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
//}
/*else if is_err_signature_does_not_match(err) {
return Err(ERR_TIER_MISSING_CREDENTIALS);
}*/
//else {
return Err(ERR_TIER_PERM_ERR);
//}
}
if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
return Err(ERR_TIER_PERM_ERR);
};
Ok(())
}
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
let mut d: Option<WarmBackendImpl> = None;
match tier.tier_type {
TierType::S3 => {
let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
info!("{}", err);
return Err(ERR_TIER_INVALID_CONFIG);
}
d = Some(Box::new(dd.expect("err")));
}
TierType::RustFS => {
let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(ERR_TIER_INVALID_CONFIG);
}
d = Some(Box::new(dd.expect("err")));
}
TierType::MinIO => {
let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(ERR_TIER_INVALID_CONFIG);
}
d = Some(Box::new(dd.expect("err")));
}
_ => {
return Err(ERR_TIER_TYPE_UNSUPPORTED);
}
}
Ok(d.expect("err"))
}

129
ecstore/src/tier/warm_backend_minio.rs Normal file
View File

@@ -0,0 +1,129 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use tracing::warn;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions,
};
use crate::tier::{
tier_config::TierMinIO,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendMinIO(WarmBackendS3);
impl WarmBackendMinIO {
pub async fn new(conf: &TierMinIO, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" {
443
} else {
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendMinIO {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
}).await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}

126
ecstore/src/tier/warm_backend_rustfs.rs Normal file
View File

@@ -0,0 +1,126 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use tracing::warn;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions,
};
use crate::tier::{
tier_config::TierRustFS,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendRustFS(WarmBackendS3);
impl WarmBackendRustFS {
pub async fn new(conf: &TierRustFS, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => return Err(std::io::Error::other(e)),
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" {
443
} else {
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendRustFS {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
}).await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}

153
ecstore/src/tier/warm_backend_s3.rs Normal file
View File

@@ -0,0 +1,153 @@
use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::client::{
api_get_options::GetObjectOptions,
credentials::{Credentials, Static, Value, SignatureType},
transition_api::{ReaderImpl, ReadCloser},
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{Options, TransitionClient, TransitionCore,},
};
use rustfs_utils::path::SLASH_SEPARATOR;
use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts,}
};
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
pub core: TransitionCore,
pub bucket: String,
pub prefix: String,
pub storage_class: String,
}
impl WarmBackendS3 {
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
let u = match Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" {
return Err(std::io::Error::other("both the token file and the role ARN are required"));
}
else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required"));
}
else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") {
return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file"));
}
else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let mut creds: Credentials<Static>;
if conf.access_key != "" && conf.secret_key != "" {
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
}
else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
..Default::default()
};
let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?;
//client.set_appinfo(format!("s3-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(),
storage_class: conf.storage_class.clone(),
})
}
fn to_object_err(&self, err: ErrorResponse, params: Vec<&str>) -> std::io::Error {
let mut object = "";
if params.len() >= 1 {
object = params.first().cloned().unwrap_or_default();
}
error_resp_to_object_err(err, vec![&self.bucket, &self.get_dest(object)])
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
}
return dest_obj;
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
let client = self.client.clone();
let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions {
send_content_md5: true,
storage_class: self.storage_class.clone(),
user_metadata: meta,
..Default::default()
}).await?;
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let mut gopts = GetObjectOptions::default();
if rv != "" {
gopts.version_id = rv.to_string();
}
if opts.start_offset >= 0 && opts.length > 0 {
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset+opts.length-1) {
return Err(std::io::Error::other(err));
}
}
let c = TransitionCore(Arc::clone(&self.client));
let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
Ok(r)
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
let mut ropts = RemoveObjectOptions::default();
if rv != "" {
ropts.version_id = rv.to_string();
}
let client = self.client.clone();
let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await;
Err(std::io::Error::other(err.expect("err")))
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
}
}

27
reader/Cargo.toml Normal file
View File

@@ -0,0 +1,27 @@
[package]
name = "reader"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[lints]
workspace = true
[dependencies]
tracing.workspace = true
s3s.workspace = true
thiserror.workspace = true
bytes.workspace = true
pin-project-lite.workspace = true
hex-simd = "0.8.0"
base64-simd = "0.8.0"
md-5.workspace = true
sha2 = { version = "0.11.0-pre.4" }
futures.workspace = true
async-trait.workspace = true
common.workspace = true
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

12
reader/src/error.rs Normal file
View File

@@ -0,0 +1,12 @@
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum ReaderError {
#[error("stream input error {0}")]
StreamInput(String),
//
#[error("etag: expected ETag {0} does not match computed ETag {1}")]
VerifyError(String, String),
#[error("Bad checksum: Want {0} does not match calculated {1}")]
ChecksumMismatch(String, String),
#[error("Bad sha256: Expected {0} does not match calculated {1}")]
SHA256Mismatch(String, String),
}

184
reader/src/hasher.rs Normal file
View File

@@ -0,0 +1,184 @@
use md5::{Digest as Md5Digest, Md5};
use sha2::{
digest::{Reset, Update},
Digest, Sha256 as sha_sha256,
};
pub trait Hasher {
fn write(&mut self, bytes: &[u8]);
fn reset(&mut self);
fn sum(&mut self) -> String;
fn size(&self) -> usize;
fn block_size(&self) -> usize;
}
#[derive(Default)]
pub enum HashType {
#[default]
Undefined,
Uuid(Uuid),
Md5(MD5),
Sha256(Sha256),
}
impl Hasher for HashType {
fn write(&mut self, bytes: &[u8]) {
match self {
HashType::Md5(md5) => md5.write(bytes),
HashType::Sha256(sha256) => sha256.write(bytes),
HashType::Uuid(uuid) => uuid.write(bytes),
HashType::Undefined => (),
}
}
fn reset(&mut self) {
match self {
HashType::Md5(md5) => md5.reset(),
HashType::Sha256(sha256) => sha256.reset(),
HashType::Uuid(uuid) => uuid.reset(),
HashType::Undefined => (),
}
}
fn sum(&mut self) -> String {
match self {
HashType::Md5(md5) => md5.sum(),
HashType::Sha256(sha256) => sha256.sum(),
HashType::Uuid(uuid) => uuid.sum(),
HashType::Undefined => "".to_owned(),
}
}
fn size(&self) -> usize {
match self {
HashType::Md5(md5) => md5.size(),
HashType::Sha256(sha256) => sha256.size(),
HashType::Uuid(uuid) => uuid.size(),
HashType::Undefined => 0,
}
}
fn block_size(&self) -> usize {
match self {
HashType::Md5(md5) => md5.block_size(),
HashType::Sha256(sha256) => sha256.block_size(),
HashType::Uuid(uuid) => uuid.block_size(),
HashType::Undefined => 64,
}
}
}
#[derive(Debug)]
pub struct Sha256 {
hasher: sha_sha256,
}
impl Sha256 {
pub fn new() -> Self {
Self {
hasher: sha_sha256::new(),
}
}
}
impl Default for Sha256 {
fn default() -> Self {
Self::new()
}
}
impl Hasher for Sha256 {
fn write(&mut self, bytes: &[u8]) {
Update::update(&mut self.hasher, bytes);
}
fn reset(&mut self) {
Reset::reset(&mut self.hasher);
}
fn sum(&mut self) -> String {
hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower)
}
fn size(&self) -> usize {
32
}
fn block_size(&self) -> usize {
64
}
}
#[derive(Debug)]
pub struct MD5 {
hasher: Md5,
}
impl MD5 {
pub fn new() -> Self {
Self { hasher: Md5::new() }
}
}
impl Default for MD5 {
fn default() -> Self {
Self::new()
}
}
impl Hasher for MD5 {
fn write(&mut self, bytes: &[u8]) {
self.hasher.update(bytes);
}
fn reset(&mut self) {}
fn sum(&mut self) -> String {
hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower)
}
fn size(&self) -> usize {
32
}
fn block_size(&self) -> usize {
64
}
}
pub struct Uuid {
id: String,
}
impl Uuid {
pub fn new(id: String) -> Self {
Self { id }
}
}
impl Hasher for Uuid {
fn write(&mut self, _bytes: &[u8]) {}
fn reset(&mut self) {}
fn sum(&mut self) -> String {
self.id.clone()
}
fn size(&self) -> usize {
self.id.len()
}
fn block_size(&self) -> usize {
64
}
}
pub fn sum_sha256_hex(data: &[u8]) -> String {
let mut hash = Sha256::new();
hash.write(data);
base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum())
}
pub fn sum_md5_base64(data: &[u8]) -> String {
let mut hash = MD5::new();
hash.write(data);
base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum())
}

7
reader/src/lib.rs Normal file
View File

@@ -0,0 +1,7 @@
pub mod error;
pub mod hasher;
pub mod reader;
pub fn hex(data: impl AsRef<[u8]>) -> String {
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
}

559
reader/src/reader.rs Normal file
View File

@@ -0,0 +1,559 @@
use bytes::Bytes;
use s3s::StdError;
use std::any::Any;
use std::io::Read;
use std::{collections::VecDeque, io::Cursor};
use std::pin::Pin;
use std::task::Poll;
use crate::{
error::ReaderError,
hasher::{HashType, Uuid},
};
// use futures::stream::Stream;
use std::io::{Error, Result};
use super::hasher::{Hasher, Sha256, MD5};
use futures::Stream;
pin_project_lite::pin_project! {
#[derive(Default)]
pub struct EtagReader<S> {
#[pin]
inner: S,
md5: HashType,
checksum:Option<String>,
bytes_read:usize,
}
}
impl<S> EtagReader<S> {
pub fn new(inner: S, etag: Option<String>, force_md5: Option<String>) -> Self {
let md5 = {
if let Some(m) = force_md5 {
HashType::Uuid(Uuid::new(m))
} else {
HashType::Md5(MD5::new())
}
};
Self {
inner,
md5,
checksum: etag,
bytes_read: 0,
}
}
pub fn etag(&mut self) -> String {
self.md5.sum()
}
}
impl<S> Stream for EtagReader<S>
where
S: Stream<Item = std::result::Result<Bytes, StdError>>,
{
type Item = std::result::Result<Bytes, StdError>;
fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.project();
let poll = this.inner.poll_next(cx);
if let Poll::Ready(ref res) = poll {
match res {
Some(Ok(bytes)) => {
*this.bytes_read += bytes.len();
this.md5.write(bytes);
}
Some(Err(err)) => {
return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string())))));
}
None => {
if let Some(etag) = this.checksum {
let got = this.md5.sum();
if got.as_str() != etag.as_str() {
return Poll::Ready(Some(Err(Box::new(ReaderError::VerifyError(etag.to_owned(), got)))));
}
}
}
}
}
poll
}
}
pin_project_lite::pin_project! {
#[derive(Default)]
pub struct HashReader<S> {
#[pin]
inner: S,
sha256: Option<Sha256>,
md5: Option<MD5>,
md5_hex:Option<String>,
sha256_hex:Option<String>,
size:usize,
actual_size: usize,
bytes_read:usize,
}
}
impl<S> HashReader<S> {
pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self {
let md5 = {
if md5_hex.is_some() {
Some(MD5::new())
} else {
None
}
};
let sha256 = {
if sha256_hex.is_some() {
Some(Sha256::new())
} else {
None
}
};
Self {
inner,
size,
actual_size,
md5_hex,
sha256_hex,
bytes_read: 0,
md5,
sha256,
}
}
}
impl<S> Stream for HashReader<S>
where
S: Stream<Item = std::result::Result<Bytes, StdError>>,
{
type Item = std::result::Result<Bytes, StdError>;
fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.project();
let poll = this.inner.poll_next(cx);
if let Poll::Ready(ref res) = poll {
match res {
Some(Ok(bytes)) => {
*this.bytes_read += bytes.len();
if let Some(sha) = this.sha256 {
sha.write(bytes);
}
if let Some(md5) = this.md5 {
md5.write(bytes);
}
}
Some(Err(err)) => {
return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string())))));
}
None => {
if let Some(hash) = this.sha256 {
if let Some(hex) = this.sha256_hex {
let got = hash.sum();
let src = hex.as_str();
if src != got.as_str() {
println!("sha256 err src:{},got:{}", src, got);
return Poll::Ready(Some(Err(Box::new(ReaderError::SHA256Mismatch(src.to_string(), got)))));
}
}
}
if let Some(hash) = this.md5 {
if let Some(hex) = this.md5_hex {
let got = hash.sum();
let src = hex.as_str();
if src != got.as_str() {
// TODO: ERR
println!("md5 err src:{},got:{}", src, got);
return Poll::Ready(Some(Err(Box::new(ReaderError::ChecksumMismatch(src.to_string(), got)))));
}
}
}
}
}
}
// println!("poll {:?}", poll);
poll
}
}
#[async_trait::async_trait]
pub trait Reader {
async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize>;
async fn seek(&mut self, offset: usize) -> Result<()>;
async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize>;
async fn read_all(&mut self) -> Result<Vec<u8>> {
let mut data = Vec::new();
Ok(data)
}
fn as_any(&self) -> &dyn Any;
}
#[derive(Debug)]
pub struct BufferReader {
pub inner: Cursor<Vec<u8>>,
pos: usize,
}
impl BufferReader {
pub fn new(inner: Vec<u8>) -> Self {
Self {
inner: Cursor::new(inner),
pos: 0,
}
}
}
#[async_trait::async_trait]
impl Reader for BufferReader {
#[tracing::instrument(level = "debug", skip(self, buf))]
async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize> {
self.seek(offset).await?;
self.read_exact(buf).await
}
#[tracing::instrument(level = "debug", skip(self))]
async fn seek(&mut self, offset: usize) -> Result<()> {
if self.pos != offset {
self.inner.set_position(offset as u64);
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize> {
let bytes_read = self.inner.read_exact(buf)?;
self.pos += buf.len();
//Ok(bytes_read)
Ok(0)
}
async fn read_all(&mut self) -> Result<Vec<u8>> {
let mut data = Vec::new();
self.inner.read_to_end(&mut data)?;
Ok(data)
}
fn as_any(&self) -> &dyn Any {
self
}
}
pin_project_lite::pin_project! {
pub struct ChunkedStream<S> {
#[pin]
inner: S,
chuck_size: usize,
streams: VecDeque<Bytes>,
remaining:Vec<u8>,
}
}
impl<S> ChunkedStream<S> {
pub fn new(inner: S, chuck_size: usize) -> Self {
Self {
inner,
chuck_size,
streams: VecDeque::new(),
remaining: Vec::new(),
}
}
}
impl<S> Stream for ChunkedStream<S>
where
S: Stream<Item = std::result::Result<Bytes, StdError>> + Send + Sync,
// E: std::error::Error + Send + Sync,
{
type Item = std::result::Result<Bytes, StdError>;
fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
let (items, op_items) = self.inner.size_hint();
let this = self.project();
if let Some(b) = this.streams.pop_front() {
return Poll::Ready(Some(Ok(b)));
}
let poll = this.inner.poll_next(cx);
match poll {
Poll::Ready(res_op) => match res_op {
Some(res) => match res {
Ok(bytes) => {
let chuck_size = *this.chuck_size;
let mut bytes = bytes;
// println!("get len {}", bytes.len());
// 如果有剩余
if !this.remaining.is_empty() {
let need_size = chuck_size - this.remaining.len();
// 传入的数据大小需要补齐的大小，使用传入数据补齐
if bytes.len() >= need_size {
let add_bytes = bytes.split_to(need_size);
this.remaining.extend_from_slice(&add_bytes);
this.streams.push_back(Bytes::from(this.remaining.clone()));
this.remaining.clear();
} else {
// 不够，直接追加
let need_size = bytes.len();
let add_bytes = bytes.split_to(need_size);
this.remaining.extend_from_slice(&add_bytes);
}
}
loop {
if bytes.len() < chuck_size {
break;
}
let chuck = bytes.split_to(chuck_size);
this.streams.push_back(chuck);
}
if !bytes.is_empty() {
this.remaining.extend_from_slice(&bytes);
}
if let Some(b) = this.streams.pop_front() {
return Poll::Ready(Some(Ok(b)));
}
if items > 0 || op_items.is_some() {
return Poll::Pending;
}
if !this.remaining.is_empty() {
let b = this.remaining.clone();
this.remaining.clear();
return Poll::Ready(Some(Ok(Bytes::from(b))));
}
Poll::Ready(None)
}
Err(err) => Poll::Ready(Some(Err(err))),
},
None => {
// println!("get empty");
if let Some(b) = this.streams.pop_front() {
return Poll::Ready(Some(Ok(b)));
}
if !this.remaining.is_empty() {
let b = this.remaining.clone();
this.remaining.clear();
return Poll::Ready(Some(Ok(Bytes::from(b))));
}
Poll::Ready(None)
}
},
Poll::Pending => {
// println!("get Pending");
Poll::Pending
}
}
// if let Poll::Ready(Some(res)) = poll {
// warn!("poll res ...");
// match res {
// Ok(bytes) => {
// let chuck_size = *this.chuck_size;
// let mut bytes = bytes;
// if this.remaining.len() > 0 {
// let need_size = chuck_size - this.remaining.len();
// let add_bytes = bytes.split_to(need_size);
// this.remaining.extend_from_slice(&add_bytes);
// warn!("poll push_back remaining ...1");
// this.streams.push_back(Bytes::from(this.remaining.clone()));
// this.remaining.clear();
// }
// loop {
// if bytes.len() < chuck_size {
// break;
// }
// let chuck = bytes.split_to(chuck_size);
// warn!("poll push_back ...1");
// this.streams.push_back(chuck);
// }
// warn!("poll remaining extend_from_slice...1");
// this.remaining.extend_from_slice(&bytes);
// }
// Err(err) => return Poll::Ready(Some(Err(err))),
// }
// }
// if let Some(b) = this.streams.pop_front() {
// warn!("poll pop_front ...");
// return Poll::Ready(Some(Ok(b)));
// }
// if this.remaining.len() > 0 {
// let b = this.remaining.clone();
// this.remaining.clear();
// warn!("poll remaining ...1");
// return Poll::Ready(Some(Ok(Bytes::from(b))));
// }
// Poll::Pending
}
fn size_hint(&self) -> (usize, Option<usize>) {
let mut items = self.streams.len();
if !self.remaining.is_empty() {
items += 1;
}
(items, Some(items))
}
}
#[cfg(test)]
mod test {
use super::*;
use futures::StreamExt;
#[tokio::test]
async fn test_etag_reader() {
let data1 = vec![1u8; 60]; // 65536
let data2 = vec![0u8; 32]; // 65536
let chunk1 = Bytes::from(data1);
let chunk2 = Bytes::from(data2);
let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2)];
let mut stream = futures::stream::iter(chunk_results);
let mut hash_reader = EtagReader::new(&mut stream, None, None);
// let chunk_size = 8;
// let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size);
loop {
match hash_reader.next().await {
Some(res) => match res {
Ok(bytes) => {
println!("bytes: {}, {:?}", bytes.len(), bytes);
}
Err(err) => {
println!("err:{:?}", err);
break;
}
},
None => {
println!("next none");
break;
}
}
}
println!("etag:{}", hash_reader.etag());
// 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45
// println!("md5: {:?}", hash_reader.hex());
}
#[tokio::test]
async fn test_hash_reader() {
let data1 = vec![1u8; 60]; // 65536
let data2 = vec![0u8; 32]; // 65536
let size = data1.len() + data2.len();
let chunk1 = Bytes::from(data1);
let chunk2 = Bytes::from(data2);
let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2)];
let mut stream = futures::stream::iter(chunk_results);
let mut hash_reader = HashReader::new(
&mut stream,
size,
Some("d94c485610a7a00a574df55e45d3cc0c".to_string()),
Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()),
0,
);
// let chunk_size = 8;
// let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size);
loop {
match hash_reader.next().await {
Some(res) => match res {
Ok(bytes) => {
println!("bytes: {}, {:?}", bytes.len(), bytes);
}
Err(err) => {
println!("err:{:?}", err);
break;
}
},
None => {
println!("next none");
break;
}
}
}
// BUG: borrow of moved value: `md5_stream`
// 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45
// println!("md5: {:?}", hash_reader.hex());
}
#[tokio::test]
async fn test_chunked_stream() {
let data1 = vec![1u8; 60]; // 65536
let data2 = vec![0u8; 33]; // 65536
let data3 = vec![4u8; 5]; // 65536
let chunk1 = Bytes::from(data1);
let chunk2 = Bytes::from(data2);
let chunk3 = Bytes::from(data3);
let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2), Ok(chunk3)];
let mut stream = futures::stream::iter(chunk_results);
// let mut hash_reader = HashReader::new(
// &mut stream,
// size,
// Some("d94c485610a7a00a574df55e45d3cc0c".to_string()),
// Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()),
// 0,
// );
let chunk_size = 8;
let mut etag_reader = EtagReader::new(&mut stream, None, None);
let mut chunked_stream = ChunkedStream::new(&mut etag_reader, chunk_size);
loop {
match chunked_stream.next().await {
Some(res) => match res {
Ok(bytes) => {
println!("bytes: {}, {:?}", bytes.len(), bytes);
}
Err(err) => {
println!("err:{:?}", err);
break;
}
},
None => {
println!("next none");
break;
}
}
}
println!("etag:{}", etag_reader.etag());
}
}

1
rustfs/src/admin/handlers.rs
View File

@@ -65,6 +65,7 @@ pub mod service_account;
pub mod sts;
pub mod trace;
pub mod user;
pub mod tier;
use urlencoding::decode;
#[derive(Debug, Serialize, Default)]

364
rustfs/src/admin/handlers/tier.rs Normal file
View File

@@ -0,0 +1,364 @@
use std::str::from_utf8;
use http::{HeaderMap, StatusCode};
use iam::get_global_action_cred;
use matchit::Params;
use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
use serde::Deserialize;
use serde_urlencoded::from_bytes;
use tracing::{warn, debug, info};
use crypto::{encrypt_data, decrypt_data};
use crate::{
admin::{router::Operation, utils::has_space_be},
auth::{check_key_valid, get_session_token},
};
use ecstore::{
client::admin_handler_utils::AdminError,
config::storageclass,
global::GLOBAL_TierConfigMgr,
tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, ERR_TIER_CONNECT_ERR,
ERR_TIER_INVALID_CREDENTIALS,
},
tier::{ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_BACKEND_IN_USE},
},
};
#[derive(Debug, Deserialize, Default)]
pub struct AddTierQuery {
#[serde(rename = "accessKey")]
pub access_key: Option<String>,
pub status: Option<String>,
pub tier: Option<String>,
pub force: String,
}
pub struct AddTier {}
#[async_trait::async_trait]
impl Operation for AddTier {
#[tracing::instrument(level = "debug", skip(self))]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
input
} else {
AddTierQuery::default()
}
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, _owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
let mut input = req.input;
let body = match input.store_all_unlimited().await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
let mut args: TierConfig = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?;
match args.tier_type {
TierType::S3 => {
args.name = args.s3.clone().unwrap().name;
}
TierType::RustFS => {
args.name = args.rustfs.clone().unwrap().name;
}
TierType::MinIO => {
args.name = args.minio.clone().unwrap().name;
}
_ => (),
}
debug!("add tier args {:?}", args);
let mut force: bool = false;
let force_str = query.force;
if force_str != "" {
force = force_str.parse().unwrap();
}
match args.name.as_str() {
storageclass::STANDARD | storageclass::RRS => {
warn!("tier reserved name, args.name: {}", args.name);
return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
}
&_ => ()
}
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
//tier_config_mgr.reload(api);
match tier_config_mgr.add(args, force).await {
Err(ERR_TIER_ALREADY_EXISTS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameAlreadyExist".into()), "tier name already exists!"));
}
Err(ERR_TIER_NAME_NOT_UPPERCASE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameNotUppercase".into()), "tier name not uppercase!"));
}
Err(ERR_TIER_BACKEND_IN_USE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse!".into()), "tier name backend in use!"));
}
Err(ERR_TIER_CONNECT_ERR) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierConnectError".into()), "tier connect error!"));
}
Err(ERR_TIER_INVALID_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), ERR_TIER_INVALID_CREDENTIALS.message));
}
Err(e) => {
warn!("tier_config_mgr add failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier add failed"));
}
Ok(_) => (),
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed"));
}
//globalNotificationSys.LoadTransitionTierConfig(ctx);
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
}
}
pub struct EditTier {}
#[async_trait::async_trait]
impl Operation for EditTier {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
input
} else {
AddTierQuery::default()
}
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, _owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
//{"accesskey":"gggggg","secretkey":"jjjjjjj"}
//{"detailedMessage":"failed to perform PUT: The Access Key Id you provided does not exist in our records.","message":"an error occurred, please try again"}
//200 OK
let mut input = req.input;
let body = match input.store_all_unlimited().await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
let creds: TierCreds = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?;
debug!("edit tier args {:?}", creds);
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
//tier_config_mgr.reload(api);
match tier_config_mgr.edit(&tier_name, creds).await {
Err(ERR_TIER_NOT_FOUND) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!"));
}
Err(ERR_TIER_MISSING_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierMissingCredentials".into()), "tier missing credentials!"));
}
Err(e) => {
warn!("tier_config_mgr edit failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier edit failed"));
}
Ok(_) => (),
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
}
//globalNotificationSys.LoadTransitionTierConfig(ctx);
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
}
}
#[derive(Debug, Deserialize, Default)]
pub struct BucketQuery {
#[serde(rename = "bucket")]
pub bucket: String,
}
pub struct ListTiers {}
#[async_trait::async_trait]
impl Operation for ListTiers {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let query = {
if let Some(query) = req.uri.query() {
let input: BucketQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
input
} else {
BucketQuery::default()
}
};
//{"items":[{"minio":{"accesskey":"minioadmin","bucket":"mblock2","endpoint":"http://192.168.1.11:9020","name":"COLDTIER","objects":"0","prefix":"mypre/","secretkey":"REDACTED","usage":"0 B","versions":"0"},"status":true,"type":"minio"}]}
let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await;
let tiers = tier_config_mgr.list_tiers();
let data = serde_json::to_vec(&tiers)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tiers err {}", e)))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
}
}
pub struct RemoveTier {}
#[async_trait::async_trait]
impl Operation for RemoveTier {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RemoveTier");
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
input
} else {
AddTierQuery::default()
}
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, _owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
let sys_cred = get_global_action_cred()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?;
let mut force: bool = false;
let force_str = query.force;
if force_str != "" {
force = force_str.parse().unwrap();
}
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
//tier_config_mgr.reload(api);
match tier_config_mgr.remove(&tier_name, force).await {
Err(ERR_TIER_NOT_FOUND) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found."));
}
Err(ERR_TIER_BACKEND_NOT_EMPTY) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used."));
}
Err(e) => {
warn!("tier_config_mgr remove failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier remove failed"));
}
Ok(_) => (),
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed"));
}
//globalNotificationSys.LoadTransitionTierConfig(ctx);
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
}
}
pub struct VerifyTier {}
#[async_trait::async_trait]
impl Operation for VerifyTier {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RemoveTier");
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
input
} else {
AddTierQuery::default()
}
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, _owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
let sys_cred = get_global_action_cred()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?;
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
tier_config_mgr.verify(&query.tier.unwrap());
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
}
}
pub struct GetTierInfo {}
#[async_trait::async_trait]
impl Operation for GetTierInfo {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetTierInfo");
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
input
} else {
AddTierQuery::default()
}
};
let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await;
let info = tier_config_mgr.get(&query.tier.unwrap());
let data = serde_json::to_vec(&info)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tier err {}", e)))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
}
}

35
rustfs/src/admin/mod.rs
View File

@@ -7,7 +7,7 @@ pub mod utils;
use handlers::{
group, policys, pools, rebalance,
service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
sts, user,
sts, user, tier,
};
use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
@@ -311,5 +311,38 @@ fn register_user_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()>
AdminOperation(&policys::SetPolicyForUserOrGroup {}),
)?;
// ?
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(),
AdminOperation(&tier::ListTiers {}),
)?;
// ?
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/tier-stats").as_str(),
AdminOperation(&tier::GetTierInfo {}),
)?;
// ?force=xxx
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(),
AdminOperation(&tier::RemoveTier {}),
)?;
// ?force=xxx
// body: AddOrUpdateTierReq
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(),
AdminOperation(&tier::AddTier {}),
)?;
// ?
// body: AddOrUpdateTierReq
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(),
AdminOperation(&tier::EditTier {}),
)?;
Ok(())
}

49
rustfs/src/storage/ecfs.rs
View File

@@ -17,6 +17,7 @@ use chrono::Utc;
use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use datafusion::arrow::json::writer::JsonArray;
use ecstore::bucket::error::BucketMetadataError;
use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG;
use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG;
@@ -52,6 +53,7 @@ use ecstore::bucket::utils::serialize;
use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
use futures::pin_mut;
use futures::{Stream, StreamExt};
use http::HeaderMap;
@@ -93,6 +95,14 @@ use tracing::warn;
use transform_stream::AsyncTryStream;
use uuid::Uuid;
use ecstore::bucket::{
lifecycle::{
lifecycle::Lifecycle,
bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS
},
object_lock::objectlock_sys::BucketObjectLockSys,
};
macro_rules! try_ {
($result:expr) => {
match $result {
@@ -1590,12 +1600,26 @@ impl S3 for FS {
..
} = req.input;
// warn!("lifecycle_configuration {:?}", &lifecycle_configuration);
let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if rcfg.is_err() {
return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockIsNotExist".into()), "bucket lock is not exist."));
}
let rcfg = rcfg.expect("get_lifecycle_config err!").0;
// TODO: objcetLock
//info!("lifecycle_configuration: {:?}", &lifecycle_configuration);
let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
if let Err(err) = input_cfg.validate(&rcfg).await {
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed."));
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), format!("{}", err.to_string())));
}
if let Err(err) = validate_transition_tier(&input_cfg).await {
//warn!("lifecycle_configuration add failed, err: {:?}", err);
return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), format!("{}", err.to_string())));
}
let data = try_!(serialize(&input_cfg));
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data)
.await
@@ -2036,6 +2060,27 @@ impl S3 for FS {
}))
}
async fn get_object_attributes(
&self,
req: S3Request<GetObjectAttributesInput>,
) -> S3Result<S3Response<GetObjectAttributesOutput>> {
let GetObjectAttributesInput { bucket, key, .. } = req.input;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
if let Err(e) = store.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()).await {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
Ok(S3Response::new(GetObjectAttributesOutput {
delete_marker: None,
object_parts: None,
..Default::default()
}))
}
async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
let PutObjectAclInput {
bucket,