sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-17 01:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #526 from rustfs/feature-fix-keyw/ilm

Browse Source 
Feature fix keyw/ilm
This commit is contained in:
weisd
2025-07-01 09:43:03 +08:00
committed by GitHub
parent 38220697e0 3fd70de4a6
commit a2828a8015
12 changed files with 83 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/signer/src/request_signature_v2.rs
View File

@@ -13,8 +13,8 @@ const SIGN_V2_ALGORITHM: &str = "AWS";
fn encode_url2path(req: &request::Builder, _virtual_host: bool) -> String {
//path = serde_urlencoded::to_string(req.uri_ref().unwrap().path().unwrap()).unwrap();
let path = req.uri_ref().unwrap().path().to_string();
path
req.uri_ref().unwrap().path().to_string()
}
pub fn pre_sign_v2(

82
crates/signer/src/request_signature_v4.rs
View File

@@ -399,85 +399,3 @@ pub fn sign_v4_trailer(
trailer,
)
}
#[cfg(test)]
#[allow(unused_variables, unused_mut)]
mod tests {
use http::request;
use time::macros::datetime;
use super::*;
#[test]
fn example_list_objects() {
// let access_key_id = "AKIAIOSFODNN7EXAMPLE";
let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
let timestamp = "20130524T000000Z";
let t = datetime!(2013-05-24 0:00 UTC);
// let bucket = "examplebucket";
let region = "us-east-1";
let service = "s3";
let path = "/";
let mut req = request::Request::builder()
.method(http::Method::GET)
.uri("http://examplebucket.s3.amazonaws.com/?");
let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap());
let query = vec![
("max-keys".to_string(), "2".to_string()),
("prefix".to_string(), "J".to_string()),
];
let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
assert_eq!(
canonical_request,
concat!(
"GET\n",
"/\n",
"max-keys=2&prefix=J\n",
"host:examplebucket.s3.amazonaws.com\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:",
"20130524T000000Z",
"\n",
"\n",
"host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
)
);
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
assert_eq!(
string_to_sign,
concat!(
"AWS4-HMAC-SHA256\n",
"20130524T000000Z",
"\n",
"20130524/us-east-1/s3/aws4_request\n",
"df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7",
)
);
let signing_key = get_signing_key(secret_access_key, region, t, service);
let signature = get_signature(signing_key, &string_to_sign);
assert_eq!(signature, "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7");
}
}

81
crates/utils/src/net.rs
View File

@@ -5,17 +5,12 @@ use hyper::client::conn::http2::Builder;
use hyper_util::rt::TokioExecutor;
use lazy_static::lazy_static;
use std::{
collections::{HashMap, HashSet},
collections::HashSet,
fmt::Display,
net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
};
use transform_stream::AsyncTryStream;
use url::{Host, Url};
//use hyper::{client::conn::http2::Builder, rt::Executor};
//use tonic::{SharedExec, UserAgent};
//use hyper_util::rt::TokioTimer;
use s3s::header::X_AMZ_STORAGE_CLASS;
lazy_static! {
static ref LOCAL_IPS: Vec<IpAddr> = must_get_local_ips().unwrap();
@@ -140,86 +135,12 @@ pub fn new_remotetarget_http_transport(_insecure: bool) -> Builder<TokioExecutor
todo!();
}
lazy_static! {
static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("attributes".to_string(), true);
m.insert("partNumber".to_string(), true);
m.insert("versionId".to_string(), true);
m.insert("response-cache-control".to_string(), true);
m.insert("response-content-disposition".to_string(), true);
m.insert("response-content-encoding".to_string(), true);
m.insert("response-content-language".to_string(), true);
m.insert("response-content-type".to_string(), true);
m.insert("response-expires".to_string(), true);
m
};
static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("content-type".to_string(), true);
m.insert("cache-control".to_string(), true);
m.insert("content-encoding".to_string(), true);
m.insert("content-disposition".to_string(), true);
m.insert("content-language".to_string(), true);
m.insert("x-amz-website-redirect-location".to_string(), true);
m.insert("x-amz-object-lock-mode".to_string(), true);
m.insert("x-amz-metadata-directive".to_string(), true);
m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
m.insert("expires".to_string(), true);
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
SUPPORTED_QUERY_VALUES[qs_key]
}
const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-";
pub fn is_custom_query_value(qs_key: &str) -> bool {
qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX)
}
pub fn is_storageclass_header(header_key: &str) -> bool {
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
}
pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
}
pub fn is_rustfs_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-rustfs-")
}
pub fn is_rustfs_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-rustfs-")
}
#[derive(Debug, Clone)]
pub struct XHost {
pub name: String,

1
ecstore/src/client/api_error_response.rs
View File

@@ -89,7 +89,6 @@ pub fn http_resp_to_error_response(
object_name: &str,
) -> ErrorResponse {
let err_body = String::from_utf8(b).unwrap();
//let err_body = xml_decode_and_body(resp.body, &err_resp);
let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
let mut err_resp = ErrorResponse::default();
if err_resp_.is_err() {

2
ecstore/src/client/api_put_object.rs
View File

@@ -28,10 +28,10 @@ use crate::client::{
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
credentials::SignatureType,
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
};
use rustfs_utils::{
crypto::base64_encode,
net::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
};
#[derive(Debug, Clone)]

3
ecstore/src/client/api_remove.rs
View File

@@ -64,9 +64,6 @@ pub struct RemoveObjectOptions {
impl TransitionClient {
pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
let headers = HeaderMap::new();
/*if opts.force_delete {
headers.insert(rustFSForceDelete, "true");
}*/
let resp = self
.execute_method(

1
ecstore/src/client/mod.rs
View File

@@ -16,3 +16,4 @@ pub mod credentials;
pub mod object_api_utils;
pub mod object_handlers_common;
pub mod transition_api;
pub mod utils;

78
ecstore/src/client/utils.rs Normal file
View File

@@ -0,0 +1,78 @@
use lazy_static::lazy_static;
use std::collections::HashMap;
use s3s::header::X_AMZ_STORAGE_CLASS;
lazy_static! {
static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("attributes".to_string(), true);
m.insert("partNumber".to_string(), true);
m.insert("versionId".to_string(), true);
m.insert("response-cache-control".to_string(), true);
m.insert("response-content-disposition".to_string(), true);
m.insert("response-content-encoding".to_string(), true);
m.insert("response-content-language".to_string(), true);
m.insert("response-content-type".to_string(), true);
m.insert("response-expires".to_string(), true);
m
};
static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("content-type".to_string(), true);
m.insert("cache-control".to_string(), true);
m.insert("content-encoding".to_string(), true);
m.insert("content-disposition".to_string(), true);
m.insert("content-language".to_string(), true);
m.insert("x-amz-website-redirect-location".to_string(), true);
m.insert("x-amz-object-lock-mode".to_string(), true);
m.insert("x-amz-metadata-directive".to_string(), true);
m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
m.insert("expires".to_string(), true);
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
SUPPORTED_QUERY_VALUES[qs_key]
}
pub fn is_storageclass_header(header_key: &str) -> bool {
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
}
pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
}
pub fn is_rustfs_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-rustfs-")
}
pub fn is_minio_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-minio-")
}

33
ecstore/src/error.rs
View File

@@ -817,30 +817,9 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i
let version_id = version_id.to_string();
match r_err.code {
/*S3ErrorCode::SlowDownWrite => {
err = StorageError::InsufficientWriteQuorum;//{bucket: bucket, object: object};
}*/
/*S3ErrorCode::SlowDown/* SlowDownRead */ => {
err = std::io::Error::other(StorageError::InsufficientReadQuorum.to_string());
}*/
/*S3ErrorCode::PreconditionFailed => {
err = Error::from(StorageError::PreConditionFailed);
}*/
/*S3ErrorCode::InvalidRange => {
err = Error::from(StorageError::InvalidRange);
}*/
/*S3ErrorCode::BucketAlreadyOwnedByYou => {
err = Error::from(StorageError::BucketAlreadyOwnedByYou);
}*/
S3ErrorCode::BucketNotEmpty => {
err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string());
}
/*S3ErrorCode::NoSuchBucketPolicy => {
err = Error::from(StorageError::BucketPolicyNotFound);
}*/
/*S3ErrorCode::NoSuchLifecycleConfiguration => {
err = Error::from(StorageError::BucketLifecycleNotFound);
}*/
S3ErrorCode::InvalidBucketName => {
err = std::io::Error::other(StorageError::BucketNameInvalid(bucket));
}
@@ -864,24 +843,12 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
}
/*S3ErrorCode::XRustFsInvalidObjectName => {
err = Error::from(StorageError::ObjectNameInvalid(bucket, object));
}*/
S3ErrorCode::AccessDenied => {
err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object));
}
/*S3ErrorCode::XAmzContentSHA256Mismatch => {
err = hash.SHA256Mismatch{};
}*/
S3ErrorCode::NoSuchUpload => {
err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id));
}
/*S3ErrorCode::EntityTooSmall => {
err = std::io::Error::other(StorageError::PartTooSmall);
}*/
/*S3ErrorCode::ReplicationPermissionCheck => {
err = std::io::Error::other(StorageError::ReplicationPermissionCheck);
}*/
_ => {
err = err_;
}

2
ecstore/src/tier/tier_config.rs
View File

@@ -110,8 +110,6 @@ impl Clone for TierConfig {
tier_type: self.tier_type.clone(),
name: self.name.clone(),
s3,
//azure: az,
//gcs: gcs,
rustfs: r,
minio: m,
}

1
ecstore/src/tier/warm_backend_minio.rs
View File

@@ -63,7 +63,6 @@ impl WarmBackendMinIO {
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));

1
ecstore/src/tier/warm_backend_rustfs.rs
View File

@@ -60,7 +60,6 @@ impl WarmBackendRustFS {
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));