sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-17 01:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Windows Path Separator Handling in rustfs_utils (#1464)

Browse Source 
Co-authored-by: reatang <tangtang1251@qq.com>
This commit is contained in:
houseme
2026-01-11 19:53:51 +08:00
committed by GitHub
parent 6b2eebee1d
commit 760cb1d734
18 changed files with 999 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

341
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

9
Cargo.toml
View File

@@ -130,7 +130,7 @@ bytesize = "2.3.1"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.1"
prost = "0.14.3"
quick-xml = "0.38.4"
rmcp = { version = "0.12.0" }
rmp = { version = "0.8.15" }
@@ -143,7 +143,7 @@ schemars = "1.2.0"
# Cryptography and Security
aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
argon2 = { version = "0.6.0-rc.5" }
blake3 = { version = "1.8.2", features = ["rayon", "mmap"] }
blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
chacha20poly1305 = { version = "0.11.0-rc.2" }
crc-fast = "1.6.0"
hmac = { version = "0.13.0-rc.3" }
@@ -184,6 +184,7 @@ criterion = { version = "0.8", features = ["html_reports"] }
crossbeam-queue = "0.3.12"
datafusion = "51.0.0"
derive_builder = "0.20.2"
dunce = "1.0.5"
enumset = "1.1.10"
faster-hex = "0.10.0"
flate2 = "1.1.5"
@@ -197,7 +198,7 @@ hex-simd = "0.8.0"
highway = { version = "1.3.0" }
ipnetwork = { version = "0.21.1", features = ["serde"] }
lazy_static = "1.5.0"
libc = "0.2.179"
libc = "0.2.180"
libsystemd = "0.7.2"
local-ip-address = "0.6.8"
lz4 = "1.28.1"
@@ -270,7 +271,7 @@ libunftp = "0.21.0"
russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false }
russh-sftp = "2.1.1"
ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] }
suppaftp = { version = "7.0.7", features = ["tokio", "tokio-rustls", "rustls"] }
suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] }
rcgen = "0.14.6"
# Performance Analysis and Memory Profiling

2
crates/ecstore/Cargo.toml
View File

@@ -48,6 +48,7 @@ async-trait.workspace = true
bytes.workspace = true
byteorder = { workspace = true }
chrono.workspace = true
dunce.workspace = true
glob = { workspace = true }
thiserror.workspace = true
flatbuffers.workspace = true
@@ -109,7 +110,6 @@ google-cloud-auth = { workspace = true }
aws-config = { workspace = true }
faster-hex = { workspace = true }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
criterion = { workspace = true, features = ["html_reports"] }

6
crates/ecstore/src/bucket/utils.rs
View File

@@ -14,7 +14,7 @@
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::SLASH_SEPARATOR;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use s3s::xml;
pub fn is_meta_bucketname(name: &str) -> bool {
@@ -194,7 +194,7 @@ pub fn is_valid_object_name(object: &str) -> bool {
return false;
}
if object.ends_with(SLASH_SEPARATOR) {
if object.ends_with(SLASH_SEPARATOR_STR) {
return false;
}
@@ -206,7 +206,7 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned()));
}
if object.starts_with(SLASH_SEPARATOR) {
if object.starts_with(SLASH_SEPARATOR_STR) {
return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned()));
}

6
crates/ecstore/src/config/com.rs
View File

@@ -18,7 +18,7 @@ use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_utils::path::SLASH_SEPARATOR;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::LazyLock;
@@ -29,7 +29,7 @@ const CONFIG_FILE: &str = "config.json";
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{CONFIG_PREFIX}"));
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
let mut h = HashSet::new();
@@ -129,7 +129,7 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
}
fn get_config_file() -> String {
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR_STR}{CONFIG_FILE}")
}
/// Handle the situation where the configuration file does not exist, create and save a new configuration

10
crates/ecstore/src/data_usage.rs
View File

@@ -31,14 +31,14 @@ use crate::{
use rustfs_common::data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
};
use rustfs_utils::path::SLASH_SEPARATOR;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use tokio::fs;
use tracing::{error, info, warn};
use crate::error::Error;
// Data usage storage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
@@ -47,17 +47,17 @@ pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
lazy_static::lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
crate::disk::RUSTFS_META_BUCKET,
SLASH_SEPARATOR,
SLASH_SEPARATOR_STR,
crate::disk::BUCKET_META_PREFIX
);
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR,
SLASH_SEPARATOR_STR,
DATA_USAGE_OBJ_NAME
);
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR,
SLASH_SEPARATOR_STR,
DATA_USAGE_BLOOM_NAME
);
}

122
crates/ecstore/src/disk/local.rs
View File

@@ -12,39 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::error::{Error, Result};
use super::os::{is_root_disk, rename_all};
use super::{
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os,
};
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
use crate::disk::error::FileAccessDeniedWithContext;
use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
use crate::disk::fs::{
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename,
};
use crate::disk::os::{check_path_length, is_empty_dir};
use crate::disk::{
CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND,
FileReader, RUSTFS_META_TMP_DELETED_BUCKET, conv_part_err_to_int,
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts,
VolumeInfo, WalkDirOptions, conv_part_err_to_int,
endpoint::Endpoint,
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3,
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
os,
os::{check_path_length, is_empty_dir, is_root_disk, rename_all},
};
use crate::disk::{FileWriter, STORAGE_FORMAT_FILE};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use rustfs_utils::path::{
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
path_join, path_join_buf,
};
use tokio::time::interval;
use crate::erasure_coding::bitrot_verify;
use bytes::Bytes;
// use path_absolutize::Absolutize; // Replaced with direct path operations for better performance
use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use bytes::Bytes;
use parking_lot::RwLock as ParkingLotRwLock;
use rustfs_filemeta::{
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
@@ -52,6 +39,10 @@ use rustfs_filemeta::{
};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::os::get_info;
use rustfs_utils::path::{
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR_STR, clean, decode_dir_object, encode_dir_object,
has_suffix, path_join, path_join_buf,
};
use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt::Debug;
@@ -67,6 +58,7 @@ use time::OffsetDateTime;
use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
use tokio::sync::RwLock;
use tokio::time::interval;
use tracing::{debug, error, info, warn};
use uuid::Uuid;
@@ -129,7 +121,8 @@ impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk");
// Use optimized path resolution instead of absolutize() for better performance
let root = match std::fs::canonicalize(ep.get_file_path()) {
// Use dunce::canonicalize instead of std::fs::canonicalize to avoid UNC paths on Windows
let root = match dunce::canonicalize(ep.get_file_path()) {
Ok(path) => path,
Err(e) => {
if e.kind() == ErrorKind::NotFound {
@@ -483,7 +476,7 @@ impl LocalDisk {
// Async prefetch related files, don't block current read
if let Some(parent) = file_path.parent() {
prefetch_metadata_patterns(parent, &[super::STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
}
// Main read logic
@@ -507,7 +500,7 @@ impl LocalDisk {
async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
let paths: Vec<PathBuf> = requests
.iter()
.map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, super::STORAGE_FORMAT_FILE)))
.map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE)))
.collect::<Result<Vec<_>>>()?;
let cache = get_global_file_cache();
@@ -544,7 +537,7 @@ impl LocalDisk {
// TODO: async notifications for disk space checks and trash cleanup
let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
// if let Some(parent) = trash_path.parent() {
// if !parent.exists() {
// fs::create_dir_all(parent).await?;
@@ -552,7 +545,7 @@ impl LocalDisk {
// }
let err = if recursive {
rename_all(delete_path, trash_path, self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?)
rename_all(delete_path, trash_path, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?)
.await
.err()
} else {
@@ -562,12 +555,12 @@ impl LocalDisk {
.err()
};
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) {
let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR_STR) {
let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
let _ = rename_all(
encode_dir_object(delete_path.to_string_lossy().as_ref()),
trash_path2,
self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?,
self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?,
)
.await;
}
@@ -916,7 +909,7 @@ impl LocalDisk {
}
if let Some(parent) = path.as_ref().parent() {
super::os::make_dir_all(parent, skip_parent).await?;
os::make_dir_all(parent, skip_parent).await?;
}
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
@@ -942,7 +935,7 @@ impl LocalDisk {
let meta = file.metadata().await.map_err(to_file_error)?;
let file_size = meta.len() as usize;
bitrot_verify(Box::new(file), file_size, part_size, algo, bytes::Bytes::copy_from_slice(sum), shard_size)
bitrot_verify(Box::new(file), file_size, part_size, algo, Bytes::copy_from_slice(sum), shard_size)
.await
.map_err(to_file_error)?;
@@ -1038,15 +1031,16 @@ impl LocalDisk {
continue;
}
if entry.ends_with(SLASH_SEPARATOR) {
if entry.ends_with(SLASH_SEPARATOR_STR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
let entry =
format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR_STR);
dir_objes.insert(entry.clone());
*item = entry;
continue;
}
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned();
*item = entry.trim_end_matches(SLASH_SEPARATOR_STR).to_owned();
continue;
}
@@ -1058,7 +1052,7 @@ impl LocalDisk {
.await?;
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
let name = entry.trim_end_matches(SLASH_SEPARATOR);
let name = entry.trim_end_matches(SLASH_SEPARATOR_STR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
// if opts.limit > 0
@@ -1141,7 +1135,7 @@ impl LocalDisk {
Ok(res) => {
if is_dir_obj {
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
meta.name.push_str(SLASH_SEPARATOR);
meta.name.push_str(SLASH_SEPARATOR_STR);
}
meta.metadata = res;
@@ -1159,7 +1153,7 @@ impl LocalDisk {
// NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
meta.name.push_str(SLASH_SEPARATOR);
meta.name.push_str(SLASH_SEPARATOR_STR);
dir_stack.push(meta.name);
}
}
@@ -1234,7 +1228,7 @@ async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
fn skip_access_checks(p: impl AsRef<str>) -> bool {
let vols = [
super::RUSTFS_META_TMP_DELETED_BUCKET,
RUSTFS_META_TMP_DELETED_BUCKET,
super::RUSTFS_META_TMP_BUCKET,
super::RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_BUCKET,
@@ -1628,8 +1622,8 @@ impl DiskAPI for LocalDisk {
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
}
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
warn!(
@@ -1695,8 +1689,8 @@ impl DiskAPI for LocalDisk {
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) {
return Err(Error::from(DiskError::FileAccessDenied));
}
@@ -1847,12 +1841,12 @@ impl DiskAPI for LocalDisk {
}
let volume_dir = self.get_bucket_path(volume)?;
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?;
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR_STR))?;
let entries = match os::read_dir(&dir_path_abs, count).await {
Ok(res) => res,
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound
if e.kind() == ErrorKind::NotFound
&& !skip_access_checks(volume)
&& let Err(e) = access(&volume_dir).await
{
@@ -1883,11 +1877,11 @@ impl DiskAPI for LocalDisk {
let mut objs_returned = 0;
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
if opts.base_dir.ends_with(SLASH_SEPARATOR_STR) {
let fpath = self.get_object_path(
&opts.bucket,
path_join_buf(&[
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(),
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX).as_str(),
STORAGE_FORMAT_FILE,
])
.as_str(),
@@ -2119,7 +2113,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.get_bucket_path(volume)?;
if let Err(e) = access(&volume_dir).await {
if e.kind() == std::io::ErrorKind::NotFound {
if e.kind() == ErrorKind::NotFound {
os::make_dir_all(&volume_dir, self.root.as_path()).await?;
return Ok(());
}
@@ -2137,7 +2131,7 @@ impl DiskAPI for LocalDisk {
let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?;
for entry in entries {
if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) {
if !has_suffix(&entry, SLASH_SEPARATOR_STR) || !Self::is_valid_volname(clean(&entry).as_str()) {
continue;
}
@@ -2359,7 +2353,7 @@ impl DiskAPI for LocalDisk {
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
if path.starts_with(SLASH_SEPARATOR) {
if path.starts_with(SLASH_SEPARATOR_STR) {
return self
.delete(
volume,
@@ -2420,7 +2414,7 @@ impl DiskAPI for LocalDisk {
if !meta.versions.is_empty() {
let buf = meta.marshal_msg()?;
return self
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
.await;
}
@@ -2430,11 +2424,11 @@ impl DiskAPI for LocalDisk {
{
let src_path = path_join(&[
file_path.as_path(),
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
]);
let dst_path = path_join(&[
file_path.as_path(),
Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()),
Path::new(format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str()),
]);
return rename_all(src_path, dst_path, file_path).await;
}
@@ -2563,7 +2557,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
if root_disk_threshold > 0 {
disk_info.total <= root_disk_threshold
} else {
is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default()
is_root_disk(&drive_path, SLASH_SEPARATOR_STR).unwrap_or_default()
}
} else {
false
@@ -2581,7 +2575,7 @@ mod test {
// let arr = Vec::new();
let vols = [
super::super::RUSTFS_META_TMP_DELETED_BUCKET,
RUSTFS_META_TMP_DELETED_BUCKET,
super::super::RUSTFS_META_TMP_BUCKET,
super::super::RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_BUCKET,
@@ -2609,9 +2603,7 @@ mod test {
let disk = LocalDisk::new(&ep, false).await.unwrap();
let tmpp = disk
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
.unwrap();
let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
println!("ppp :{:?}", &tmpp);
@@ -2639,9 +2631,7 @@ mod test {
let disk = LocalDisk::new(&ep, false).await.unwrap();
let tmpp = disk
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
.unwrap();
let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
println!("ppp :{:?}", &tmpp);

4
crates/ecstore/src/disk/os.rs
View File

@@ -19,7 +19,7 @@ use std::{
use super::error::Result;
use crate::disk::error_conv::to_file_error;
use rustfs_utils::path::SLASH_SEPARATOR;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use tokio::fs;
use tracing::warn;
@@ -118,7 +118,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
if file_type.is_file() {
volumes.push(name);
} else if file_type.is_dir() {
volumes.push(format!("{name}{SLASH_SEPARATOR}"));
volumes.push(format!("{name}{SLASH_SEPARATOR_STR}"));
}
count -= 1;
if count == 0 {

6
crates/ecstore/src/pools.rs
View File

@@ -38,7 +38,7 @@ use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{HashReader, WarpReader};
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, encode_dir_object, path_join};
use rustfs_workers::workers::Workers;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -451,10 +451,10 @@ fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, S
let trimmed_path = path
.strip_prefix(base_path)
.unwrap_or(path)
.strip_prefix(SLASH_SEPARATOR)
.strip_prefix(SLASH_SEPARATOR_STR)
.unwrap_or(path);
// Find the position of the first '/'
let pos = trimmed_path.find(SLASH_SEPARATOR).unwrap_or(trimmed_path.len());
let pos = trimmed_path.find(SLASH_SEPARATOR_STR).unwrap_or(trimmed_path.len());
// Split into bucket and prefix
let bucket = &trimmed_path[0..pos];
let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists

8
crates/ecstore/src/set_disk.rs
View File

@@ -82,7 +82,7 @@ use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX,
use rustfs_utils::{
HashAlgorithm,
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
path::{SLASH_SEPARATOR_STR, encode_dir_object, has_suffix, path_join_buf},
};
use rustfs_workers::workers::Workers;
use s3s::header::X_AMZ_RESTORE;
@@ -5324,7 +5324,7 @@ impl StorageAPI for SetDisks {
&upload_id_path,
fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(),
]),
SLASH_SEPARATOR
SLASH_SEPARATOR_STR
);
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
@@ -5462,7 +5462,7 @@ impl StorageAPI for SetDisks {
let mut populated_upload_ids = HashSet::new();
for upload_id in upload_ids.iter() {
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR).to_string();
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR_STR).to_string();
if populated_upload_ids.contains(&upload_id) {
continue;
}
@@ -6222,7 +6222,7 @@ impl StorageAPI for SetDisks {
None
};
if has_suffix(object, SLASH_SEPARATOR) {
if has_suffix(object, SLASH_SEPARATOR_STR) {
let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?;
return Ok((result, err.map(|e| e.into())));
}

20
crates/ecstore/src/store_list_objects.rs
View File

@@ -34,7 +34,7 @@ use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use rustfs_utils::path::{self, SLASH_SEPARATOR_STR, base_dir_from_prefix};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::broadcast::{self};
@@ -132,7 +132,7 @@ impl ListPathOptions {
return;
}
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default();
let s = SLASH_SEPARATOR_STR.chars().next().unwrap_or_default();
self.filter_prefix = {
let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s);
@@ -346,7 +346,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() {
let mut found = false;
if delimiter != SLASH_SEPARATOR {
if delimiter != SLASH_SEPARATOR_STR {
for p in prefixes.iter() {
if found {
break;
@@ -470,7 +470,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() {
let mut found = false;
if delimiter != SLASH_SEPARATOR {
if delimiter != SLASH_SEPARATOR_STR {
for p in prefixes.iter() {
if found {
break;
@@ -502,7 +502,7 @@ impl ECStore {
// warn!("list_path opt {:?}", &o);
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
// if opts.prefix.ends_with(SLASH_SEPARATOR_STR) {
// return Err(Error::msg("eof"));
// }
@@ -520,11 +520,11 @@ impl ECStore {
return Err(Error::Unexpected);
}
if o.prefix.starts_with(SLASH_SEPARATOR) {
if o.prefix.starts_with(SLASH_SEPARATOR_STR) {
return Err(Error::Unexpected);
}
let slash_separator = Some(SLASH_SEPARATOR.to_owned());
let slash_separator = Some(SLASH_SEPARATOR_STR.to_owned());
o.include_directories = o.separator == slash_separator;
@@ -774,8 +774,8 @@ impl ECStore {
let mut filter_prefix = {
prefix
.trim_start_matches(&path)
.trim_start_matches(SLASH_SEPARATOR)
.trim_end_matches(SLASH_SEPARATOR)
.trim_start_matches(SLASH_SEPARATOR_STR)
.trim_end_matches(SLASH_SEPARATOR_STR)
.to_owned()
};
@@ -1130,7 +1130,7 @@ async fn merge_entry_channels(
if path::clean(&best_entry.name) == path::clean(&other_entry.name) {
let dir_matches = best_entry.is_dir() && other_entry.is_dir();
let suffix_matches =
best_entry.name.ends_with(SLASH_SEPARATOR) == other_entry.name.ends_with(SLASH_SEPARATOR);
best_entry.name.ends_with(SLASH_SEPARATOR_STR) == other_entry.name.ends_with(SLASH_SEPARATOR_STR);
if dir_matches && suffix_matches {
to_merge.push(other_idx);

6
crates/ecstore/src/tier/tier.rs
View File

@@ -51,7 +51,7 @@ use crate::{
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join};
use s3s::S3ErrorCode;
use super::{
@@ -403,7 +403,7 @@ impl TierConfigMgr {
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
@@ -483,7 +483,7 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data {
if is_err_config_not_found(&err) {

6
crates/ecstore/src/tier/warm_backend_s3.rs
View File

@@ -30,13 +30,11 @@ use crate::client::{
transition_api::{Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_utils::path::SLASH_SEPARATOR;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -178,7 +176,7 @@ impl WarmBackend for WarmBackendS3 {
async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self
.core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR_STR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)

10
crates/ecstore/src/tier/warm_backend_s3sdk.rs
View File

@@ -27,19 +27,11 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use crate::client::{
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::client::transition_api::{ReadCloser, ReaderImpl};
use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 {
pub client: Arc<Client>,

4
crates/iam/src/store/object.rs
View File

@@ -32,7 +32,7 @@ use rustfs_ecstore::{
store_api::{ObjectInfo, ObjectOptions},
};
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use serde::{Serialize, de::DeserializeOwned};
use std::sync::LazyLock;
use std::{collections::HashMap, sync::Arc};
@@ -182,7 +182,7 @@ impl ObjectStore {
} else {
info.name
};
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR);
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR_STR);
let _ = sender
.send(StringOrErr {
item: Some(name.to_owned()),

2
crates/utils/Cargo.toml
View File

@@ -83,7 +83,7 @@ ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver
io = ["dep:tokio"]
path = []
path = [] # path manipulation features
notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
string = ["dep:regex"]

756
crates/utils/src/path.rs
View File

@@ -15,12 +15,36 @@
use std::path::Path;
use std::path::PathBuf;
#[cfg(target_os = "windows")]
const SLASH_SEPARATOR: char = '\\';
#[cfg(not(target_os = "windows"))]
const SLASH_SEPARATOR: char = '/';
/// GLOBAL_DIR_SUFFIX is a special suffix used to denote directory objects
/// in object storage systems that do not have a native directory concept.
pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
pub const SLASH_SEPARATOR: &str = "/";
/// SLASH_SEPARATOR_STR is the string representation of the path separator
/// used in the current operating system.
pub const SLASH_SEPARATOR_STR: &str = if cfg!(target_os = "windows") { "\\" } else { "/" };
/// GLOBAL_DIR_SUFFIX_WITH_SLASH is the directory suffix followed by the
/// platform-specific path separator, used to denote directory objects.
#[cfg(target_os = "windows")]
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__\\";
#[cfg(not(target_os = "windows"))]
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/";
/// has_suffix checks if the string `s` ends with the specified `suffix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `suffix` - A string slice that holds the suffix to check for.
///
/// # Returns
/// A boolean indicating whether `s` ends with `suffix`.
///
pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") {
s.to_lowercase().ends_with(&suffix.to_lowercase())
@@ -29,19 +53,46 @@ pub fn has_suffix(s: &str, suffix: &str) -> bool {
}
}
/// encode_dir_object encodes a directory object by appending
/// a special suffix if it ends with a slash.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be encoded.
///
/// # Returns
/// A `String` representing the encoded directory object.
///
pub fn encode_dir_object(object: &str) -> String {
if has_suffix(object, SLASH_SEPARATOR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX)
if has_suffix(object, SLASH_SEPARATOR_STR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX)
} else {
object.to_string()
}
}
/// is_dir_object checks if the given object string represents
/// a directory object by verifying if it ends with the special suffix.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be checked.
///
/// # Returns
/// A boolean indicating whether the object is a directory object.
///
pub fn is_dir_object(object: &str) -> bool {
let obj = encode_dir_object(object);
obj.ends_with(GLOBAL_DIR_SUFFIX)
}
/// decode_dir_object decodes a directory object by removing
/// the special suffix if it is present.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be decoded.
///
/// # Returns
/// A `String` representing the decoded directory object.
///
#[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
@@ -51,21 +102,50 @@ pub fn decode_dir_object(object: &str) -> String {
}
}
/// retain_slash ensures that the given string `s` ends with a slash.
/// If it does not, a slash is appended.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be processed.
///
/// # Returns
/// A `String` that ends with a slash.
///
pub fn retain_slash(s: &str) -> String {
if s.is_empty() {
return s.to_string();
}
if s.ends_with(SLASH_SEPARATOR) {
if s.ends_with(SLASH_SEPARATOR_STR) {
s.to_string()
} else {
format!("{s}{SLASH_SEPARATOR}")
format!("{s}{SLASH_SEPARATOR_STR}")
}
}
/// strings_has_prefix_fold checks if the string `s` starts with the specified `prefix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `prefix` - A string slice that holds the prefix to check for.
///
/// # Returns
/// A boolean indicating whether `s` starts with `prefix`.
///
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
}
/// has_prefix checks if the string `s` starts with the specified `prefix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `prefix` - A string slice that holds the prefix to check for.
///
/// # Returns
/// A boolean indicating whether `s` starts with `prefix`.
///
pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix);
@@ -74,21 +154,37 @@ pub fn has_prefix(s: &str, prefix: &str) -> bool {
s.starts_with(prefix)
}
/// path_join joins multiple path elements into a single PathBuf,
/// ensuring that the resulting path is clean and properly formatted.
///
/// # Arguments
/// * `elem` - A slice of path elements to be joined.
///
/// # Returns
/// A PathBuf representing the joined path.
///
pub fn path_join<P: AsRef<Path>>(elem: &[P]) -> PathBuf {
path_join_buf(
elem.iter()
.map(|p| p.as_ref().to_string_lossy().into_owned())
.collect::<Vec<String>>()
.iter()
.map(|s| s.as_str())
.collect::<Vec<&str>>()
.as_slice(),
)
.into()
if elem.is_empty() {
return PathBuf::from(".");
}
// Collect components as owned Strings (lossy for non-UTF8)
let strs: Vec<String> = elem.iter().map(|p| p.as_ref().to_string_lossy().into_owned()).collect();
// Convert to slice of &str for path_join_buf
let refs: Vec<&str> = strs.iter().map(|s| s.as_str()).collect();
PathBuf::from(path_join_buf(&refs))
}
/// path_join_buf joins multiple string path elements into a single String,
/// ensuring that the resulting path is clean and properly formatted.
///
/// # Arguments
/// * `elements` - A slice of string path elements to be joined.
///
/// # Returns
/// A String representing the joined path.
///
pub fn path_join_buf(elements: &[&str]) -> String {
let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR));
let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR_STR));
let mut dst = String::new();
let mut added = 0;
@@ -96,7 +192,7 @@ pub fn path_join_buf(elements: &[&str]) -> String {
for e in elements {
if added > 0 || !e.is_empty() {
if added > 0 {
dst.push_str(SLASH_SEPARATOR);
dst.push(SLASH_SEPARATOR);
}
dst.push_str(e);
added += e.len();
@@ -106,168 +202,420 @@ pub fn path_join_buf(elements: &[&str]) -> String {
if path_needs_clean(dst.as_bytes()) {
let mut clean_path = clean(&dst);
if trailing_slash {
clean_path.push_str(SLASH_SEPARATOR);
clean_path.push(SLASH_SEPARATOR);
}
return clean_path;
}
if trailing_slash {
dst.push_str(SLASH_SEPARATOR);
dst.push(SLASH_SEPARATOR);
}
dst
}
/// Platform-aware separator check
fn is_sep(b: u8) -> bool {
#[cfg(target_os = "windows")]
{
b == b'/' || b == b'\\'
}
#[cfg(not(target_os = "windows"))]
{
b == b'/'
}
}
/// path_needs_clean returns whether path cleaning may change the path.
/// Will detect all cases that will be cleaned,
/// but may produce false positives on non-trivial paths.
///
/// # Arguments
/// * `path` - A byte slice that holds the path to be checked.
///
/// # Returns
/// A boolean indicating whether the path needs cleaning.
///
fn path_needs_clean(path: &[u8]) -> bool {
if path.is_empty() {
return true;
}
let rooted = path[0] == b'/';
let n = path.len();
let (mut r, mut w) = if rooted { (1, 1) } else { (0, 0) };
// On Windows: any forward slash indicates normalization to backslash is required.
#[cfg(target_os = "windows")]
{
if path.iter().any(|&b| b == b'/') {
return true;
}
}
while r < n {
match path[r] {
b if b > 127 => {
// Non ascii.
// Initialize scan index and previous-separator flag.
let mut i = 0usize;
let mut prev_was_sep = false;
// Platform-aware prefix handling to avoid flagging meaningful leading sequences:
// - Windows: handle drive letter "C:" and UNC leading "\\"
// - Non-Windows: detect and flag double leading '/' (e.g. "//abc") as needing clean
if n >= 1 && is_sep(path[0]) {
#[cfg(target_os = "windows")]
{
// If starts with two separators -> UNC prefix: allow exactly two without flag
if n >= 2 && is_sep(path[1]) {
// If a third leading separator exists, that's redundant (e.g. "///...") -> needs clean
if n >= 3 && is_sep(path[2]) {
return true;
}
b'/' => {
// multiple / elements
// Skip the two UNC leading separators for scanning; do not mark prev_was_sep true
i = 2;
prev_was_sep = false;
} else {
// Single leading separator (rooted) -> mark as seen separator so immediate next sep is duplicate
i = 1;
prev_was_sep = true;
}
}
#[cfg(not(target_os = "windows"))]
{
// POSIX: double leading '/' is redundant and should be cleaned (e.g. "//abc" -> "/abc")
if n >= 2 && is_sep(path[1]) {
return true;
}
b'.' => {
if r + 1 == n || path[r + 1] == b'/' {
// . element - assume it has to be cleaned.
i = 1;
prev_was_sep = true;
}
} else {
// If not starting with separator, check for Windows drive-letter prefix like "C:"
#[cfg(target_os = "windows")]
{
if n >= 2 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() {
// Position after "C:"
i = 2;
// If a separator immediately follows the drive (rooted like "C:\"),
// treat that first separator as seen; if more separators follow, it's redundant.
if i < n && is_sep(path[i]) {
i += 1; // consume the single allowed separator after drive
if i < n && is_sep(path[i]) {
// multiple separators after drive like "C:\\..." -> needs clean
return true;
}
if r + 1 < n && path[r + 1] == b'.' && (r + 2 == n || path[r + 2] == b'/') {
// .. element: remove to last / - assume it has to be cleaned.
return true;
}
// Handle single dot case
if r + 1 == n {
// . element - assume it has to be cleaned.
return true;
}
// Copy the dot
w += 1;
r += 1;
}
_ => {
// real path element.
// add slash if needed
if (rooted && w != 1) || (!rooted && w != 0) {
w += 1;
}
// copy element
while r < n && path[r] != b'/' {
w += 1;
r += 1;
}
// allow one slash, not at end
if r < n - 1 && path[r] == b'/' {
r += 1;
prev_was_sep = true;
} else {
prev_was_sep = false;
}
}
}
}
// Turn empty string into "."
if w == 0 {
// Generic scan for repeated separators and dot / dot-dot components.
while i < n {
let b = path[i];
if is_sep(b) {
if prev_was_sep {
// Multiple separators (except allowed UNC prefix handled above)
return true;
}
prev_was_sep = true;
i += 1;
continue;
}
// Not a separator: parse current path element
let start = i;
while i < n && !is_sep(path[i]) {
i += 1;
}
let len = i - start;
if len == 1 && path[start] == b'.' {
// single "." element -> needs cleaning
return true;
}
if len == 2 && path[start] == b'.' && path[start + 1] == b'.' {
// ".." element -> needs cleaning
return true;
}
prev_was_sep = false;
}
// Trailing separator: if last byte is a separator and path length > 1, then usually needs cleaning,
// except when the path is a platform-specific root form (e.g. "/" on POSIX, "\\" or "C:\" on Windows).
if n > 1 && is_sep(path[n - 1]) {
#[cfg(not(target_os = "windows"))]
{
// POSIX: any trailing separator except the single-root "/" needs cleaning.
return true;
}
#[cfg(target_os = "windows")]
{
// Windows special root forms that are acceptable with trailing separator:
// - UNC root: exactly two leading separators "\" "\" (i.e. "\\") -> n == 2
if n == 2 && is_sep(path[0]) && is_sep(path[1]) {
return false;
}
// - Drive root: pattern "C:\" or "C:/" (len == 3)
if n == 3 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() && is_sep(path[2]) {
return false;
}
// Otherwise, trailing separator should be cleaned.
return true;
}
}
// No conditions triggered: assume path is already clean.
false
}
pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR);
/// path_to_bucket_object_with_base_path splits a given path into bucket and object components,
/// considering a base path to trim from the start.
///
/// # Arguments
/// * `base_path` - A string slice that holds the base path to be trimmed.
/// * `path` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the bucket and object as `String`s.
///
pub fn path_to_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(base_path).trim_start_matches(SLASH_SEPARATOR);
if let Some(m) = path.find(SLASH_SEPARATOR) {
return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string());
return (path[..m].to_string(), path[m + SLASH_SEPARATOR_STR.len()..].to_string());
}
(path.to_string(), "".to_string())
}
/// path_to_bucket_object splits a given path into bucket and object components.
///
/// # Arguments
/// * `s` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the bucket and object as `String`s.
///
pub fn path_to_bucket_object(s: &str) -> (String, String) {
path_to_bucket_object_with_base_path("", s)
}
/// contains_any_sep_str checks if the given string contains any path separators.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
///
/// # Returns
/// A boolean indicating whether the string contains any path separators.
fn contains_any_sep_str(s: &str) -> bool {
#[cfg(target_os = "windows")]
{
s.contains('/') || s.contains('\\')
}
#[cfg(not(target_os = "windows"))]
{
s.contains('/')
}
}
/// base_dir_from_prefix extracts the base directory from a given prefix.
///
/// # Arguments
/// * `prefix` - A string slice that holds the prefix to be processed.
///
/// # Returns
/// A `String` representing the base directory extracted from the prefix.
///
pub fn base_dir_from_prefix(prefix: &str) -> String {
let mut base_dir = dir(prefix).to_owned();
if base_dir == "." || base_dir == "./" || base_dir == "/" {
base_dir = "".to_owned();
if !contains_any_sep_str(prefix) {
return String::new();
}
if !prefix.contains('/') {
base_dir = "".to_owned();
let mut base_dir = dir(prefix);
if base_dir == "." || base_dir == SLASH_SEPARATOR_STR {
base_dir.clear();
}
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) {
base_dir.push_str(SLASH_SEPARATOR);
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR_STR) {
base_dir.push_str(SLASH_SEPARATOR_STR);
}
base_dir
}
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
/// clean returns the shortest path name equivalent to path
/// by purely lexical processing. It applies the following rules
/// iteratively until no further processing can be done:
///
/// 1. Replace multiple slashes with a single slash.
/// 2. Eliminate each . path name element (the current directory).
/// 3. Eliminate each inner .. path name element (the parent directory)
/// along with the non-.. element that precedes it.
/// 4. Eliminate .. elements that begin a rooted path,
/// that is, replace "/.." by "/" at the beginning of a path.
///
/// If the result of this process is an empty string, clean returns the string ".".
///
/// This function is adapted to work cross-platform by using the appropriate path separator.
/// On Windows, this function is aware of drive letters (e.g., `C:`) and UNC paths
/// (e.g., `\\server\share`) and cleans them using the appropriate separator.
///
/// # Arguments
/// * `path` - A string slice that holds the path to be cleaned.
///
/// # Returns
/// A `String` representing the cleaned path.
///
pub fn clean(path: &str) -> String {
if path.is_empty() {
return ".".to_string();
}
#[cfg(target_os = "windows")]
{
use std::borrow::Cow;
let bytes = path.as_bytes();
let n = bytes.len();
// Windows-aware handling
let mut i = 0usize;
let mut drive: Option<char> = None;
let mut rooted = false;
let mut preserve_leading_double_sep = false;
// Drive letter detection
if n >= 2 && bytes[1] == b':' && (bytes[0] as char).is_ascii_alphabetic() {
drive = Some(bytes[0] as char);
i = 2;
// If next is separator, it's an absolute drive-root (e.g., "C:\")
if i < n && is_sep(bytes[i]) {
rooted = true;
// consume all leading separators after drive
while i < n && is_sep(bytes[i]) {
i += 1;
}
}
} else {
// UNC or absolute by separators
if n >= 2 && is_sep(bytes[0]) && is_sep(bytes[1]) {
rooted = true;
preserve_leading_double_sep = true;
i = 2;
// consume extra leading separators
while i < n && is_sep(bytes[i]) {
i += 1;
}
} else if is_sep(bytes[0]) {
rooted = true;
i = 1;
while i < n && is_sep(bytes[i]) {
i += 1;
}
}
}
// Component stack
let mut comps: Vec<Cow<'_, str>> = Vec::new();
let mut r = i;
while r < n {
// find next sep or end
let start = r;
while r < n && !is_sep(bytes[r]) {
r += 1;
}
// component bytes [start..r)
let comp = String::from_utf8_lossy(&bytes[start..r]);
if comp == "." {
// skip
} else if comp == ".." {
if !comps.is_empty() {
// pop last component
comps.pop();
} else if !rooted {
// relative path with .. at front must be kept
comps.push(Cow::Owned("..".to_string()));
} else {
// rooted and at root => ignore
}
} else {
comps.push(comp);
}
// skip separators
while r < n && is_sep(bytes[r]) {
r += 1;
}
}
// Build result
let mut out = String::new();
if let Some(d) = drive {
out.push(d);
out.push(':');
if rooted {
out.push(SLASH_SEPARATOR);
}
} else if preserve_leading_double_sep {
out.push(SLASH_SEPARATOR);
out.push(SLASH_SEPARATOR);
} else if rooted {
out.push(SLASH_SEPARATOR);
}
// Join components
for (idx, c) in comps.iter().enumerate() {
if !out.is_empty() && !out.ends_with(SLASH_SEPARATOR_STR) {
out.push(SLASH_SEPARATOR);
}
out.push_str(c);
}
// Special cases:
if out.is_empty() {
// No drive, no components -> "."
return ".".to_string();
}
// If output is just "C:" (drive without components and not rooted), keep as "C:"
if drive.is_some() {
if out.len() == 2 && out.as_bytes()[1] == b':' {
return out;
}
// If drive+colon+sep and no components, return "C:\"
if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) {
return out;
}
}
// Remove trailing separator unless it's a root form (single leading sep or drive root or UNC)
if out.len() > 1 && out.ends_with(SLASH_SEPARATOR_STR) {
// Determine if it's a root form: "/" or "\\" or "C:\"
let is_root = {
// "/" (non-drive single sep)
if out == SLASH_SEPARATOR_STR {
true
} else if out.starts_with(SLASH_SEPARATOR_STR) && out == format!("{}{}", SLASH_SEPARATOR_STR, SLASH_SEPARATOR_STR)
{
// only double separator
true
} else {
// drive root "C:\" length >=3 with pattern X:\
if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) {
true
} else {
false
}
}
};
if !is_root {
out.pop();
}
}
out
}
#[cfg(not(target_os = "windows"))]
{
// POSIX-like behavior (original implementation but simplified)
let rooted = path.starts_with('/');
let n = path.len();
let mut out = LazyBuf::new(path.to_string());
let mut r = 0;
let mut dotdot = 0;
let mut r = 0usize;
let mut dotdot = 0usize;
if rooted {
out.append(b'/');
@@ -328,10 +676,22 @@ pub fn clean(path: &str) -> String {
out.string()
}
}
/// split splits path immediately after the final slash,
/// separating it into a directory and file name component.
/// If there is no slash in path, split returns
/// ("", path).
///
/// # Arguments
/// * `path` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the directory and file name as string slices.
///
pub fn split(path: &str) -> (&str, &str) {
// Find the last occurrence of the '/' character
if let Some(i) = path.rfind('/') {
if let Some(i) = path.rfind(SLASH_SEPARATOR_STR) {
// Return the directory (up to and including the last '/') and the file name
return (&path[..i + 1], &path[i + 1..]);
}
@@ -339,19 +699,168 @@ pub fn split(path: &str) -> (&str, &str) {
(path, "")
}
/// dir returns all but the last element of path,
/// typically the path's directory. After dropping the final
/// element, the path is cleaned. If the path is empty,
/// dir returns ".".
///
/// # Arguments
/// * `path` - A string slice that holds the path to be processed.
///
/// # Returns
/// A `String` representing the directory part of the path.
///
pub fn dir(path: &str) -> String {
let (a, _) = split(path);
clean(a)
}
/// trim_etag removes surrounding double quotes from an ETag string.
///
/// # Arguments
/// * `etag` - A string slice that holds the ETag to be trimmed.
///
/// # Returns
/// A `String` representing the trimmed ETag.
///
pub fn trim_etag(etag: &str) -> String {
etag.trim_matches('"').to_string()
}
/// LazyBuf is a structure that efficiently builds a byte buffer
/// from a string by delaying the allocation of the buffer until
/// a modification is necessary. It allows appending bytes and
/// retrieving the current string representation.
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
/// Creates a new LazyBuf with the given string.
///
/// # Arguments
/// * `s` - A string to initialize the LazyBuf.
///
/// # Returns
/// A new instance of LazyBuf.
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_path_join_buf() {
#[cfg(not(target_os = "windows"))]
{
// Basic joining
assert_eq!(path_join_buf(&["a", "b"]), "a/b");
assert_eq!(path_join_buf(&["a/", "b"]), "a/b");
// Empty array input
assert_eq!(path_join_buf(&[]), ".");
// Single element
assert_eq!(path_join_buf(&["a"]), "a");
// Multiple elements
assert_eq!(path_join_buf(&["a", "b", "c"]), "a/b/c");
// Elements with trailing separators
assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/");
// Elements requiring cleaning (with "." and "..")
assert_eq!(path_join_buf(&["a", ".", "b"]), "a/b");
assert_eq!(path_join_buf(&["a", "..", "b"]), "b");
assert_eq!(path_join_buf(&["a", "b", ".."]), "a");
// Preservation of trailing slashes
assert_eq!(path_join_buf(&["a", "b/"]), "a/b/");
assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/");
// Empty elements
assert_eq!(path_join_buf(&["a", "", "b"]), "a/b");
// Double slashes (cleaning)
assert_eq!(path_join_buf(&["a//", "b"]), "a/b");
}
#[cfg(target_os = "windows")]
{
// Basic joining
assert_eq!(path_join_buf(&["a", "b"]), "a\\b");
assert_eq!(path_join_buf(&["a\\", "b"]), "a\\b");
// Empty array input
assert_eq!(path_join_buf(&[]), ".");
// Single element
assert_eq!(path_join_buf(&["a"]), "a");
// Multiple elements
assert_eq!(path_join_buf(&["a", "b", "c"]), "a\\b\\c");
// Elements with trailing separators
assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\");
// Elements requiring cleaning (with "." and "..")
assert_eq!(path_join_buf(&["a", ".", "b"]), "a\\b");
assert_eq!(path_join_buf(&["a", "..", "b"]), "b");
assert_eq!(path_join_buf(&["a", "b", ".."]), "a");
// Mixed separator handling
assert_eq!(path_join_buf(&["a/b", "c"]), "a\\b\\c");
assert_eq!(path_join_buf(&["a\\", "b/c"]), "a\\b\\c");
// Preservation of trailing slashes
assert_eq!(path_join_buf(&["a", "b\\"]), "a\\b\\");
assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\");
// Empty elements
assert_eq!(path_join_buf(&["a", "", "b"]), "a\\b");
// Double slashes (cleaning)
assert_eq!(path_join_buf(&["a\\\\", "b"]), "a\\b");
}
}
#[test]
fn test_trim_etag() {
// Test with quoted ETag
@@ -383,6 +892,8 @@ mod tests {
#[test]
fn test_clean() {
#[cfg(not(target_os = "windows"))]
{
assert_eq!(clean(""), ".");
assert_eq!(clean("abc"), "abc");
assert_eq!(clean("abc/def"), "abc/def");
@@ -422,6 +933,17 @@ mod tests {
assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno");
}
#[cfg(target_os = "windows")]
{
assert_eq!(clean("a\\b\\..\\c"), "a\\c");
assert_eq!(clean("a\\\\b"), "a\\b");
assert_eq!(clean("C:\\"), "C:\\");
assert_eq!(clean("C:\\a\\..\\b"), "C:\\b");
assert_eq!(clean("C:a\\b\\..\\c"), "C:a\\c");
assert_eq!(clean("\\\\server\\share\\a\\\\b"), "\\\\server\\share\\a\\b");
}
}
#[test]
fn test_path_needs_clean() {
struct PathTest {

15
rustfs/src/admin/handlers/bucket_meta.rs
View File

@@ -12,11 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
collections::HashMap,
io::{Cursor, Read as _, Write as _},
};
use crate::{
admin::{auth::validate_admin_request, router::Operation},
auth::{check_key_valid, get_session_token},
@@ -49,7 +44,7 @@ use rustfs_policy::policy::{
BucketPolicy,
action::{Action, AdminAction},
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use s3s::{
Body, S3Request, S3Response, S3Result,
dto::{
@@ -61,6 +56,10 @@ use s3s::{
};
use serde::Deserialize;
use serde_urlencoded::from_bytes;
use std::{
collections::HashMap,
io::{Cursor, Read as _, Write as _},
};
use time::OffsetDateTime;
use tracing::warn;
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
@@ -424,7 +423,7 @@ impl Operation for ImportBucketMetadata {
// Extract bucket names
let mut bucket_names = Vec::new();
for (file_path, _) in &file_contents {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path);
@@ -463,7 +462,7 @@ impl Operation for ImportBucketMetadata {
// Second pass: process file contents
for (file_path, content) in file_contents {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path);