diff --git a/Cargo.lock b/Cargo.lock index 821a203d..70720aad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,7 +430,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "lexical-core", "memchr", @@ -532,7 +532,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", "synstructure 0.13.2", ] @@ -544,7 +544,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -612,7 +612,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -623,7 +623,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -649,7 +649,7 @@ checksum = "99e1aca718ea7b89985790c94aad72d77533063fe00bc497bb79a7c2dae6a661" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -1280,15 +1280,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.4.2", + "cpufeatures", "memmap2", "rayon-core", ] @@ -1333,9 +1334,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.8.1" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c" dependencies = [ "bon-macros", "rustversion", @@ -1343,17 +1344,17 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.8.1" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -1528,9 +1529,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.51" +version = "1.2.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3" dependencies = [ "find-msvc-tools", "jobserver", @@ -1690,7 +1691,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -1747,9 +1748,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-oid" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dabb6555f92fb9ee4140454eb5dcd14c7960e1225c6d1a6cc361f032947713e" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "const-random" @@ -1788,7 +1789,7 @@ checksum = "1d3e0f24ee268386bd3ab4e04fc60df9a818ad801b5ffe592f388a6acc5053fb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -1817,6 +1818,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "convert_case" version = "0.10.0" @@ -2037,9 +2044,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.0-rc.14" +version = "0.7.0-rc.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c6daa2049db6a5fad90a981b8c63f023dbaf75a0fae73db4dcf234556fc957" +checksum = "1a9e36ac79ac44866b74e08a0b4925f97b984e3fff17680d2c6fbce8317ab0f6" dependencies = [ "ctutils", "num-traits", @@ -2075,7 +2082,7 @@ version = "0.7.0-pre.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0b07a7a616370e8b6efca0c6a25e5f4c6d02fde11f3d570e4af64d8ed7e2e9" dependencies = [ - "crypto-bigint 0.7.0-rc.14", + "crypto-bigint 0.7.0-rc.15", "libm", "rand_core 0.10.0-rc-3", ] @@ -2167,7 +2174,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2235,7 +2242,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2249,7 +2256,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2262,7 +2269,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2284,7 +2291,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2295,7 +2302,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2306,7 +2313,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2325,9 +2332,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" @@ -2446,7 +2453,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "libc", "log", "object_store", @@ -2644,7 +2651,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", "recursive", @@ -2660,7 +2667,7 @@ checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", ] @@ -2804,7 +2811,7 @@ checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -2819,7 +2826,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "recursive", @@ -2842,11 +2849,11 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot", "paste", - "petgraph 0.8.3", + "petgraph", ] [[package]] @@ -2920,7 +2927,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "parking_lot", @@ -2970,7 +2977,7 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "recursive", "regex", @@ -3000,7 +3007,7 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3030,7 +3037,7 @@ version = "0.8.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02c1d73e9668ea6b6a28172aa55f3ebec38507131ce179051c8033b5c6037653" dependencies = [ - "const-oid 0.10.1", + "const-oid 0.10.2", "pem-rfc7468 1.0.0", "zeroize", ] @@ -3067,7 +3074,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3109,7 +3116,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3129,7 +3136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3151,7 +3158,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.113", + "syn 2.0.114", "unicode-xid", ] @@ -3189,7 +3196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebf9423bafb058e4142194330c52273c343f8a5beb7176d052f0e73b17dd35b9" dependencies = [ "block-buffer 0.11.0", - "const-oid 0.10.1", + "const-oid 0.10.2", "crypto-common 0.2.0-rc.9", "subtle", ] @@ -3223,7 +3230,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3428,7 +3435,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3449,7 +3456,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3488,7 +3495,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3610,9 +3617,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41" [[package]] name = "findshlibs" @@ -3820,7 +3827,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -3930,7 +3937,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -4026,7 +4033,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "percent-encoding", "pin-project", - "prost 0.14.1", + "prost 0.14.3", "prost-types", "reqwest", "rustc_version", @@ -4136,7 +4143,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project", - "prost 0.14.1", + "prost 0.14.3", "prost-types", "reqwest", "serde", @@ -4204,9 +4211,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", "bytes", @@ -4214,7 +4221,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -4304,7 +4311,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -4753,9 +4760,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -4770,7 +4777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.12.1", + "indexmap 2.13.0", "is-terminal", "itoa", "log", @@ -4793,7 +4800,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "log", "num-format", @@ -5003,7 +5010,7 @@ dependencies = [ "p384", "pem", "rand 0.8.5", - "rsa 0.9.9", + "rsa 0.9.10", "serde", "serde_json", "sha2 0.10.9", @@ -5051,7 +5058,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -5128,9 +5135,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.179" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libcrux-intrinsics" @@ -5402,9 +5409,9 @@ dependencies = [ [[package]] name = "lzma-rust2" -version = "0.15.4" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48172246aa7c3ea28e423295dd1ca2589a24617cc4e588bb8cfe177cb2c54d95" +checksum = "7fa48f5024824ecd3e8282cc948bd46fbd095aed5a98939de0594601a59b4e2b" dependencies = [ "crc", "sha2 0.10.9", @@ -5626,7 +5633,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -6057,7 +6064,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.14.1", + "prost 0.14.3", "reqwest", "thiserror 2.0.17", "tracing", @@ -6071,7 +6078,7 @@ checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.14.1", + "prost 0.14.3", "tonic", "tonic-prost", ] @@ -6392,16 +6399,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.12.1", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -6410,7 +6407,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", ] @@ -6464,7 +6461,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -6502,7 +6499,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -6762,7 +6759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -6802,14 +6799,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] name = "proc-macro2" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" dependencies = [ "unicode-ident", ] @@ -6840,33 +6837,32 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive 0.14.1", + "prost-derive 0.14.3", ] [[package]] name = "prost-build" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", "itertools 0.14.0", "log", "multimap", - "once_cell", - "petgraph 0.7.1", + "petgraph", "prettyplease", - "prost 0.14.1", + "prost 0.14.3", "prost-types", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.113", + "syn 2.0.114", "tempfile", ] @@ -6880,29 +6876,29 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] name = "prost-types" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "prost 0.14.1", + "prost 0.14.3", ] [[package]] @@ -6938,7 +6934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "protobuf", "protobuf-support", @@ -6989,9 +6985,9 @@ dependencies = [ [[package]] name = "pulldown-cmark-to-cmark" -version = "21.1.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8246feae3db61428fd0bb94285c690b460e4517d83152377543ca802357785f1" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ "pulldown-cmark", ] @@ -7083,9 +7079,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] @@ -7230,7 +7226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -7291,7 +7287,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -7458,7 +7454,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -7482,9 +7478,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid 0.9.6", "digest 0.10.7", @@ -7506,8 +7502,8 @@ version = "0.10.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d27d813937fdf8e9ad15e3e422a55da4021d29639000139ca19d99f3949060da" dependencies = [ - "const-oid 0.10.1", - "crypto-bigint 0.7.0-rc.14", + "const-oid 0.10.2", + "crypto-bigint 0.7.0-rc.15", "crypto-primes", "digest 0.11.0-rc.5", "pkcs1 0.8.0-rc.4", @@ -7544,7 +7540,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.113", + "syn 2.0.114", "unicode-ident", ] @@ -7693,7 +7689,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.113", + "syn 2.0.114", "walkdir", ] @@ -7974,6 +7970,7 @@ dependencies = [ "bytesize", "chrono", "criterion", + "dunce", "enumset", "faster-hex", "flatbuffers", @@ -8252,7 +8249,7 @@ name = "rustfs-protos" version = "0.0.5" dependencies = [ "flatbuffers", - "prost 0.14.1", + "prost 0.14.3", "rustfs-common", "tonic", "tonic-prost", @@ -8703,7 +8700,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -8839,7 +8836,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -8850,7 +8847,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -8917,7 +8914,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.13.0", "schemars 0.9.0", "schemars 1.2.0", "serde_core", @@ -8935,7 +8932,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -8971,7 +8968,7 @@ checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9226,7 +9223,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9312,7 +9309,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9367,7 +9364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ad6a09263583e83e934fcd436b7e3bb9d69602e2feef3787adb615c1fe3a343" dependencies = [ "base64ct", - "crypto-bigint 0.7.0-rc.14", + "crypto-bigint 0.7.0-rc.15", "digest 0.11.0-rc.5", "pem-rfc7468 1.0.0", "subtle", @@ -9476,7 +9473,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9487,9 +9484,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "7.0.7" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8928c89e226be233f0eb1594e9bd023f72a948dc06581c0d908387f57de1de" +checksum = "69a15b325bbe0a1f85de3dbf988a3a14e9cd321537dffcbf6641381dd6d7586f" dependencies = [ "async-trait", "chrono", @@ -9618,9 +9615,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.113" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678faa00651c9eb72dd2020cbdf275d92eccb2400d568e419efdd64838145cb4" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -9665,7 +9662,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9750,7 +9747,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9761,7 +9758,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", "test-case-core", ] @@ -9791,7 +9788,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9802,7 +9799,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9951,7 +9948,7 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -9979,7 +9976,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -10064,7 +10061,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -10078,7 +10075,7 @@ version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "winnow", @@ -10140,7 +10137,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -10150,7 +10147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost 0.14.1", + "prost 0.14.3", "tonic", ] @@ -10165,7 +10162,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.113", + "syn 2.0.114", "tempfile", "tonic-build", ] @@ -10178,7 +10175,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.12.1", + "indexmap 2.13.0", "pin-project-lite", "slab", "sync_wrapper", @@ -10258,7 +10255,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -10404,9 +10401,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-ident" @@ -10516,7 +10513,7 @@ checksum = "39d11901c36b3650df7acb0f9ebe624f35b5ac4e1922ecd3c57f444648429594" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -10672,7 +10669,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", "wasm-bindgen-shared", ] @@ -10878,7 +10875,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -10889,7 +10886,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -11178,7 +11175,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -11287,28 +11284,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -11328,7 +11325,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", "synstructure 0.13.2", ] @@ -11349,7 +11346,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -11382,7 +11379,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.114", ] [[package]] @@ -11394,14 +11391,14 @@ dependencies = [ "aes 0.8.4", "arbitrary", "bzip2 0.6.1", - "constant_time_eq", + "constant_time_eq 0.3.1", "crc32fast", "deflate64", "flate2", "generic-array 0.14.7", "getrandom 0.3.4", "hmac 0.12.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "lzma-rust2", "memchr", "pbkdf2 0.12.2", @@ -11421,9 +11418,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] name = "zmij" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e0d8dffbae3d840f64bda38e28391faef673a7b5a6017840f2a106c8145868" +checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index b09e7730..cc839ea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,7 +130,7 @@ bytesize = "2.3.1" byteorder = "1.5.0" flatbuffers = "25.12.19" form_urlencoded = "1.2.2" -prost = "0.14.1" +prost = "0.14.3" quick-xml = "0.38.4" rmcp = { version = "0.12.0" } rmp = { version = "0.8.15" } @@ -143,7 +143,7 @@ schemars = "1.2.0" # Cryptography and Security aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] } argon2 = { version = "0.6.0-rc.5" } -blake3 = { version = "1.8.2", features = ["rayon", "mmap"] } +blake3 = { version = "1.8.3", features = ["rayon", "mmap"] } chacha20poly1305 = { version = "0.11.0-rc.2" } crc-fast = "1.6.0" hmac = { version = "0.13.0-rc.3" } @@ -184,6 +184,7 @@ criterion = { version = "0.8", features = ["html_reports"] } crossbeam-queue = "0.3.12" datafusion = "51.0.0" derive_builder = "0.20.2" +dunce = "1.0.5" enumset = "1.1.10" faster-hex = "0.10.0" flate2 = "1.1.5" @@ -197,7 +198,7 @@ hex-simd = "0.8.0" highway = { version = "1.3.0" } ipnetwork = { version = "0.21.1", features = ["serde"] } lazy_static = "1.5.0" -libc = "0.2.179" +libc = "0.2.180" libsystemd = "0.7.2" local-ip-address = "0.6.8" lz4 = "1.28.1" @@ -270,7 +271,7 @@ libunftp = "0.21.0" russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false } russh-sftp = "2.1.1" ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] } -suppaftp = { version = "7.0.7", features = ["tokio", "tokio-rustls", "rustls"] } +suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] } rcgen = "0.14.6" # Performance Analysis and Memory Profiling diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index d4fe7d4c..5d6e2ff1 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -48,6 +48,7 @@ async-trait.workspace = true bytes.workspace = true byteorder = { workspace = true } chrono.workspace = true +dunce.workspace = true glob = { workspace = true } thiserror.workspace = true flatbuffers.workspace = true @@ -109,7 +110,6 @@ google-cloud-auth = { workspace = true } aws-config = { workspace = true } faster-hex = { workspace = true } - [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } criterion = { workspace = true, features = ["html_reports"] } diff --git a/crates/ecstore/src/bucket/utils.rs b/crates/ecstore/src/bucket/utils.rs index a012798b..8eb60ccb 100644 --- a/crates/ecstore/src/bucket/utils.rs +++ b/crates/ecstore/src/bucket/utils.rs @@ -14,7 +14,7 @@ use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result, StorageError}; -use rustfs_utils::path::SLASH_SEPARATOR; +use rustfs_utils::path::SLASH_SEPARATOR_STR; use s3s::xml; pub fn is_meta_bucketname(name: &str) -> bool { @@ -194,7 +194,7 @@ pub fn is_valid_object_name(object: &str) -> bool { return false; } - if object.ends_with(SLASH_SEPARATOR) { + if object.ends_with(SLASH_SEPARATOR_STR) { return false; } @@ -206,7 +206,7 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned())); } - if object.starts_with(SLASH_SEPARATOR) { + if object.starts_with(SLASH_SEPARATOR_STR) { return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned())); } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index b1010bf0..3ad8256a 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -18,7 +18,7 @@ use crate::error::{Error, Result}; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use http::HeaderMap; use rustfs_config::DEFAULT_DELIMITER; -use rustfs_utils::path::SLASH_SEPARATOR; +use rustfs_utils::path::SLASH_SEPARATOR_STR; use std::collections::HashSet; use std::sync::Arc; use std::sync::LazyLock; @@ -29,7 +29,7 @@ const CONFIG_FILE: &str = "config.json"; pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class"; -static CONFIG_BUCKET: LazyLock = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}")); +static CONFIG_BUCKET: LazyLock = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{CONFIG_PREFIX}")); static SUB_SYSTEMS_DYNAMIC: LazyLock> = LazyLock::new(|| { let mut h = HashSet::new(); @@ -129,7 +129,7 @@ async fn new_and_save_server_config(api: Arc) -> Result String { - format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}") + format!("{CONFIG_PREFIX}{SLASH_SEPARATOR_STR}{CONFIG_FILE}") } /// Handle the situation where the configuration file does not exist, create and save a new configuration diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index cc908e6a..df3ffede 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -31,14 +31,14 @@ use crate::{ use rustfs_common::data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary, }; -use rustfs_utils::path::SLASH_SEPARATOR; +use rustfs_utils::path::SLASH_SEPARATOR_STR; use tokio::fs; use tracing::{error, info, warn}; use crate::error::Error; // Data usage storage constants -pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; +pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR; const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; @@ -47,17 +47,17 @@ pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; lazy_static::lazy_static! { pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", crate::disk::RUSTFS_META_BUCKET, - SLASH_SEPARATOR, + SLASH_SEPARATOR_STR, crate::disk::BUCKET_META_PREFIX ); pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, + SLASH_SEPARATOR_STR, DATA_USAGE_OBJ_NAME ); pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, + SLASH_SEPARATOR_STR, DATA_USAGE_BLOOM_NAME ); } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 09991fc5..7632a34d 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -12,39 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::error::{Error, Result}; -use super::os::{is_root_disk, rename_all}; -use super::{ - BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, - FileInfoVersions, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os, -}; -use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; - use crate::config::storageclass::DEFAULT_INLINE_BLOCK; use crate::data_usage::local_snapshot::ensure_data_usage_layout; -use crate::disk::error::FileAccessDeniedWithContext; -use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}; -use crate::disk::fs::{ - O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename, -}; -use crate::disk::os::{check_path_length, is_empty_dir}; use crate::disk::{ - CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, - FileReader, RUSTFS_META_TMP_DELETED_BUCKET, conv_part_err_to_int, + BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, + CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, + FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, + ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, + VolumeInfo, WalkDirOptions, conv_part_err_to_int, + endpoint::Endpoint, + error::{DiskError, Error, FileAccessDeniedWithContext, Result}, + error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, + format::FormatV3, + fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, + os, + os::{check_path_length, is_empty_dir, is_root_disk, rename_all}, }; -use crate::disk::{FileWriter, STORAGE_FORMAT_FILE}; -use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; -use rustfs_utils::path::{ - GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix, - path_join, path_join_buf, -}; -use tokio::time::interval; - use crate::erasure_coding::bitrot_verify; -use bytes::Bytes; -// use path_absolutize::Absolutize; // Replaced with direct path operations for better performance use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached}; +use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; +use bytes::Bytes; use parking_lot::RwLock as ParkingLotRwLock; use rustfs_filemeta::{ Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn, @@ -52,6 +39,10 @@ use rustfs_filemeta::{ }; use rustfs_utils::HashAlgorithm; use rustfs_utils::os::get_info; +use rustfs_utils::path::{ + GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR_STR, clean, decode_dir_object, encode_dir_object, + has_suffix, path_join, path_join_buf, +}; use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; @@ -67,6 +58,7 @@ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind}; use tokio::sync::RwLock; +use tokio::time::interval; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -129,7 +121,8 @@ impl LocalDisk { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { debug!("Creating local disk"); // Use optimized path resolution instead of absolutize() for better performance - let root = match std::fs::canonicalize(ep.get_file_path()) { + // Use dunce::canonicalize instead of std::fs::canonicalize to avoid UNC paths on Windows + let root = match dunce::canonicalize(ep.get_file_path()) { Ok(path) => path, Err(e) => { if e.kind() == ErrorKind::NotFound { @@ -483,7 +476,7 @@ impl LocalDisk { // Async prefetch related files, don't block current read if let Some(parent) = file_path.parent() { - prefetch_metadata_patterns(parent, &[super::STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await; + prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await; } // Main read logic @@ -507,7 +500,7 @@ impl LocalDisk { async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result>>> { let paths: Vec = requests .iter() - .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, super::STORAGE_FORMAT_FILE))) + .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE))) .collect::>>()?; let cache = get_global_file_cache(); @@ -544,7 +537,7 @@ impl LocalDisk { // TODO: async notifications for disk space checks and trash cleanup - let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; + let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; // if let Some(parent) = trash_path.parent() { // if !parent.exists() { // fs::create_dir_all(parent).await?; @@ -552,7 +545,7 @@ impl LocalDisk { // } let err = if recursive { - rename_all(delete_path, trash_path, self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?) + rename_all(delete_path, trash_path, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?) .await .err() } else { @@ -562,12 +555,12 @@ impl LocalDisk { .err() }; - if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) { - let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; + if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR_STR) { + let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; let _ = rename_all( encode_dir_object(delete_path.to_string_lossy().as_ref()), trash_path2, - self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?, + self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, ) .await; } @@ -916,7 +909,7 @@ impl LocalDisk { } if let Some(parent) = path.as_ref().parent() { - super::os::make_dir_all(parent, skip_parent).await?; + os::make_dir_all(parent, skip_parent).await?; } let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; @@ -942,7 +935,7 @@ impl LocalDisk { let meta = file.metadata().await.map_err(to_file_error)?; let file_size = meta.len() as usize; - bitrot_verify(Box::new(file), file_size, part_size, algo, bytes::Bytes::copy_from_slice(sum), shard_size) + bitrot_verify(Box::new(file), file_size, part_size, algo, Bytes::copy_from_slice(sum), shard_size) .await .map_err(to_file_error)?; @@ -1038,15 +1031,16 @@ impl LocalDisk { continue; } - if entry.ends_with(SLASH_SEPARATOR) { + if entry.ends_with(SLASH_SEPARATOR_STR) { if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { - let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR); + let entry = + format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR_STR); dir_objes.insert(entry.clone()); *item = entry; continue; } - *item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned(); + *item = entry.trim_end_matches(SLASH_SEPARATOR_STR).to_owned(); continue; } @@ -1058,7 +1052,7 @@ impl LocalDisk { .await?; let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned(); - let name = entry.trim_end_matches(SLASH_SEPARATOR); + let name = entry.trim_end_matches(SLASH_SEPARATOR_STR); let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str()); // if opts.limit > 0 @@ -1141,7 +1135,7 @@ impl LocalDisk { Ok(res) => { if is_dir_obj { meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned(); - meta.name.push_str(SLASH_SEPARATOR); + meta.name.push_str(SLASH_SEPARATOR_STR); } meta.metadata = res; @@ -1159,7 +1153,7 @@ impl LocalDisk { // NOT an object, append to stack (with slash) // If dirObject, but no metadata (which is unexpected) we skip it. if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { - meta.name.push_str(SLASH_SEPARATOR); + meta.name.push_str(SLASH_SEPARATOR_STR); dir_stack.push(meta.name); } } @@ -1234,7 +1228,7 @@ async fn read_file_metadata(p: impl AsRef) -> Result { fn skip_access_checks(p: impl AsRef) -> bool { let vols = [ - super::RUSTFS_META_TMP_DELETED_BUCKET, + RUSTFS_META_TMP_DELETED_BUCKET, super::RUSTFS_META_TMP_BUCKET, super::RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_BUCKET, @@ -1628,8 +1622,8 @@ impl DiskAPI for LocalDisk { super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))? } - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); + let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR); + let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR); if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir { warn!( @@ -1695,8 +1689,8 @@ impl DiskAPI for LocalDisk { .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; } - let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); - let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); + let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR); + let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR); if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) { return Err(Error::from(DiskError::FileAccessDenied)); } @@ -1847,12 +1841,12 @@ impl DiskAPI for LocalDisk { } let volume_dir = self.get_bucket_path(volume)?; - let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?; + let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR_STR))?; let entries = match os::read_dir(&dir_path_abs, count).await { Ok(res) => res, Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound + if e.kind() == ErrorKind::NotFound && !skip_access_checks(volume) && let Err(e) = access(&volume_dir).await { @@ -1883,11 +1877,11 @@ impl DiskAPI for LocalDisk { let mut objs_returned = 0; - if opts.base_dir.ends_with(SLASH_SEPARATOR) { + if opts.base_dir.ends_with(SLASH_SEPARATOR_STR) { let fpath = self.get_object_path( &opts.bucket, path_join_buf(&[ - format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(), + format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX).as_str(), STORAGE_FORMAT_FILE, ]) .as_str(), @@ -2119,7 +2113,7 @@ impl DiskAPI for LocalDisk { let volume_dir = self.get_bucket_path(volume)?; if let Err(e) = access(&volume_dir).await { - if e.kind() == std::io::ErrorKind::NotFound { + if e.kind() == ErrorKind::NotFound { os::make_dir_all(&volume_dir, self.root.as_path()).await?; return Ok(()); } @@ -2137,7 +2131,7 @@ impl DiskAPI for LocalDisk { let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?; for entry in entries { - if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) { + if !has_suffix(&entry, SLASH_SEPARATOR_STR) || !Self::is_valid_volname(clean(&entry).as_str()) { continue; } @@ -2359,7 +2353,7 @@ impl DiskAPI for LocalDisk { force_del_marker: bool, opts: DeleteOptions, ) -> Result<()> { - if path.starts_with(SLASH_SEPARATOR) { + if path.starts_with(SLASH_SEPARATOR_STR) { return self .delete( volume, @@ -2420,7 +2414,7 @@ impl DiskAPI for LocalDisk { if !meta.versions.is_empty() { let buf = meta.marshal_msg()?; return self - .write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true) + .write_all_meta(volume, format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true) .await; } @@ -2430,11 +2424,11 @@ impl DiskAPI for LocalDisk { { let src_path = path_join(&[ file_path.as_path(), - Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()), + Path::new(format!("{old_data_dir}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()), ]); let dst_path = path_join(&[ file_path.as_path(), - Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()), + Path::new(format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str()), ]); return rename_all(src_path, dst_path, file_path).await; } @@ -2563,7 +2557,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf if root_disk_threshold > 0 { disk_info.total <= root_disk_threshold } else { - is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default() + is_root_disk(&drive_path, SLASH_SEPARATOR_STR).unwrap_or_default() } } else { false @@ -2581,7 +2575,7 @@ mod test { // let arr = Vec::new(); let vols = [ - super::super::RUSTFS_META_TMP_DELETED_BUCKET, + RUSTFS_META_TMP_DELETED_BUCKET, super::super::RUSTFS_META_TMP_BUCKET, super::super::RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_BUCKET, @@ -2609,9 +2603,7 @@ mod test { let disk = LocalDisk::new(&ep, false).await.unwrap(); - let tmpp = disk - .resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET)) - .unwrap(); + let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap(); println!("ppp :{:?}", &tmpp); @@ -2639,9 +2631,7 @@ mod test { let disk = LocalDisk::new(&ep, false).await.unwrap(); - let tmpp = disk - .resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET)) - .unwrap(); + let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap(); println!("ppp :{:?}", &tmpp); diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 7ec5d0ff..660deec5 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -19,7 +19,7 @@ use std::{ use super::error::Result; use crate::disk::error_conv::to_file_error; -use rustfs_utils::path::SLASH_SEPARATOR; +use rustfs_utils::path::SLASH_SEPARATOR_STR; use tokio::fs; use tracing::warn; @@ -118,7 +118,7 @@ pub async fn read_dir(path: impl AsRef, count: i32) -> std::io::Result (String, S let trimmed_path = path .strip_prefix(base_path) .unwrap_or(path) - .strip_prefix(SLASH_SEPARATOR) + .strip_prefix(SLASH_SEPARATOR_STR) .unwrap_or(path); // Find the position of the first '/' - let pos = trimmed_path.find(SLASH_SEPARATOR).unwrap_or(trimmed_path.len()); + let pos = trimmed_path.find(SLASH_SEPARATOR_STR).unwrap_or(trimmed_path.len()); // Split into bucket and prefix let bucket = &trimmed_path[0..pos]; let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 92089d02..678d2dab 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -82,7 +82,7 @@ use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX, use rustfs_utils::{ HashAlgorithm, crypto::hex, - path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, + path::{SLASH_SEPARATOR_STR, encode_dir_object, has_suffix, path_join_buf}, }; use rustfs_workers::workers::Workers; use s3s::header::X_AMZ_RESTORE; @@ -5324,7 +5324,7 @@ impl StorageAPI for SetDisks { &upload_id_path, fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(), ]), - SLASH_SEPARATOR + SLASH_SEPARATOR_STR ); let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await { @@ -5462,7 +5462,7 @@ impl StorageAPI for SetDisks { let mut populated_upload_ids = HashSet::new(); for upload_id in upload_ids.iter() { - let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR).to_string(); + let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR_STR).to_string(); if populated_upload_ids.contains(&upload_id) { continue; } @@ -6222,7 +6222,7 @@ impl StorageAPI for SetDisks { None }; - if has_suffix(object, SLASH_SEPARATOR) { + if has_suffix(object, SLASH_SEPARATOR_STR) { let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?; return Ok((result, err.map(|e| e.into()))); } diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index c28a4c42..ff10fc41 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -34,7 +34,7 @@ use rustfs_filemeta::{ MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams, merge_file_meta_versions, }; -use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix}; +use rustfs_utils::path::{self, SLASH_SEPARATOR_STR, base_dir_from_prefix}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::broadcast::{self}; @@ -132,7 +132,7 @@ impl ListPathOptions { return; } - let s = SLASH_SEPARATOR.chars().next().unwrap_or_default(); + let s = SLASH_SEPARATOR_STR.chars().next().unwrap_or_default(); self.filter_prefix = { let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s); @@ -346,7 +346,7 @@ impl ECStore { if let Some(delimiter) = &delimiter { if obj.is_dir && obj.mod_time.is_none() { let mut found = false; - if delimiter != SLASH_SEPARATOR { + if delimiter != SLASH_SEPARATOR_STR { for p in prefixes.iter() { if found { break; @@ -470,7 +470,7 @@ impl ECStore { if let Some(delimiter) = &delimiter { if obj.is_dir && obj.mod_time.is_none() { let mut found = false; - if delimiter != SLASH_SEPARATOR { + if delimiter != SLASH_SEPARATOR_STR { for p in prefixes.iter() { if found { break; @@ -502,7 +502,7 @@ impl ECStore { // warn!("list_path opt {:?}", &o); check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?; - // if opts.prefix.ends_with(SLASH_SEPARATOR) { + // if opts.prefix.ends_with(SLASH_SEPARATOR_STR) { // return Err(Error::msg("eof")); // } @@ -520,11 +520,11 @@ impl ECStore { return Err(Error::Unexpected); } - if o.prefix.starts_with(SLASH_SEPARATOR) { + if o.prefix.starts_with(SLASH_SEPARATOR_STR) { return Err(Error::Unexpected); } - let slash_separator = Some(SLASH_SEPARATOR.to_owned()); + let slash_separator = Some(SLASH_SEPARATOR_STR.to_owned()); o.include_directories = o.separator == slash_separator; @@ -774,8 +774,8 @@ impl ECStore { let mut filter_prefix = { prefix .trim_start_matches(&path) - .trim_start_matches(SLASH_SEPARATOR) - .trim_end_matches(SLASH_SEPARATOR) + .trim_start_matches(SLASH_SEPARATOR_STR) + .trim_end_matches(SLASH_SEPARATOR_STR) .to_owned() }; @@ -1130,7 +1130,7 @@ async fn merge_entry_channels( if path::clean(&best_entry.name) == path::clean(&other_entry.name) { let dir_matches = best_entry.is_dir() && other_entry.is_dir(); let suffix_matches = - best_entry.name.ends_with(SLASH_SEPARATOR) == other_entry.name.ends_with(SLASH_SEPARATOR); + best_entry.name.ends_with(SLASH_SEPARATOR_STR) == other_entry.name.ends_with(SLASH_SEPARATOR_STR); if dir_matches && suffix_matches { to_merge.push(other_idx); diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index 1d11ad9e..37079e27 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -51,7 +51,7 @@ use crate::{ store_api::{ObjectOptions, PutObjReader}, }; use rustfs_rio::HashReader; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; +use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join}; use s3s::S3ErrorCode; use super::{ @@ -403,7 +403,7 @@ impl TierConfigMgr { pub async fn save_tiering_config(&self, api: Arc) -> std::result::Result<(), std::io::Error> { let data = self.marshal()?; - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE); self.save_config(api, &config_file, data).await } @@ -483,7 +483,7 @@ async fn new_and_save_tiering_config(api: Arc) -> Result) -> std::result::Result { - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE); let data = read_config(api.clone(), config_file.as_str()).await; if let Err(err) = data { if is_err_config_not_found(&err) { diff --git a/crates/ecstore/src/tier/warm_backend_s3.rs b/crates/ecstore/src/tier/warm_backend_s3.rs index e1b500c5..85453311 100644 --- a/crates/ecstore/src/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/tier/warm_backend_s3.rs @@ -30,13 +30,11 @@ use crate::client::{ transition_api::{Options, TransitionClient, TransitionCore}, transition_api::{ReadCloser, ReaderImpl}, }; -use crate::error::ErrorResponse; -use crate::error::error_resp_to_object_err; use crate::tier::{ tier_config::TierS3, warm_backend::{WarmBackend, WarmBackendGetOpts}, }; -use rustfs_utils::path::SLASH_SEPARATOR; +use rustfs_utils::path::SLASH_SEPARATOR_STR; pub struct WarmBackendS3 { pub client: Arc, @@ -178,7 +176,7 @@ impl WarmBackend for WarmBackendS3 { async fn in_use(&self) -> Result { let result = self .core - .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1) + .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR_STR, 1) .await?; Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) diff --git a/crates/ecstore/src/tier/warm_backend_s3sdk.rs b/crates/ecstore/src/tier/warm_backend_s3sdk.rs index 15ae827b..8e7d29ce 100644 --- a/crates/ecstore/src/tier/warm_backend_s3sdk.rs +++ b/crates/ecstore/src/tier/warm_backend_s3sdk.rs @@ -27,19 +27,11 @@ use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; -use crate::client::{ - api_get_options::GetObjectOptions, - api_put_object::PutObjectOptions, - api_remove::RemoveObjectOptions, - transition_api::{ReadCloser, ReaderImpl}, -}; -use crate::error::ErrorResponse; -use crate::error::error_resp_to_object_err; +use crate::client::transition_api::{ReadCloser, ReaderImpl}; use crate::tier::{ tier_config::TierS3, warm_backend::{WarmBackend, WarmBackendGetOpts}, }; -use rustfs_utils::path::SLASH_SEPARATOR; pub struct WarmBackendS3 { pub client: Arc, diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 930a8743..bc50d5ea 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -32,7 +32,7 @@ use rustfs_ecstore::{ store_api::{ObjectInfo, ObjectOptions}, }; use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; +use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf}; use serde::{Serialize, de::DeserializeOwned}; use std::sync::LazyLock; use std::{collections::HashMap, sync::Arc}; @@ -182,7 +182,7 @@ impl ObjectStore { } else { info.name }; - let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR); + let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR_STR); let _ = sender .send(StringOrErr { item: Some(name.to_owned()), diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 9ba7c712..80f9aa8a 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -83,7 +83,7 @@ ip = ["dep:local-ip-address"] # ip characteristics and their dependencies tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver io = ["dep:tokio"] -path = [] +path = [] # path manipulation features notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex"] diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index 0eb7c9f7..0bf75064 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -15,12 +15,36 @@ use std::path::Path; use std::path::PathBuf; +#[cfg(target_os = "windows")] +const SLASH_SEPARATOR: char = '\\'; +#[cfg(not(target_os = "windows"))] +const SLASH_SEPARATOR: char = '/'; + +/// GLOBAL_DIR_SUFFIX is a special suffix used to denote directory objects +/// in object storage systems that do not have a native directory concept. pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; -pub const SLASH_SEPARATOR: &str = "/"; +/// SLASH_SEPARATOR_STR is the string representation of the path separator +/// used in the current operating system. +pub const SLASH_SEPARATOR_STR: &str = if cfg!(target_os = "windows") { "\\" } else { "/" }; +/// GLOBAL_DIR_SUFFIX_WITH_SLASH is the directory suffix followed by the +/// platform-specific path separator, used to denote directory objects. +#[cfg(target_os = "windows")] +pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__\\"; +#[cfg(not(target_os = "windows"))] pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/"; +/// has_suffix checks if the string `s` ends with the specified `suffix`, +/// performing a case-insensitive comparison on Windows platforms. +/// +/// # Arguments +/// * `s` - A string slice that holds the string to be checked. +/// * `suffix` - A string slice that holds the suffix to check for. +/// +/// # Returns +/// A boolean indicating whether `s` ends with `suffix`. +/// pub fn has_suffix(s: &str, suffix: &str) -> bool { if cfg!(target_os = "windows") { s.to_lowercase().ends_with(&suffix.to_lowercase()) @@ -29,19 +53,46 @@ pub fn has_suffix(s: &str, suffix: &str) -> bool { } } +/// encode_dir_object encodes a directory object by appending +/// a special suffix if it ends with a slash. +/// +/// # Arguments +/// * `object` - A string slice that holds the object to be encoded. +/// +/// # Returns +/// A `String` representing the encoded directory object. +/// pub fn encode_dir_object(object: &str) -> String { - if has_suffix(object, SLASH_SEPARATOR) { - format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX) + if has_suffix(object, SLASH_SEPARATOR_STR) { + format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX) } else { object.to_string() } } +/// is_dir_object checks if the given object string represents +/// a directory object by verifying if it ends with the special suffix. +/// +/// # Arguments +/// * `object` - A string slice that holds the object to be checked. +/// +/// # Returns +/// A boolean indicating whether the object is a directory object. +/// pub fn is_dir_object(object: &str) -> bool { let obj = encode_dir_object(object); obj.ends_with(GLOBAL_DIR_SUFFIX) } +/// decode_dir_object decodes a directory object by removing +/// the special suffix if it is present. +/// +/// # Arguments +/// * `object` - A string slice that holds the object to be decoded. +/// +/// # Returns +/// A `String` representing the decoded directory object. +/// #[allow(dead_code)] pub fn decode_dir_object(object: &str) -> String { if has_suffix(object, GLOBAL_DIR_SUFFIX) { @@ -51,21 +102,50 @@ pub fn decode_dir_object(object: &str) -> String { } } +/// retain_slash ensures that the given string `s` ends with a slash. +/// If it does not, a slash is appended. +/// +/// # Arguments +/// * `s` - A string slice that holds the string to be processed. +/// +/// # Returns +/// A `String` that ends with a slash. +/// pub fn retain_slash(s: &str) -> String { if s.is_empty() { return s.to_string(); } - if s.ends_with(SLASH_SEPARATOR) { + if s.ends_with(SLASH_SEPARATOR_STR) { s.to_string() } else { - format!("{s}{SLASH_SEPARATOR}") + format!("{s}{SLASH_SEPARATOR_STR}") } } +/// strings_has_prefix_fold checks if the string `s` starts with the specified `prefix`, +/// performing a case-insensitive comparison on Windows platforms. +/// +/// # Arguments +/// * `s` - A string slice that holds the string to be checked. +/// * `prefix` - A string slice that holds the prefix to check for. +/// +/// # Returns +/// A boolean indicating whether `s` starts with `prefix`. +/// pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) } +/// has_prefix checks if the string `s` starts with the specified `prefix`, +/// performing a case-insensitive comparison on Windows platforms. +/// +/// # Arguments +/// * `s` - A string slice that holds the string to be checked. +/// * `prefix` - A string slice that holds the prefix to check for. +/// +/// # Returns +/// A boolean indicating whether `s` starts with `prefix`. +/// pub fn has_prefix(s: &str, prefix: &str) -> bool { if cfg!(target_os = "windows") { return strings_has_prefix_fold(s, prefix); @@ -74,21 +154,37 @@ pub fn has_prefix(s: &str, prefix: &str) -> bool { s.starts_with(prefix) } +/// path_join joins multiple path elements into a single PathBuf, +/// ensuring that the resulting path is clean and properly formatted. +/// +/// # Arguments +/// * `elem` - A slice of path elements to be joined. +/// +/// # Returns +/// A PathBuf representing the joined path. +/// pub fn path_join>(elem: &[P]) -> PathBuf { - path_join_buf( - elem.iter() - .map(|p| p.as_ref().to_string_lossy().into_owned()) - .collect::>() - .iter() - .map(|s| s.as_str()) - .collect::>() - .as_slice(), - ) - .into() + if elem.is_empty() { + return PathBuf::from("."); + } + // Collect components as owned Strings (lossy for non-UTF8) + let strs: Vec = elem.iter().map(|p| p.as_ref().to_string_lossy().into_owned()).collect(); + // Convert to slice of &str for path_join_buf + let refs: Vec<&str> = strs.iter().map(|s| s.as_str()).collect(); + PathBuf::from(path_join_buf(&refs)) } +/// path_join_buf joins multiple string path elements into a single String, +/// ensuring that the resulting path is clean and properly formatted. +/// +/// # Arguments +/// * `elements` - A slice of string path elements to be joined. +/// +/// # Returns +/// A String representing the joined path. +/// pub fn path_join_buf(elements: &[&str]) -> String { - let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR)); + let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR_STR)); let mut dst = String::new(); let mut added = 0; @@ -96,7 +192,7 @@ pub fn path_join_buf(elements: &[&str]) -> String { for e in elements { if added > 0 || !e.is_empty() { if added > 0 { - dst.push_str(SLASH_SEPARATOR); + dst.push(SLASH_SEPARATOR); } dst.push_str(e); added += e.len(); @@ -106,113 +202,535 @@ pub fn path_join_buf(elements: &[&str]) -> String { if path_needs_clean(dst.as_bytes()) { let mut clean_path = clean(&dst); if trailing_slash { - clean_path.push_str(SLASH_SEPARATOR); + clean_path.push(SLASH_SEPARATOR); } return clean_path; } if trailing_slash { - dst.push_str(SLASH_SEPARATOR); + dst.push(SLASH_SEPARATOR); } dst } +/// Platform-aware separator check +fn is_sep(b: u8) -> bool { + #[cfg(target_os = "windows")] + { + b == b'/' || b == b'\\' + } + #[cfg(not(target_os = "windows"))] + { + b == b'/' + } +} + /// path_needs_clean returns whether path cleaning may change the path. /// Will detect all cases that will be cleaned, /// but may produce false positives on non-trivial paths. +/// +/// # Arguments +/// * `path` - A byte slice that holds the path to be checked. +/// +/// # Returns +/// A boolean indicating whether the path needs cleaning. +/// fn path_needs_clean(path: &[u8]) -> bool { if path.is_empty() { return true; } - let rooted = path[0] == b'/'; let n = path.len(); - let (mut r, mut w) = if rooted { (1, 1) } else { (0, 0) }; + // On Windows: any forward slash indicates normalization to backslash is required. + #[cfg(target_os = "windows")] + { + if path.iter().any(|&b| b == b'/') { + return true; + } + } - while r < n { - match path[r] { - b if b > 127 => { - // Non ascii. + // Initialize scan index and previous-separator flag. + let mut i = 0usize; + let mut prev_was_sep = false; + + // Platform-aware prefix handling to avoid flagging meaningful leading sequences: + // - Windows: handle drive letter "C:" and UNC leading "\\" + // - Non-Windows: detect and flag double leading '/' (e.g. "//abc") as needing clean + if n >= 1 && is_sep(path[0]) { + #[cfg(target_os = "windows")] + { + // If starts with two separators -> UNC prefix: allow exactly two without flag + if n >= 2 && is_sep(path[1]) { + // If a third leading separator exists, that's redundant (e.g. "///...") -> needs clean + if n >= 3 && is_sep(path[2]) { + return true; + } + // Skip the two UNC leading separators for scanning; do not mark prev_was_sep true + i = 2; + prev_was_sep = false; + } else { + // Single leading separator (rooted) -> mark as seen separator so immediate next sep is duplicate + i = 1; + prev_was_sep = true; + } + } + + #[cfg(not(target_os = "windows"))] + { + // POSIX: double leading '/' is redundant and should be cleaned (e.g. "//abc" -> "/abc") + if n >= 2 && is_sep(path[1]) { return true; } - b'/' => { - // multiple / elements - return true; - } - b'.' => { - if r + 1 == n || path[r + 1] == b'/' { - // . element - assume it has to be cleaned. - return true; - } - if r + 1 < n && path[r + 1] == b'.' && (r + 2 == n || path[r + 2] == b'/') { - // .. element: remove to last / - assume it has to be cleaned. - return true; - } - // Handle single dot case - if r + 1 == n { - // . element - assume it has to be cleaned. - return true; - } - // Copy the dot - w += 1; - r += 1; - } - _ => { - // real path element. - // add slash if needed - if (rooted && w != 1) || (!rooted && w != 0) { - w += 1; - } - // copy element - while r < n && path[r] != b'/' { - w += 1; - r += 1; - } - // allow one slash, not at end - if r < n - 1 && path[r] == b'/' { - r += 1; + i = 1; + prev_was_sep = true; + } + } else { + // If not starting with separator, check for Windows drive-letter prefix like "C:" + #[cfg(target_os = "windows")] + { + if n >= 2 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() { + // Position after "C:" + i = 2; + // If a separator immediately follows the drive (rooted like "C:\"), + // treat that first separator as seen; if more separators follow, it's redundant. + if i < n && is_sep(path[i]) { + i += 1; // consume the single allowed separator after drive + if i < n && is_sep(path[i]) { + // multiple separators after drive like "C:\\..." -> needs clean + return true; + } + prev_was_sep = true; + } else { + prev_was_sep = false; } } } } - // Turn empty string into "." - if w == 0 { - return true; + // Generic scan for repeated separators and dot / dot-dot components. + while i < n { + let b = path[i]; + if is_sep(b) { + if prev_was_sep { + // Multiple separators (except allowed UNC prefix handled above) + return true; + } + prev_was_sep = true; + i += 1; + continue; + } + + // Not a separator: parse current path element + let start = i; + while i < n && !is_sep(path[i]) { + i += 1; + } + let len = i - start; + if len == 1 && path[start] == b'.' { + // single "." element -> needs cleaning + return true; + } + if len == 2 && path[start] == b'.' && path[start + 1] == b'.' { + // ".." element -> needs cleaning + return true; + } + prev_was_sep = false; } + // Trailing separator: if last byte is a separator and path length > 1, then usually needs cleaning, + // except when the path is a platform-specific root form (e.g. "/" on POSIX, "\\" or "C:\" on Windows). + if n > 1 && is_sep(path[n - 1]) { + #[cfg(not(target_os = "windows"))] + { + // POSIX: any trailing separator except the single-root "/" needs cleaning. + return true; + } + #[cfg(target_os = "windows")] + { + // Windows special root forms that are acceptable with trailing separator: + // - UNC root: exactly two leading separators "\" "\" (i.e. "\\") -> n == 2 + if n == 2 && is_sep(path[0]) && is_sep(path[1]) { + return false; + } + // - Drive root: pattern "C:\" or "C:/" (len == 3) + if n == 3 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() && is_sep(path[2]) { + return false; + } + // Otherwise, trailing separator should be cleaned. + return true; + } + } + + // No conditions triggered: assume path is already clean. false } -pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) { - let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR); +/// path_to_bucket_object_with_base_path splits a given path into bucket and object components, +/// considering a base path to trim from the start. +/// +/// # Arguments +/// * `base_path` - A string slice that holds the base path to be trimmed. +/// * `path` - A string slice that holds the path to be split. +/// +/// # Returns +/// A tuple containing the bucket and object as `String`s. +/// +pub fn path_to_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) { + let path = path.trim_start_matches(base_path).trim_start_matches(SLASH_SEPARATOR); if let Some(m) = path.find(SLASH_SEPARATOR) { - return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string()); + return (path[..m].to_string(), path[m + SLASH_SEPARATOR_STR.len()..].to_string()); } (path.to_string(), "".to_string()) } +/// path_to_bucket_object splits a given path into bucket and object components. +/// +/// # Arguments +/// * `s` - A string slice that holds the path to be split. +/// +/// # Returns +/// A tuple containing the bucket and object as `String`s. +/// pub fn path_to_bucket_object(s: &str) -> (String, String) { path_to_bucket_object_with_base_path("", s) } +/// contains_any_sep_str checks if the given string contains any path separators. +/// +/// # Arguments +/// * `s` - A string slice that holds the string to be checked. +/// +/// # Returns +/// A boolean indicating whether the string contains any path separators. +fn contains_any_sep_str(s: &str) -> bool { + #[cfg(target_os = "windows")] + { + s.contains('/') || s.contains('\\') + } + #[cfg(not(target_os = "windows"))] + { + s.contains('/') + } +} + +/// base_dir_from_prefix extracts the base directory from a given prefix. +/// +/// # Arguments +/// * `prefix` - A string slice that holds the prefix to be processed. +/// +/// # Returns +/// A `String` representing the base directory extracted from the prefix. +/// pub fn base_dir_from_prefix(prefix: &str) -> String { - let mut base_dir = dir(prefix).to_owned(); - if base_dir == "." || base_dir == "./" || base_dir == "/" { - base_dir = "".to_owned(); + if !contains_any_sep_str(prefix) { + return String::new(); } - if !prefix.contains('/') { - base_dir = "".to_owned(); + let mut base_dir = dir(prefix); + if base_dir == "." || base_dir == SLASH_SEPARATOR_STR { + base_dir.clear(); } - if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) { - base_dir.push_str(SLASH_SEPARATOR); + if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR_STR) { + base_dir.push_str(SLASH_SEPARATOR_STR); } base_dir } +/// clean returns the shortest path name equivalent to path +/// by purely lexical processing. It applies the following rules +/// iteratively until no further processing can be done: +/// +/// 1. Replace multiple slashes with a single slash. +/// 2. Eliminate each . path name element (the current directory). +/// 3. Eliminate each inner .. path name element (the parent directory) +/// along with the non-.. element that precedes it. +/// 4. Eliminate .. elements that begin a rooted path, +/// that is, replace "/.." by "/" at the beginning of a path. +/// +/// If the result of this process is an empty string, clean returns the string ".". +/// +/// This function is adapted to work cross-platform by using the appropriate path separator. +/// On Windows, this function is aware of drive letters (e.g., `C:`) and UNC paths +/// (e.g., `\\server\share`) and cleans them using the appropriate separator. +/// +/// # Arguments +/// * `path` - A string slice that holds the path to be cleaned. +/// +/// # Returns +/// A `String` representing the cleaned path. +/// +pub fn clean(path: &str) -> String { + if path.is_empty() { + return ".".to_string(); + } + + #[cfg(target_os = "windows")] + { + use std::borrow::Cow; + let bytes = path.as_bytes(); + let n = bytes.len(); + // Windows-aware handling + let mut i = 0usize; + let mut drive: Option = None; + let mut rooted = false; + let mut preserve_leading_double_sep = false; + + // Drive letter detection + if n >= 2 && bytes[1] == b':' && (bytes[0] as char).is_ascii_alphabetic() { + drive = Some(bytes[0] as char); + i = 2; + // If next is separator, it's an absolute drive-root (e.g., "C:\") + if i < n && is_sep(bytes[i]) { + rooted = true; + // consume all leading separators after drive + while i < n && is_sep(bytes[i]) { + i += 1; + } + } + } else { + // UNC or absolute by separators + if n >= 2 && is_sep(bytes[0]) && is_sep(bytes[1]) { + rooted = true; + preserve_leading_double_sep = true; + i = 2; + // consume extra leading separators + while i < n && is_sep(bytes[i]) { + i += 1; + } + } else if is_sep(bytes[0]) { + rooted = true; + i = 1; + while i < n && is_sep(bytes[i]) { + i += 1; + } + } + } + + // Component stack + let mut comps: Vec> = Vec::new(); + let mut r = i; + while r < n { + // find next sep or end + let start = r; + while r < n && !is_sep(bytes[r]) { + r += 1; + } + // component bytes [start..r) + let comp = String::from_utf8_lossy(&bytes[start..r]); + if comp == "." { + // skip + } else if comp == ".." { + if !comps.is_empty() { + // pop last component + comps.pop(); + } else if !rooted { + // relative path with .. at front must be kept + comps.push(Cow::Owned("..".to_string())); + } else { + // rooted and at root => ignore + } + } else { + comps.push(comp); + } + // skip separators + while r < n && is_sep(bytes[r]) { + r += 1; + } + } + + // Build result + let mut out = String::new(); + if let Some(d) = drive { + out.push(d); + out.push(':'); + if rooted { + out.push(SLASH_SEPARATOR); + } + } else if preserve_leading_double_sep { + out.push(SLASH_SEPARATOR); + out.push(SLASH_SEPARATOR); + } else if rooted { + out.push(SLASH_SEPARATOR); + } + + // Join components + for (idx, c) in comps.iter().enumerate() { + if !out.is_empty() && !out.ends_with(SLASH_SEPARATOR_STR) { + out.push(SLASH_SEPARATOR); + } + out.push_str(c); + } + + // Special cases: + if out.is_empty() { + // No drive, no components -> "." + return ".".to_string(); + } + + // If output is just "C:" (drive without components and not rooted), keep as "C:" + if drive.is_some() { + if out.len() == 2 && out.as_bytes()[1] == b':' { + return out; + } + // If drive+colon+sep and no components, return "C:\" + if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) { + return out; + } + } + + // Remove trailing separator unless it's a root form (single leading sep or drive root or UNC) + if out.len() > 1 && out.ends_with(SLASH_SEPARATOR_STR) { + // Determine if it's a root form: "/" or "\\" or "C:\" + let is_root = { + // "/" (non-drive single sep) + if out == SLASH_SEPARATOR_STR { + true + } else if out.starts_with(SLASH_SEPARATOR_STR) && out == format!("{}{}", SLASH_SEPARATOR_STR, SLASH_SEPARATOR_STR) + { + // only double separator + true + } else { + // drive root "C:\" length >=3 with pattern X:\ + if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) { + true + } else { + false + } + } + }; + if !is_root { + out.pop(); + } + } + + out + } + + #[cfg(not(target_os = "windows"))] + { + // POSIX-like behavior (original implementation but simplified) + let rooted = path.starts_with('/'); + let n = path.len(); + let mut out = LazyBuf::new(path.to_string()); + let mut r = 0usize; + let mut dotdot = 0usize; + + if rooted { + out.append(b'/'); + r = 1; + dotdot = 1; + } + + while r < n { + match path.as_bytes()[r] { + b'/' => { + // Empty path element + r += 1; + } + b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => { + // . element + r += 1; + } + b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => { + // .. element: remove to last / + r += 2; + + if out.w > dotdot { + // Can backtrack + out.w -= 1; + while out.w > dotdot && out.index(out.w) != b'/' { + out.w -= 1; + } + } else if !rooted { + // Cannot backtrack but not rooted, so append .. element. + if out.w > 0 { + out.append(b'/'); + } + out.append(b'.'); + out.append(b'.'); + dotdot = out.w; + } + } + _ => { + // Real path element. + // Add slash if needed + if (rooted && out.w != 1) || (!rooted && out.w != 0) { + out.append(b'/'); + } + + // Copy element + while r < n && path.as_bytes()[r] != b'/' { + out.append(path.as_bytes()[r]); + r += 1; + } + } + } + } + + // Turn empty string into "." + if out.w == 0 { + return ".".to_string(); + } + + out.string() + } +} + +/// split splits path immediately after the final slash, +/// separating it into a directory and file name component. +/// If there is no slash in path, split returns +/// ("", path). +/// +/// # Arguments +/// * `path` - A string slice that holds the path to be split. +/// +/// # Returns +/// A tuple containing the directory and file name as string slices. +/// +pub fn split(path: &str) -> (&str, &str) { + // Find the last occurrence of the '/' character + if let Some(i) = path.rfind(SLASH_SEPARATOR_STR) { + // Return the directory (up to and including the last '/') and the file name + return (&path[..i + 1], &path[i + 1..]); + } + // If no '/' is found, return an empty string for the directory and the whole path as the file name + (path, "") +} + +/// dir returns all but the last element of path, +/// typically the path's directory. After dropping the final +/// element, the path is cleaned. If the path is empty, +/// dir returns ".". +/// +/// # Arguments +/// * `path` - A string slice that holds the path to be processed. +/// +/// # Returns +/// A `String` representing the directory part of the path. +/// +pub fn dir(path: &str) -> String { + let (a, _) = split(path); + clean(a) +} + +/// trim_etag removes surrounding double quotes from an ETag string. +/// +/// # Arguments +/// * `etag` - A string slice that holds the ETag to be trimmed. +/// +/// # Returns +/// A `String` representing the trimmed ETag. +/// +pub fn trim_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} + +/// LazyBuf is a structure that efficiently builds a byte buffer +/// from a string by delaying the allocation of the buffer until +/// a modification is necessary. It allows appending bytes and +/// retrieving the current string representation. pub struct LazyBuf { s: String, buf: Option>, @@ -220,6 +738,13 @@ pub struct LazyBuf { } impl LazyBuf { + /// Creates a new LazyBuf with the given string. + /// + /// # Arguments + /// * `s` - A string to initialize the LazyBuf. + /// + /// # Returns + /// A new instance of LazyBuf. pub fn new(s: String) -> Self { LazyBuf { s, buf: None, w: 0 } } @@ -258,100 +783,84 @@ impl LazyBuf { } } -pub fn clean(path: &str) -> String { - if path.is_empty() { - return ".".to_string(); - } - - let rooted = path.starts_with('/'); - let n = path.len(); - let mut out = LazyBuf::new(path.to_string()); - let mut r = 0; - let mut dotdot = 0; - - if rooted { - out.append(b'/'); - r = 1; - dotdot = 1; - } - - while r < n { - match path.as_bytes()[r] { - b'/' => { - // Empty path element - r += 1; - } - b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => { - // . element - r += 1; - } - b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => { - // .. element: remove to last / - r += 2; - - if out.w > dotdot { - // Can backtrack - out.w -= 1; - while out.w > dotdot && out.index(out.w) != b'/' { - out.w -= 1; - } - } else if !rooted { - // Cannot backtrack but not rooted, so append .. element. - if out.w > 0 { - out.append(b'/'); - } - out.append(b'.'); - out.append(b'.'); - dotdot = out.w; - } - } - _ => { - // Real path element. - // Add slash if needed - if (rooted && out.w != 1) || (!rooted && out.w != 0) { - out.append(b'/'); - } - - // Copy element - while r < n && path.as_bytes()[r] != b'/' { - out.append(path.as_bytes()[r]); - r += 1; - } - } - } - } - - // Turn empty string into "." - if out.w == 0 { - return ".".to_string(); - } - - out.string() -} - -pub fn split(path: &str) -> (&str, &str) { - // Find the last occurrence of the '/' character - if let Some(i) = path.rfind('/') { - // Return the directory (up to and including the last '/') and the file name - return (&path[..i + 1], &path[i + 1..]); - } - // If no '/' is found, return an empty string for the directory and the whole path as the file name - (path, "") -} - -pub fn dir(path: &str) -> String { - let (a, _) = split(path); - clean(a) -} - -pub fn trim_etag(etag: &str) -> String { - etag.trim_matches('"').to_string() -} - #[cfg(test)] mod tests { use super::*; + #[test] + fn test_path_join_buf() { + #[cfg(not(target_os = "windows"))] + { + // Basic joining + assert_eq!(path_join_buf(&["a", "b"]), "a/b"); + assert_eq!(path_join_buf(&["a/", "b"]), "a/b"); + + // Empty array input + assert_eq!(path_join_buf(&[]), "."); + + // Single element + assert_eq!(path_join_buf(&["a"]), "a"); + + // Multiple elements + assert_eq!(path_join_buf(&["a", "b", "c"]), "a/b/c"); + + // Elements with trailing separators + assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/"); + + // Elements requiring cleaning (with "." and "..") + assert_eq!(path_join_buf(&["a", ".", "b"]), "a/b"); + assert_eq!(path_join_buf(&["a", "..", "b"]), "b"); + assert_eq!(path_join_buf(&["a", "b", ".."]), "a"); + + // Preservation of trailing slashes + assert_eq!(path_join_buf(&["a", "b/"]), "a/b/"); + assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/"); + + // Empty elements + assert_eq!(path_join_buf(&["a", "", "b"]), "a/b"); + + // Double slashes (cleaning) + assert_eq!(path_join_buf(&["a//", "b"]), "a/b"); + } + #[cfg(target_os = "windows")] + { + // Basic joining + assert_eq!(path_join_buf(&["a", "b"]), "a\\b"); + assert_eq!(path_join_buf(&["a\\", "b"]), "a\\b"); + + // Empty array input + assert_eq!(path_join_buf(&[]), "."); + + // Single element + assert_eq!(path_join_buf(&["a"]), "a"); + + // Multiple elements + assert_eq!(path_join_buf(&["a", "b", "c"]), "a\\b\\c"); + + // Elements with trailing separators + assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\"); + + // Elements requiring cleaning (with "." and "..") + assert_eq!(path_join_buf(&["a", ".", "b"]), "a\\b"); + assert_eq!(path_join_buf(&["a", "..", "b"]), "b"); + assert_eq!(path_join_buf(&["a", "b", ".."]), "a"); + + // Mixed separator handling + assert_eq!(path_join_buf(&["a/b", "c"]), "a\\b\\c"); + assert_eq!(path_join_buf(&["a\\", "b/c"]), "a\\b\\c"); + + // Preservation of trailing slashes + assert_eq!(path_join_buf(&["a", "b\\"]), "a\\b\\"); + assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\"); + + // Empty elements + assert_eq!(path_join_buf(&["a", "", "b"]), "a\\b"); + + // Double slashes (cleaning) + assert_eq!(path_join_buf(&["a\\\\", "b"]), "a\\b"); + } + } + #[test] fn test_trim_etag() { // Test with quoted ETag @@ -383,43 +892,56 @@ mod tests { #[test] fn test_clean() { - assert_eq!(clean(""), "."); - assert_eq!(clean("abc"), "abc"); - assert_eq!(clean("abc/def"), "abc/def"); - assert_eq!(clean("a/b/c"), "a/b/c"); - assert_eq!(clean("."), "."); - assert_eq!(clean(".."), ".."); - assert_eq!(clean("../.."), "../.."); - assert_eq!(clean("../../abc"), "../../abc"); - assert_eq!(clean("/abc"), "/abc"); - assert_eq!(clean("/"), "/"); - assert_eq!(clean("abc/"), "abc"); - assert_eq!(clean("abc/def/"), "abc/def"); - assert_eq!(clean("a/b/c/"), "a/b/c"); - assert_eq!(clean("./"), "."); - assert_eq!(clean("../"), ".."); - assert_eq!(clean("../../"), "../.."); - assert_eq!(clean("/abc/"), "/abc"); - assert_eq!(clean("abc//def//ghi"), "abc/def/ghi"); - assert_eq!(clean("//abc"), "/abc"); - assert_eq!(clean("///abc"), "/abc"); - assert_eq!(clean("//abc//"), "/abc"); - assert_eq!(clean("abc//"), "abc"); - assert_eq!(clean("abc/./def"), "abc/def"); - assert_eq!(clean("/./abc/def"), "/abc/def"); - assert_eq!(clean("abc/."), "abc"); - assert_eq!(clean("abc/./../def"), "def"); - assert_eq!(clean("abc//./../def"), "def"); - assert_eq!(clean("abc/../../././../def"), "../../def"); + #[cfg(not(target_os = "windows"))] + { + assert_eq!(clean(""), "."); + assert_eq!(clean("abc"), "abc"); + assert_eq!(clean("abc/def"), "abc/def"); + assert_eq!(clean("a/b/c"), "a/b/c"); + assert_eq!(clean("."), "."); + assert_eq!(clean(".."), ".."); + assert_eq!(clean("../.."), "../.."); + assert_eq!(clean("../../abc"), "../../abc"); + assert_eq!(clean("/abc"), "/abc"); + assert_eq!(clean("/"), "/"); + assert_eq!(clean("abc/"), "abc"); + assert_eq!(clean("abc/def/"), "abc/def"); + assert_eq!(clean("a/b/c/"), "a/b/c"); + assert_eq!(clean("./"), "."); + assert_eq!(clean("../"), ".."); + assert_eq!(clean("../../"), "../.."); + assert_eq!(clean("/abc/"), "/abc"); + assert_eq!(clean("abc//def//ghi"), "abc/def/ghi"); + assert_eq!(clean("//abc"), "/abc"); + assert_eq!(clean("///abc"), "/abc"); + assert_eq!(clean("//abc//"), "/abc"); + assert_eq!(clean("abc//"), "abc"); + assert_eq!(clean("abc/./def"), "abc/def"); + assert_eq!(clean("/./abc/def"), "/abc/def"); + assert_eq!(clean("abc/."), "abc"); + assert_eq!(clean("abc/./../def"), "def"); + assert_eq!(clean("abc//./../def"), "def"); + assert_eq!(clean("abc/../../././../def"), "../../def"); - assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl"); - assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl"); - assert_eq!(clean("abc/def/.."), "abc"); - assert_eq!(clean("abc/def/../.."), "."); - assert_eq!(clean("/abc/def/../.."), "/"); - assert_eq!(clean("abc/def/../../.."), ".."); - assert_eq!(clean("/abc/def/../../.."), "/"); - assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno"); + assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl"); + assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl"); + assert_eq!(clean("abc/def/.."), "abc"); + assert_eq!(clean("abc/def/../.."), "."); + assert_eq!(clean("/abc/def/../.."), "/"); + assert_eq!(clean("abc/def/../../.."), ".."); + assert_eq!(clean("/abc/def/../../.."), "/"); + assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno"); + } + + #[cfg(target_os = "windows")] + { + assert_eq!(clean("a\\b\\..\\c"), "a\\c"); + assert_eq!(clean("a\\\\b"), "a\\b"); + assert_eq!(clean("C:\\"), "C:\\"); + assert_eq!(clean("C:\\a\\..\\b"), "C:\\b"); + assert_eq!(clean("C:a\\b\\..\\c"), "C:a\\c"); + assert_eq!(clean("\\\\server\\share\\a\\\\b"), "\\\\server\\share\\a\\b"); + } } #[test] diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 013b79aa..5fc2d934 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -12,11 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{ - collections::HashMap, - io::{Cursor, Read as _, Write as _}, -}; - use crate::{ admin::{auth::validate_admin_request, router::Operation}, auth::{check_key_valid, get_session_token}, @@ -49,7 +44,7 @@ use rustfs_policy::policy::{ BucketPolicy, action::{Action, AdminAction}, }; -use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; +use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf}; use s3s::{ Body, S3Request, S3Response, S3Result, dto::{ @@ -61,6 +56,10 @@ use s3s::{ }; use serde::Deserialize; use serde_urlencoded::from_bytes; +use std::{ + collections::HashMap, + io::{Cursor, Read as _, Write as _}, +}; use time::OffsetDateTime; use tracing::warn; use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; @@ -424,7 +423,7 @@ impl Operation for ImportBucketMetadata { // Extract bucket names let mut bucket_names = Vec::new(); for (file_path, _) in &file_contents { - let file_path_split = file_path.split(SLASH_SEPARATOR).collect::>(); + let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::>(); if file_path_split.len() < 2 { warn!("file path is invalid: {}", file_path); @@ -463,7 +462,7 @@ impl Operation for ImportBucketMetadata { // Second pass: process file contents for (file_path, content) in file_contents { - let file_path_split = file_path.split(SLASH_SEPARATOR).collect::>(); + let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::>(); if file_path_split.len() < 2 { warn!("file path is invalid: {}", file_path);