mirror of
https://github.com/rustfs/rustfs.git
synced 2026-03-17 14:24:08 +00:00
Merge branch 'main' into fix/fix-issues-1553
This commit is contained in:
houseme
281
Cargo.lock
generated
281
Cargo.lock
generated
@@ -224,11 +224,11 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
version = "0.2.0"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a"
|
||||
checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
|
||||
dependencies = [
|
||||
"object 0.32.2",
|
||||
"object",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -868,9 +868,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-async"
|
||||
version = "1.2.7"
|
||||
version = "1.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c"
|
||||
checksum = "4e0e99800414b0c4cae85ed775a1559f8992f4e69f5ebafe9c936e29609eae78"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
@@ -899,9 +899,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.60.14"
|
||||
version = "0.60.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc12f8b310e38cad85cf3bef45ad236f470717393c613266ce0a89512286b650"
|
||||
checksum = "b1adc2eb689a24741c9dcc21ec19be78839a7899594883d3305cf4c7abae9b3d"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
@@ -932,9 +932,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.1.5"
|
||||
version = "1.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a"
|
||||
checksum = "23141f8daeab46574a1969ddb7316bc2928732e5721b3abfa8d1e16927ea9a52"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -965,18 +965,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-observability"
|
||||
version = "0.2.0"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef1fcbefc7ece1d70dcce29e490f269695dfca2d2bacdeaf9e5c3f799e4e6a42"
|
||||
checksum = "112e30b3c5379273de88c8cedfc96ce0211e9af22115ceb6975b5c072eccdfb9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.60.9"
|
||||
version = "0.60.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d"
|
||||
checksum = "2a57c5122eafc566cba4d3cbaacb53dd8a0cacd71155c728d1f4a9179cdd75ae"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"urlencoding",
|
||||
@@ -1008,9 +1008,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.10.0"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "efce7aaaf59ad53c5412f14fc19b2d5c6ab2c3ec688d272fd31f76ec12f44fb0"
|
||||
checksum = "1d09ba34c17c65a53b65b0ec0d80cf7a934947d407cb7c4fb7753f96de147663"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-types",
|
||||
@@ -1025,9 +1025,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.4.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2d447863bdec38c899e5753a48c0abcf590f3ec629e257ad5a9ef8806ad7714"
|
||||
checksum = "8169129deda9dc18731b7b160f75121507e02f45f2101e48f0252dcd997e9da1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -1156,7 +1156,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object 0.37.3",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
"windows-link",
|
||||
]
|
||||
@@ -1225,15 +1225,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -1453,9 +1444,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.53"
|
||||
version = "1.2.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
|
||||
checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -1538,7 +1529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"phf 0.12.1",
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3205,15 +3196,6 @@ version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9"
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
dependencies = [
|
||||
"phf 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.5"
|
||||
@@ -3255,13 +3237,13 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"sha2 0.11.0-rc.3",
|
||||
"suppaftp",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -3583,9 +3565,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "findshlibs"
|
||||
@@ -4012,9 +3994,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-iam-v1"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84b431125034e0928e41e8c117bcbc40b0b55b55464b2e964b26e1ffcb15323"
|
||||
checksum = "4887dd50f1e7510e9c91b9581313827120f903c489426e15231f367ee191030c"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -4023,7 +4005,7 @@ dependencies = [
|
||||
"google-cloud-type",
|
||||
"google-cloud-wkt",
|
||||
"lazy_static",
|
||||
"reqwest 0.12.28",
|
||||
"reqwest 0.13.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
@@ -4032,9 +4014,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-longrunning"
|
||||
version = "1.4.0"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0612f4062f42b141b4d050d1a8a2f860e907a548bde28cb82d4fdf0eb346a3"
|
||||
checksum = "cd5ac41700cf35600752386270a744345d550b843b5356feaa75064e97ceb94a"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -4043,7 +4025,7 @@ dependencies = [
|
||||
"google-cloud-rpc",
|
||||
"google-cloud-wkt",
|
||||
"lazy_static",
|
||||
"reqwest 0.12.28",
|
||||
"reqwest 0.13.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
@@ -4052,9 +4034,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-lro"
|
||||
version = "1.2.0"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49747b7b684b804a2d1040c2cdb21238b3d568a41ab9e36c423554509112f61d"
|
||||
checksum = "a52a26fb12ea9b67eab358dbb8dc5e22aac666cab947a10d15699ff95f203b87"
|
||||
dependencies = [
|
||||
"google-cloud-gax",
|
||||
"google-cloud-longrunning",
|
||||
@@ -4301,44 +4283,6 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "heed"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a56c94661ddfb51aa9cdfbf102cfcc340aa69267f95ebccc4af08d7c530d393"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"heed-types",
|
||||
"libc",
|
||||
"lmdb-master-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"serde",
|
||||
"synchronoise",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heed-traits"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.5.2"
|
||||
@@ -4484,9 +4428,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.5"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f471e0a81b2f90ffc0cb2f951ae04da57de8baa46fa99112b062a5173a5088d0"
|
||||
checksum = "b41fb3dc24fe72c2e3a4685eed55917c2fb228851257f4a8f2d985da9443c3e5"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"zeroize",
|
||||
@@ -4575,9 +4519,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.64"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
@@ -5241,9 +5185,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "liblzma-sys"
|
||||
version = "0.4.4"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"
|
||||
checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -5252,9 +5196,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.15"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
@@ -5348,17 +5292,6 @@ version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-master-sys"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "864808e0b19fb6dd3b70ba94ee671b82fce17554cf80aeb0a155c65bb08027df"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "local-ip-address"
|
||||
version = "0.6.9"
|
||||
@@ -5623,9 +5556,9 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
||||
|
||||
[[package]]
|
||||
name = "neli"
|
||||
version = "0.7.3"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e23bebbf3e157c402c4d5ee113233e5e0610cc27453b2f07eefce649c7365dcc"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"byteorder",
|
||||
@@ -5735,9 +5668,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "notify-types"
|
||||
version = "2.0.0"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d"
|
||||
checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
@@ -5931,15 +5867,6 @@ dependencies = [
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.32.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.37.3"
|
||||
@@ -6008,9 +5935,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry"
|
||||
@@ -6423,55 +6350,13 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
|
||||
dependencies = [
|
||||
"phf_shared 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared 0.11.3",
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared 0.11.3",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6564,9 +6449,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0-rc.8"
|
||||
version = "0.11.0-rc.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77089aec8290d0b7bb01b671b091095cf1937670725af4fd73d47249f03b12c0"
|
||||
checksum = "b226d2cc389763951db8869584fd800cbbe2962bf454e2edeb5172b31ee99774"
|
||||
dependencies = [
|
||||
"der 0.8.0-rc.10",
|
||||
"spki 0.8.0-rc.4",
|
||||
@@ -6689,9 +6574,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppmd-rust"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4"
|
||||
checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
|
||||
|
||||
[[package]]
|
||||
name = "pprof"
|
||||
@@ -6794,9 +6679,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.105"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -6954,9 +6839,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "psm"
|
||||
version = "0.1.28"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01"
|
||||
checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d"
|
||||
dependencies = [
|
||||
"ar_archive_writer",
|
||||
"cc",
|
||||
@@ -7079,9 +6964,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.43"
|
||||
version = "1.0.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
|
||||
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -7521,7 +7406,7 @@ dependencies = [
|
||||
"crypto-primes",
|
||||
"digest 0.11.0-rc.5",
|
||||
"pkcs1",
|
||||
"pkcs8 0.11.0-rc.8",
|
||||
"pkcs8 0.11.0-rc.10",
|
||||
"rand_core 0.10.0-rc-3",
|
||||
"sha2 0.11.0-rc.3",
|
||||
"signature 3.0.0-rc.6",
|
||||
@@ -8017,24 +7902,17 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"futures",
|
||||
"heed",
|
||||
"rand 0.10.0-rc.6",
|
||||
"reqwest 0.13.1",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-madmin",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -8623,7 +8501,7 @@ checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.13.0-alpha.2"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?branch=main#26ce04f59c14f130c87b23789d0c3723d9429e41"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=3cdb3fe22fe8a1b7fc3f71ead4beacac2683ba7f#3cdb3fe22fe8a1b7fc3f71ead4beacac2683ba7f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -8797,9 +8675,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sec1"
|
||||
version = "0.8.0-rc.11"
|
||||
version = "0.8.0-rc.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2568531a8ace88b848310caa98fb2115b151ef924d54aa523e659c21b9d32d71"
|
||||
checksum = "7a2400ed44a13193820aa528a19f376c3843141a8ce96ff34b11104cc79763f2"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"hybrid-array",
|
||||
@@ -9421,7 +9299,7 @@ dependencies = [
|
||||
"ed25519-dalek 3.0.0-pre.4",
|
||||
"rand_core 0.10.0-rc-3",
|
||||
"rsa",
|
||||
"sec1 0.8.0-rc.11",
|
||||
"sec1 0.8.0-rc.13",
|
||||
"sha2 0.11.0-rc.3",
|
||||
"signature 3.0.0-rc.6",
|
||||
"ssh-cipher 0.3.0-rc.5",
|
||||
@@ -9674,15 +9552,6 @@ dependencies = [
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synchronoise"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
|
||||
dependencies = [
|
||||
"crossbeam-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.12.6"
|
||||
@@ -10140,9 +10009,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.14.2"
|
||||
version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3"
|
||||
checksum = "27aac809edf60b741e2d7db6367214d078856b8a5bff0087e94ff330fb97b6fc"
|
||||
dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -10388,9 +10257,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
||||
|
||||
[[package]]
|
||||
name = "tz-rs"
|
||||
version = "0.7.1"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14eff19b8dc1ace5bf7e4d920b2628ae3837f422ff42210cb1567cbf68b5accf"
|
||||
checksum = "4fc6c929ffa10fb34f4a3c7e9a73620a83ef2e85e47f9ec3381b8289e6762f42"
|
||||
|
||||
[[package]]
|
||||
name = "tzdb"
|
||||
@@ -11266,18 +11135,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.33"
|
||||
version = "0.8.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
|
||||
checksum = "dafd85c832c1b68bbb4ec0c72c7f6f4fc5179627d2bc7c26b30e4c0cc11e76cc"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.33"
|
||||
version = "0.8.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
|
||||
checksum = "7cb7e4e8436d9db52fbd6625dbf2f45243ab84994a72882ec8227b99e72b439a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -11394,9 +11263,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.15"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94f63c051f4fe3c1509da62131a678643c5b6fbdc9273b2b79d4378ebda003d2"
|
||||
checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
|
||||
@@ -175,12 +175,12 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.8.12" }
|
||||
aws-credential-types = { version = "1.2.11" }
|
||||
aws-sdk-s3 = { version = "1.121.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-types = { version = "1.4.0" }
|
||||
aws-smithy-types = { version = "1.4.1" }
|
||||
base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.2"
|
||||
cfg-if = "1.0.4"
|
||||
clap = { version = "4.5.55", features = ["derive", "env"] }
|
||||
clap = { version = "4.5.56", features = ["derive", "env"] }
|
||||
const-str = { version = "1.0.0", features = ["std", "proc"] }
|
||||
convert_case = "0.10.0"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
@@ -195,7 +195,6 @@ glob = "0.3.3"
|
||||
google-cloud-storage = "1.7.0"
|
||||
google-cloud-auth = "1.5.0"
|
||||
hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
|
||||
heed = { version = "0.22.0" }
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
@@ -227,7 +226,7 @@ rumqttc = { version = "0.25.1" }
|
||||
rustix = { version = "1.1.3", features = ["fs"] }
|
||||
rust-embed = { version = "8.11.0" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
s3s = { version = "0.13.0-alpha.2", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", branch = "main" }
|
||||
s3s = { version = "0.13.0-alpha.2", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3cdb3fe22fe8a1b7fc3f71ead4beacac2683ba7f" }
|
||||
serial_test = "3.3.1"
|
||||
shadow-rs = { version = "1.7.0", default-features = false }
|
||||
siphasher = "1.0.2"
|
||||
|
||||
@@ -36,7 +36,6 @@ serde_json.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
url.workspace = true
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes.workspace = true
|
||||
@@ -52,6 +51,7 @@ base64 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
suppaftp.workspace = true
|
||||
rcgen.workspace = true
|
||||
anyhow.workspace = true
|
||||
|
||||
229
crates/e2e_test/src/checksum_upload_test.rs
Normal file
229
crates/e2e_test/src/checksum_upload_test.rs
Normal file
@@ -0,0 +1,229 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! E2E tests for PutObject and MultipartUpload with checksums (Content-MD5, x-amz-checksum-*).
|
||||
//! Verifies that uploads with Content-MD5 and x-amz-checksum-sha256 succeed and content is correct.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use base64::Engine;
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
env.create_s3_client()
|
||||
}
|
||||
|
||||
async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
match client.create_bucket().bucket(bucket).send().await {
|
||||
Ok(_) => {
|
||||
info!("Bucket {} created successfully", bucket);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
if e.to_string().contains("BucketAlreadyOwnedByYou") || e.to_string().contains("BucketAlreadyExists") {
|
||||
info!("Bucket {} already exists", bucket);
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Box::new(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn content_md5_base64(body: &[u8]) -> String {
|
||||
let digest = md5::compute(body);
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
fn checksum_sha256_base64(body: &[u8]) -> String {
|
||||
let digest = Sha256::digest(body);
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_content_md5() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with Content-MD5");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-md5";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-md5.txt";
|
||||
let content = b"Hello world with Content-MD5 checksum";
|
||||
let content_md5 = content_md5_base64(content);
|
||||
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.content_md5(&content_md5)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "PutObject with Content-MD5 failed: {:?}", result.err());
|
||||
|
||||
let get_result = client.get_object().bucket(bucket).key(key).send().await;
|
||||
assert!(get_result.is_ok(), "GetObject failed: {:?}", get_result.err());
|
||||
let body_bytes = get_result.unwrap().body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(body_bytes.as_ref(), content, "GetObject body must match uploaded content");
|
||||
info!("PASSED: PutObject with Content-MD5 and GetObject content match");
|
||||
}
|
||||
|
||||
/// PutObject with x-amz-checksum-sha256: upload succeeds and GetObject returns same content.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_checksum_sha256() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with x-amz-checksum-sha256");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-sha256";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-sha256.txt";
|
||||
let content = b"Hello world with x-amz-checksum-sha256";
|
||||
let checksum = checksum_sha256_base64(content);
|
||||
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.checksum_sha256(&checksum)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "PutObject with checksum_sha256 failed: {:?}", result.err());
|
||||
|
||||
let get_result = client.get_object().bucket(bucket).key(key).send().await;
|
||||
assert!(get_result.is_ok(), "GetObject failed: {:?}", get_result.err());
|
||||
let body_bytes = get_result.unwrap().body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(body_bytes.as_ref(), content, "GetObject body must match uploaded content");
|
||||
info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
|
||||
}
|
||||
|
||||
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
||||
/// Uses part size >= 5MB (server minimum) for two parts.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_upload_with_checksum() {
|
||||
init_logging();
|
||||
info!("TEST: MultipartUpload with checksum (checksum_sha256 on parts)");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-multipart-checksum";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "multipart-with-checksum.bin";
|
||||
const PART_SIZE: usize = 6 * 1024 * 1024; // 6 MB per part (>= 5MB minimum)
|
||||
let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 256) as u8).collect();
|
||||
let part2: Vec<u8> = (0..PART_SIZE).map(|i| ((i + 1) % 256) as u8).collect();
|
||||
let full_content: Vec<u8> = part1.iter().chain(part2.iter()).copied().collect();
|
||||
|
||||
let create_result = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to create multipart upload");
|
||||
|
||||
let upload_id = create_result.upload_id().expect("No upload_id").to_string();
|
||||
|
||||
let checksum1 = checksum_sha256_base64(&part1);
|
||||
let upload1 = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1.clone()))
|
||||
.checksum_sha256(&checksum1)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to upload part 1");
|
||||
|
||||
let etag1 = upload1.e_tag().expect("No etag part 1").to_string();
|
||||
let checksum_sha256_1 = upload1.checksum_sha256().map(|s| s.to_string());
|
||||
|
||||
let checksum2 = checksum_sha256_base64(&part2);
|
||||
let upload2 = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.checksum_sha256(&checksum2)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to upload part 2");
|
||||
|
||||
let etag2 = upload2.e_tag().expect("No etag part 2").to_string();
|
||||
let checksum_sha256_2 = upload2.checksum_sha256().map(|s| s.to_string());
|
||||
|
||||
let mut part1_builder = CompletedPart::builder().part_number(1).e_tag(etag1);
|
||||
if let Some(ref cs) = checksum_sha256_1 {
|
||||
part1_builder = part1_builder.checksum_sha256(cs);
|
||||
}
|
||||
let mut part2_builder = CompletedPart::builder().part_number(2).e_tag(etag2);
|
||||
if let Some(ref cs) = checksum_sha256_2 {
|
||||
part2_builder = part2_builder.checksum_sha256(cs);
|
||||
}
|
||||
|
||||
let completed_parts = vec![part1_builder.build(), part2_builder.build()];
|
||||
let completed_upload = CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build();
|
||||
|
||||
let complete_result = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(complete_result.is_ok(), "CompleteMultipartUpload failed: {:?}", complete_result.err());
|
||||
|
||||
let get_result = client.get_object().bucket(bucket).key(key).send().await;
|
||||
assert!(get_result.is_ok(), "GetObject failed: {:?}", get_result.err());
|
||||
let body_bytes = get_result.unwrap().body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body_bytes.as_ref(),
|
||||
full_content.as_slice(),
|
||||
"GetObject body must match concatenated parts"
|
||||
);
|
||||
info!("PASSED: MultipartUpload with checksum and GetObject content match");
|
||||
}
|
||||
}
|
||||
@@ -60,3 +60,7 @@ mod protocols;
|
||||
// Object Lock tests
|
||||
#[cfg(test)]
|
||||
mod object_lock;
|
||||
|
||||
// PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
|
||||
#[cfg(test)]
|
||||
mod checksum_upload_test;
|
||||
|
||||
@@ -5730,16 +5730,11 @@ impl StorageAPI for SetDisks {
|
||||
return Err(Error::other("checksum type not found"));
|
||||
};
|
||||
|
||||
if let Some(want) = &opts.want_checksum
|
||||
&& !want
|
||||
.checksum_type
|
||||
.is(rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct))
|
||||
checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct);
|
||||
if let Some(want) = opts.want_checksum.as_ref()
|
||||
&& !want.checksum_type.is(checksum_type)
|
||||
{
|
||||
return Err(Error::other(format!(
|
||||
"checksum type mismatch, got {:?}, want {:?}",
|
||||
want,
|
||||
rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct)
|
||||
)));
|
||||
return Err(Error::other(format!("checksum type mismatch, got {:?}, want {:?}", want, checksum_type)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,30 +30,22 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
anyhow = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
heed = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{DiskInfo, IOStats};
|
||||
use crate::os::{DiskInfo, IOStats};
|
||||
use rustix::fs::statfs;
|
||||
use std::fs::File;
|
||||
use std::io::{self, BufRead, Error, ErrorKind};
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{DiskInfo, IOStats};
|
||||
use crate::os::{DiskInfo, IOStats};
|
||||
use rustix::fs::{StatVfs, statvfs};
|
||||
use std::io::Error;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{DiskInfo, IOStats};
|
||||
#![allow(unsafe_code)] // TODO: audit unsafe code
|
||||
|
||||
use crate::os::{DiskInfo, IOStats};
|
||||
use std::io::Error;
|
||||
use std::path::Path;
|
||||
use windows::Win32::Foundation::MAX_PATH;
|
||||
@@ -20,17 +22,15 @@ use windows::Win32::Storage::FileSystem::{GetDiskFreeSpaceExW, GetDiskFreeSpaceW
|
||||
|
||||
/// Returns total and free bytes available in a directory, e.g. `C:\`.
|
||||
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
|
||||
let path_wide = p
|
||||
.as_ref()
|
||||
.to_string_lossy()
|
||||
.encode_utf16()
|
||||
.chain(std::iter::once(0))
|
||||
.collect::<Vec<u16>>();
|
||||
let path_wide = to_wide_path(p.as_ref());
|
||||
|
||||
let mut free_bytes_available = 0u64;
|
||||
let mut total_number_of_bytes = 0u64;
|
||||
let mut total_number_of_free_bytes = 0u64;
|
||||
|
||||
// SAFETY:
|
||||
// 1. `path_wide` is a valid null-terminated UTF-16 string.
|
||||
// 2. Pointers to `u64` variables are valid and point to initialized stack memory.
|
||||
unsafe {
|
||||
GetDiskFreeSpaceExW(
|
||||
windows::core::PCWSTR::from_raw(path_wide.as_ptr()),
|
||||
@@ -56,6 +56,9 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
|
||||
let mut number_of_free_clusters = 0u32;
|
||||
let mut total_number_of_clusters = 0u32;
|
||||
|
||||
// SAFETY:
|
||||
// 1. `path_wide` is a valid null-terminated UTF-16 string.
|
||||
// 2. Pointers to `u32` variables are valid and point to initialized stack memory.
|
||||
unsafe {
|
||||
GetDiskFreeSpaceW(
|
||||
windows::core::PCWSTR::from_raw(path_wide.as_ptr()),
|
||||
@@ -87,6 +90,10 @@ fn get_windows_fs_type(p: &[u16]) -> std::io::Result<String> {
|
||||
let mut volume_name_buffer = [0u16; MAX_PATH as usize];
|
||||
let mut file_system_name_buffer = [0u16; MAX_PATH as usize];
|
||||
|
||||
// SAFETY:
|
||||
// 1. `path` is a valid null-terminated UTF-16 string (volume root path).
|
||||
// 2. Buffers are allocated with `MAX_PATH` size, which is sufficient for standard Windows paths.
|
||||
// 3. Pointers to output variables are valid.
|
||||
unsafe {
|
||||
GetVolumeInformationW(
|
||||
windows::core::PCWSTR::from_raw(path.as_ptr()),
|
||||
@@ -105,6 +112,11 @@ fn get_windows_fs_type(p: &[u16]) -> std::io::Result<String> {
|
||||
fn get_volume_name(v: &[u16]) -> std::io::Result<Vec<u16>> {
|
||||
let mut volume_name_buffer = [0u16; MAX_PATH as usize];
|
||||
|
||||
// SAFETY:
|
||||
// 1. `v` is a valid null-terminated UTF-16 string.
|
||||
// 2. `volume_name_buffer` is allocated with `MAX_PATH` size.
|
||||
// 3. `GetVolumePathNameW` writes to the buffer and respects the buffer size (implicitly MAX_PATH for this API context usually, though explicit length param isn't present, it expects a buffer large enough).
|
||||
// Note: GetVolumePathNameW documentation says "The buffer should be large enough to hold the path". MAX_PATH is generally safe for volume roots.
|
||||
unsafe {
|
||||
GetVolumePathNameW(windows::core::PCWSTR::from_raw(v.as_ptr()), &mut volume_name_buffer)
|
||||
.map_err(|e| Error::from_raw_os_error(e.code().0 as i32))?;
|
||||
@@ -122,9 +134,16 @@ fn utf16_to_string(v: &[u16]) -> String {
|
||||
String::from_utf16_lossy(&v[..len])
|
||||
}
|
||||
|
||||
fn to_wide_path(path: &Path) -> Vec<u16> {
|
||||
path.as_os_str().encode_wide().chain(std::iter::once(0)).collect()
|
||||
}
|
||||
|
||||
// Helper trait to access encode_wide which is only available on Windows
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
|
||||
pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
|
||||
let path1_wide: Vec<u16> = disk1.encode_utf16().chain(std::iter::once(0)).collect();
|
||||
let path2_wide: Vec<u16> = disk2.encode_utf16().chain(std::iter::once(0)).collect();
|
||||
let path1_wide = to_wide_path(Path::new(disk1));
|
||||
let path2_wide = to_wide_path(Path::new(disk2));
|
||||
|
||||
let volume1 = get_volume_name(&path1_wide)?;
|
||||
let volume2 = get_volume_name(&path2_wide)?;
|
||||
|
||||
@@ -34,7 +34,7 @@ use rustfs_ecstore::bucket::metadata_sys;
|
||||
use rustfs_ecstore::bucket::target::BucketTarget;
|
||||
use rustfs_ecstore::bucket::utils::is_valid_object_prefix;
|
||||
use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use rustfs_ecstore::data_usage::{compute_bucket_usage, load_data_usage_from_backend, store_data_usage_in_backend};
|
||||
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use rustfs_ecstore::global::global_rustfs_port;
|
||||
use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
@@ -520,40 +520,6 @@ impl Operation for DataUsageInfoHandler {
|
||||
s3_error!(InternalError, "load_data_usage_from_backend failed")
|
||||
})?;
|
||||
|
||||
let last_update_age = info.last_update.and_then(|ts| ts.elapsed().ok());
|
||||
let data_missing = info.objects_total_count == 0 && info.buckets_count == 0;
|
||||
let stale = last_update_age
|
||||
.map(|elapsed| elapsed > std::time::Duration::from_secs(300))
|
||||
.unwrap_or(true);
|
||||
|
||||
if data_missing {
|
||||
info!("No data usage statistics found, attempting real-time collection");
|
||||
|
||||
if let Err(e) = collect_realtime_data_usage(&mut info, store.clone()).await {
|
||||
warn!("Failed to collect real-time data usage: {}", e);
|
||||
} else if let Err(e) = store_data_usage_in_backend(info.clone(), store.clone()).await {
|
||||
warn!("Failed to persist refreshed data usage: {}", e);
|
||||
}
|
||||
} else if stale {
|
||||
info!(
|
||||
"Data usage statistics are stale (last update {:?} ago), refreshing asynchronously",
|
||||
last_update_age
|
||||
);
|
||||
|
||||
let mut info_for_refresh = info.clone();
|
||||
let store_for_refresh = store.clone();
|
||||
spawn(async move {
|
||||
if let Err(e) = collect_realtime_data_usage(&mut info_for_refresh, store_for_refresh.clone()).await {
|
||||
warn!("Background data usage refresh failed: {}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = store_data_usage_in_backend(info_for_refresh, store_for_refresh).await {
|
||||
warn!("Background data usage persistence failed: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let sinfo = store.storage_info().await;
|
||||
|
||||
// 🔧 Use the fixed capacity calculation function (built-in deduplication)
|
||||
@@ -1310,62 +1276,6 @@ impl Operation for RemoveRemoteTargetHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Real-time data collection function
|
||||
async fn collect_realtime_data_usage(
|
||||
info: &mut rustfs_common::data_usage::DataUsageInfo,
|
||||
store: Arc<rustfs_ecstore::store::ECStore>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
// Get bucket list and collect basic statistics
|
||||
let buckets = store.list_bucket(&BucketOptions::default()).await?;
|
||||
|
||||
info.buckets_count = buckets.len() as u64;
|
||||
info.last_update = Some(std::time::SystemTime::now());
|
||||
info.buckets_usage.clear();
|
||||
info.bucket_sizes.clear();
|
||||
info.disk_usage_status.clear();
|
||||
info.objects_total_count = 0;
|
||||
info.objects_total_size = 0;
|
||||
info.versions_total_count = 0;
|
||||
info.delete_markers_total_count = 0;
|
||||
|
||||
let mut total_objects = 0u64;
|
||||
let mut total_versions = 0u64;
|
||||
let mut total_size = 0u64;
|
||||
let mut total_delete_markers = 0u64;
|
||||
|
||||
// For each bucket, try to get object count
|
||||
for bucket_info in buckets {
|
||||
let bucket_name = &bucket_info.name;
|
||||
|
||||
// Skip system buckets
|
||||
if bucket_name.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
|
||||
match compute_bucket_usage(store.clone(), bucket_name).await {
|
||||
Ok(bucket_usage) => {
|
||||
total_objects = total_objects.saturating_add(bucket_usage.objects_count);
|
||||
total_versions = total_versions.saturating_add(bucket_usage.versions_count);
|
||||
total_size = total_size.saturating_add(bucket_usage.size);
|
||||
total_delete_markers = total_delete_markers.saturating_add(bucket_usage.delete_markers_count);
|
||||
|
||||
info.buckets_usage.insert(bucket_name.clone(), bucket_usage.clone());
|
||||
info.bucket_sizes.insert(bucket_name.clone(), bucket_usage.size);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to compute bucket usage for {}: {}", bucket_name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info.objects_total_count = total_objects;
|
||||
info.objects_total_size = total_size;
|
||||
info.versions_total_count = total_versions;
|
||||
info.delete_markers_total_count = total_delete_markers;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct ProfileHandler {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ProfileHandler {
|
||||
|
||||
@@ -347,24 +347,6 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
init_heal_manager(heal_storage, None).await?;
|
||||
|
||||
init_data_scanner(ctx.clone(), store.clone()).await;
|
||||
|
||||
// if enable_heal {
|
||||
// // Initialize heal manager with channel processor
|
||||
// let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
|
||||
// let heal_manager = init_heal_manager(heal_storage, None).await?;
|
||||
|
||||
// if enable_scanner {
|
||||
// info!(target: "rustfs::main::run","Starting scanner with heal manager...");
|
||||
// let scanner = Scanner::new(Some(ScannerConfig::default()), Some(heal_manager));
|
||||
// scanner.start().await?;
|
||||
// } else {
|
||||
// info!(target: "rustfs::main::run","Scanner disabled, but heal manager is initialized and available");
|
||||
// }
|
||||
// } else if enable_scanner {
|
||||
// info!("Starting scanner without heal manager...");
|
||||
// let scanner = Scanner::new(Some(ScannerConfig::default()), None);
|
||||
// scanner.start().await?;
|
||||
// }
|
||||
} else {
|
||||
info!(target: "rustfs::main::run","Both scanner and heal are disabled, skipping AHM service initialization");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user