mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-17 17:40:38 +00:00
Compare commits
1 Commits
1.0.0-alph
...
multipartc
| Author | SHA1 | Date | |
|---|---|---|---|
|
d2a83505fa |
9
.github/actions/setup/action.yml
vendored
9
.github/actions/setup/action.yml
vendored
@@ -52,19 +52,24 @@ runs:
|
||||
sudo apt-get install -y \
|
||||
musl-tools \
|
||||
build-essential \
|
||||
lld \
|
||||
libdbus-1-dev \
|
||||
libwayland-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
libxdo-dev \
|
||||
pkg-config \
|
||||
libssl-dev
|
||||
|
||||
- name: Install protoc
|
||||
uses: arduino/setup-protoc@v3
|
||||
with:
|
||||
version: "33.1"
|
||||
version: "31.1"
|
||||
repo-token: ${{ inputs.github-token }}
|
||||
|
||||
- name: Install flatc
|
||||
uses: Nugine/setup-flatc@v1
|
||||
with:
|
||||
version: "25.9.23"
|
||||
version: "25.2.10"
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
12
.github/dependabot.yml
vendored
12
.github/dependabot.yml
vendored
@@ -22,18 +22,8 @@ updates:
|
||||
- package-ecosystem: "cargo" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
timezone: "Asia/Shanghai"
|
||||
time: "08:00"
|
||||
interval: "monthly"
|
||||
groups:
|
||||
s3s:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
patterns:
|
||||
- "s3s"
|
||||
- "s3s-*"
|
||||
dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
186
Cargo.lock
generated
186
Cargo.lock
generated
@@ -163,22 +163,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
version = "3.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -628,9 +628,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.15.0"
|
||||
version = "1.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5932a7d9d28b0d2ea34c6b3779d35e3dd6f6345317c34e73438c4f1f29144151"
|
||||
checksum = "879b6c89592deb404ba4dc0ae6b58ffd1795c78991cbb5b8bc441c48a070440d"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
@@ -638,9 +638,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.33.0"
|
||||
version = "0.32.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1826f2e4cfc2cd19ee53c42fbf68e2f81ec21108e0b7ecf6a71cf062137360fc"
|
||||
checksum = "107a4e9d9cab9963e04e84bb8dee0e25f2a987f9a8bad5ed054abd439caa8f8c"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
@@ -754,9 +754,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.92.0"
|
||||
version = "1.91.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0c7808adcff8333eaa76a849e6de926c6ac1a1268b9fd6afe32de9c29ef29d2"
|
||||
checksum = "8f8090151d4d1e971269957b10dbf287bba551ab812e591ce0516b1c73b75d27"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -882,7 +882,7 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.32",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-rustls 0.24.2",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
@@ -1017,9 +1017,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.7"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425"
|
||||
checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"bytes",
|
||||
@@ -1028,7 +1028,7 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit 0.8.4",
|
||||
@@ -1069,9 +1069,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "axum-extra"
|
||||
version = "0.12.2"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbfe9f610fe4e99cf0cfcd03ccf8c63c28c616fe714d80475ef731f3b13dd21b"
|
||||
checksum = "5136e6c5e7e7978fe23e9876fb924af2c0f84c72127ac6ac17e7c46f457d362c"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"axum-core",
|
||||
@@ -1090,16 +1090,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "axum-server"
|
||||
version = "0.7.3"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1ab4a3ec9ea8a657c72d99a03a824af695bd0fb5ec639ccbd9cd3543b41a5f9"
|
||||
checksum = "495c05f60d6df0093e8fb6e74aa5846a0ad06abaf96d76166283720bf740f8ab"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
"fs-err",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"rustls 0.23.35",
|
||||
@@ -1364,9 +1364,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.0"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
||||
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -1481,9 +1481,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.46"
|
||||
version = "1.2.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36"
|
||||
checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -1871,6 +1871,15 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc64fast-nvme"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3"
|
||||
dependencies = [
|
||||
"crc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.7.0"
|
||||
@@ -3368,9 +3377,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.5"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
|
||||
checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
|
||||
|
||||
[[package]]
|
||||
name = "findshlibs"
|
||||
@@ -3474,9 +3483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fs-err"
|
||||
version = "3.2.0"
|
||||
version = "3.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62d91fd049c123429b018c47887d3f75a265540dd3c30ba9cb7bae9197edb03a"
|
||||
checksum = "6ad492b2cf1d89d568a43508ab24f98501fe03f2f31c01e1d0fe7366a71745d2"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"tokio",
|
||||
@@ -3588,9 +3597,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -3647,9 +3656,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-auth"
|
||||
version = "1.2.0"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1cc977b20996b87e207b0a004ea34aa5f0f8692c44a1ca8c8802a08f553bf79c"
|
||||
checksum = "5628e0c17140a50dd4d75d37465bf190d26a6c67909519c2e3cf87a9e45d5cf6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -3670,9 +3679,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax"
|
||||
version = "1.3.1"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c26e6f1be47e93e5360a77e67e4e996a2d838b1924ffe0763bcb21d47be68b"
|
||||
checksum = "3c5aa07295f49565ee1ae52e0799e42bd67284396e042734f078b8737a816047"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -3690,9 +3699,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax-internal"
|
||||
version = "0.7.5"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69168fd1f81869bb8682883d27c56e3e499840d45b27b884b289ec0d5f2b442a"
|
||||
checksum = "6dbd41e77921bbf75ed32acc8a648f087b3227ca88c62ddd9b37b43230c91554"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"google-cloud-auth",
|
||||
@@ -3701,7 +3710,7 @@ dependencies = [
|
||||
"google-cloud-wkt",
|
||||
"http 1.3.1",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"opentelemetry-semantic-conventions",
|
||||
"percent-encoding",
|
||||
"prost 0.14.1",
|
||||
@@ -3715,7 +3724,6 @@ dependencies = [
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tower",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -3761,9 +3769,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-lro"
|
||||
version = "1.1.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5259a172f712809460ad10336b322caf0cd37cf1469aecc950bf6bf0026fbd7"
|
||||
checksum = "a93fd80965a47da86e1d21c90dcd10858b859e6c702c5c6e1b383686692ae73e"
|
||||
dependencies = [
|
||||
"google-cloud-gax",
|
||||
"google-cloud-longrunning",
|
||||
@@ -3788,9 +3796,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-storage"
|
||||
version = "1.4.0"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "931b69ac5996d0216e74e22e1843e025bef605ba8c062003d40c1565d90594b4"
|
||||
checksum = "12e6a4d24384f8ffae6d295d55095b25d21c9099855c5a96a0edf6777178b35b"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -3808,7 +3816,7 @@ dependencies = [
|
||||
"google-cloud-wkt",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"lazy_static",
|
||||
"md5",
|
||||
"mime",
|
||||
@@ -3823,7 +3831,6 @@ dependencies = [
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.17",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tracing",
|
||||
"uuid",
|
||||
@@ -4206,9 +4213,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.8.1"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
|
||||
checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -4250,7 +4257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
|
||||
dependencies = [
|
||||
"http 1.3.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"log",
|
||||
"rustls 0.23.35",
|
||||
@@ -4268,7 +4275,7 @@ version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
|
||||
dependencies = [
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
@@ -4277,9 +4284,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.18"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56"
|
||||
checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -4288,7 +4295,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"ipnet",
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
@@ -4476,9 +4483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "inferno"
|
||||
version = "0.12.4"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d35223c50fdd26419a4ccea2c73be68bd2b29a3d7d6123ffe101c17f4c20a52a"
|
||||
checksum = "e96d2465363ed2d81857759fc864cf6bb7997f79327aec028d65bd7989393685"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"clap",
|
||||
@@ -4491,7 +4498,7 @@ dependencies = [
|
||||
"log",
|
||||
"num-format",
|
||||
"once_cell",
|
||||
"quick-xml 0.38.4",
|
||||
"quick-xml 0.37.5",
|
||||
"rgb",
|
||||
"str_stack",
|
||||
]
|
||||
@@ -4664,7 +4671,7 @@ dependencies = [
|
||||
"p384",
|
||||
"pem",
|
||||
"rand 0.8.5",
|
||||
"rsa 0.9.9",
|
||||
"rsa 0.9.8",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
@@ -5302,9 +5309,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint-dig"
|
||||
version = "0.8.6"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7"
|
||||
checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libm",
|
||||
@@ -6121,14 +6128,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pprof_util"
|
||||
version = "0.8.1"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4429d44e5e2c8a69399fc0070379201eed018e3df61e04eb7432811df073c224"
|
||||
checksum = "f9aba4251d95ac86f14c33e688d57a9344bfcff29e9b0c5a063fc66b5facc8a1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"backtrace",
|
||||
"flate2",
|
||||
"inferno 0.12.4",
|
||||
"inferno 0.12.3",
|
||||
"num",
|
||||
"paste",
|
||||
"prost 0.13.5",
|
||||
@@ -6668,7 +6675,7 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
@@ -6800,9 +6807,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.9.9"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88"
|
||||
checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b"
|
||||
dependencies = [
|
||||
"const-oid 0.9.6",
|
||||
"digest 0.10.7",
|
||||
@@ -6938,7 +6945,7 @@ dependencies = [
|
||||
"hex-simd",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"jemalloc_pprof",
|
||||
"libsystemd",
|
||||
@@ -7146,7 +7153,7 @@ dependencies = [
|
||||
"hex-simd",
|
||||
"hmac 0.13.0-rc.3",
|
||||
"http 1.3.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
@@ -7207,7 +7214,7 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"crc32fast",
|
||||
"criterion",
|
||||
"lazy_static",
|
||||
"regex",
|
||||
@@ -7259,6 +7266,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"md5",
|
||||
"moka",
|
||||
"once_cell",
|
||||
"rand 0.10.0-rc.5",
|
||||
"reqwest",
|
||||
"serde",
|
||||
@@ -7304,7 +7312,7 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"humantime",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
@@ -7427,7 +7435,9 @@ dependencies = [
|
||||
"aes-gcm",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"crc32c",
|
||||
"crc32fast",
|
||||
"crc64fast-nvme",
|
||||
"faster-hex",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
@@ -7497,7 +7507,7 @@ dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"http 1.3.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde_urlencoded",
|
||||
@@ -7534,7 +7544,7 @@ dependencies = [
|
||||
"brotli 8.0.2",
|
||||
"bytes",
|
||||
"convert_case",
|
||||
"crc-fast",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hashbrown 0.16.0",
|
||||
@@ -7542,7 +7552,7 @@ dependencies = [
|
||||
"highway",
|
||||
"hmac 0.13.0-rc.3",
|
||||
"http 1.3.1",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"libc",
|
||||
"local-ip-address",
|
||||
"lz4",
|
||||
@@ -7778,7 +7788,7 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.12.0-rc.3"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=ba9f902#ba9f902df5a0b68dc9b0eeed4f06625fc0633d20"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=1ab064b#1ab064b6e8cfe0c36a2f763e9f6aa14b56592220"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"async-trait",
|
||||
@@ -7789,7 +7799,9 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"chrono",
|
||||
"const-str",
|
||||
"crc-fast",
|
||||
"crc32c",
|
||||
"crc32fast",
|
||||
"crc64fast-nvme",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0-rc.3",
|
||||
@@ -7797,7 +7809,7 @@ dependencies = [
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"itoa",
|
||||
"md-5 0.11.0-rc.3",
|
||||
"memchr",
|
||||
@@ -8110,9 +8122,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.16.0"
|
||||
version = "3.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10574371d41b0d9b2cff89418eda27da52bcaff2cc8741db26382a77c29131f1"
|
||||
checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
@@ -8129,9 +8141,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.16.0"
|
||||
version = "3.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08a72d8216842fdd57820dc78d840bef99248e35fb2554ff923319e60f2d686b"
|
||||
checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955"
|
||||
dependencies = [
|
||||
"darling 0.21.3",
|
||||
"proc-macro2",
|
||||
@@ -9166,7 +9178,7 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper 1.7.0",
|
||||
"hyper-timeout",
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
@@ -9777,9 +9789,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wildmatch"
|
||||
version = "2.6.1"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29333c3ea1ba8b17211763463ff24ee84e41c78224c16b001cd907e663a38c68"
|
||||
checksum = "2d654e41fe05169e03e27b97e0c23716535da037c1652a31fd99c6b2fad84059"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -9920,13 +9932,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.6.1"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
|
||||
checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
"windows-strings 0.5.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result 0.3.4",
|
||||
"windows-strings 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
23
Cargo.toml
23
Cargo.toml
@@ -97,15 +97,15 @@ async-channel = "2.5.0"
|
||||
async-compression = { version = "0.4.19" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.89"
|
||||
axum = "0.8.7"
|
||||
axum-extra = "0.12.2"
|
||||
axum-server = { version = "0.7.3", features = ["tls-rustls-no-provider"], default-features = false }
|
||||
axum = "0.8.6"
|
||||
axum-extra = "0.12.1"
|
||||
axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false }
|
||||
futures = "0.3.31"
|
||||
futures-core = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
|
||||
hyper = { version = "1.7.0", features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] }
|
||||
hyper-util = { version = "0.1.18", features = ["tokio", "server-auto", "server-graceful"] }
|
||||
hyper-util = { version = "0.1.17", features = ["tokio", "server-auto", "server-graceful"] }
|
||||
http = "1.3.1"
|
||||
http-body = "1.0.1"
|
||||
reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
|
||||
@@ -122,7 +122,7 @@ tower = { version = "0.5.2", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.6", features = ["cors"] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
bytes = { version = "1.11.0", features = ["serde"] }
|
||||
bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.2.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.9.23"
|
||||
@@ -143,6 +143,9 @@ argon2 = { version = "0.6.0-rc.2", features = ["std"] }
|
||||
blake3 = { version = "1.8.2" }
|
||||
chacha20poly1305 = { version = "0.11.0-rc.2" }
|
||||
crc-fast = "1.6.0"
|
||||
crc32c = "0.6.8"
|
||||
crc32fast = "1.5.0"
|
||||
crc64fast-nvme = "1.2.0"
|
||||
hmac = { version = "0.13.0-rc.3" }
|
||||
jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] }
|
||||
pbkdf2 = "0.13.0-rc.2"
|
||||
@@ -185,8 +188,8 @@ faster-hex = "0.10.0"
|
||||
flate2 = "1.1.5"
|
||||
flexi_logger = { version = "0.31.7", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
|
||||
glob = "0.3.3"
|
||||
google-cloud-storage = "1.4.0"
|
||||
google-cloud-auth = "1.2.0"
|
||||
google-cloud-storage = "1.3.0"
|
||||
google-cloud-auth = "1.1.1"
|
||||
hashbrown = { version = "0.16.0", features = ["serde", "rayon"] }
|
||||
heed = { version = "0.22.0" }
|
||||
hex-simd = "0.8.0"
|
||||
@@ -222,7 +225,7 @@ regex = { version = "1.12.2" }
|
||||
rumqttc = { version = "0.25.0" }
|
||||
rust-embed = { version = "8.9.0" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "ba9f902", version = "0.12.0-rc.3", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "1ab064b", version = "0.12.0-rc.3", features = ["minio"] }
|
||||
serial_test = "3.2.0"
|
||||
shadow-rs = { version = "1.4.0", default-features = false }
|
||||
siphasher = "1.0.1"
|
||||
@@ -249,7 +252,7 @@ urlencoding = "2.1.3"
|
||||
uuid = { version = "1.18.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
vaultrs = { version = "0.7.4" }
|
||||
walkdir = "2.5.0"
|
||||
wildmatch = { version = "2.6.1", features = ["serde"] }
|
||||
wildmatch = { version = "2.6.0", features = ["serde"] }
|
||||
winapi = { version = "0.3.9" }
|
||||
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
|
||||
zip = "6.0.0"
|
||||
|
||||
@@ -66,8 +66,8 @@ COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
RUN addgroup -g 1000 -S rustfs && \
|
||||
adduser -u 1000 -G rustfs -S rustfs -D && \
|
||||
mkdir -p /data /logs && \
|
||||
chown -R rustfs:rustfs /data /logs && \
|
||||
chmod 0750 /data /logs
|
||||
@@ -82,8 +82,9 @@ ENV RUSTFS_ADDRESS=":9000" \
|
||||
RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
|
||||
RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
RUSTFS_OBS_LOG_DIRECTORY="/logs"
|
||||
|
||||
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
|
||||
RUSTFS_SINKS_FILE_PATH="/logs"
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
VOLUME ["/data", "/logs"]
|
||||
|
||||
@@ -167,6 +167,7 @@ ENV RUSTFS_ADDRESS=":9000" \
|
||||
RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
|
||||
RUSTFS_SINKS_FILE_PATH="/logs" \
|
||||
RUSTFS_USERNAME="rustfs" \
|
||||
RUSTFS_GROUPNAME="rustfs" \
|
||||
RUSTFS_UID="1000" \
|
||||
|
||||
25
README.md
25
README.md
@@ -1,6 +1,6 @@
|
||||
[](https://rustfs.com)
|
||||
|
||||
<p align="center">RustFS is a high-performance, distributed object storage system built in Rust.</p>
|
||||
<p align="center">RustFS is a high-performance distributed object storage software built using Rust</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
|
||||
@@ -29,13 +29,13 @@ English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简
|
||||
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
|
||||
</p>
|
||||
|
||||
RustFS is a high-performance, distributed object storage system built in Rust., one of the most popular languages
|
||||
worldwide. RustFS combines the simplicity of MinIO with the memory safety and performance of Rust., S3 compatibility, open-source nature,
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
|
||||
worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
|
||||
support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
|
||||
comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
|
||||
RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
|
||||
> ⚠️ **Current Status: Beta / Technical Preview. Not yet recommended for critical production workloads.**
|
||||
> ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**
|
||||
|
||||
## Features
|
||||
|
||||
@@ -65,8 +65,8 @@ Stress test server parameters
|
||||
|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
|
||||
| Powerful Console | Simple and useless Console |
|
||||
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
|
||||
| Guaranteed Data Sovereignty: No telemetry or unauthorized data egress | Reporting logs to other third countries may violate national security laws |
|
||||
| Permissive Apache 2.0 License | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
|
||||
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
|
||||
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices |
|
||||
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
|
||||
@@ -84,20 +84,15 @@ To get started with RustFS, follow these steps:
|
||||
|
||||
2. **Docker Quick Start (Option 2)**
|
||||
|
||||
RustFS container run as non-root user `rustfs` with id `1000`, if you run docker with `-v` to mount host directory into docker container, please make sure the owner of host directory has been changed to `1000`, otherwise you will encounter permission denied error.
|
||||
|
||||
```bash
|
||||
# create data and logs directories
|
||||
mkdir -p data logs
|
||||
|
||||
# change the owner of those two ditectories
|
||||
chown -R 10001:10001 data logs
|
||||
# using latest alpha version
|
||||
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:alpha
|
||||
|
||||
# using latest version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.68
|
||||
# Specific version
|
||||
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.45
|
||||
```
|
||||
|
||||
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under
|
||||
|
||||
@@ -49,9 +49,8 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
/// execute erasure set heal with resume
|
||||
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
|
||||
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
|
||||
info!("Starting erasure set heal");
|
||||
info!("Starting erasure set heal for {} buckets on set disk {}", buckets.len(), set_disk_id);
|
||||
|
||||
// 1. generate or get task id
|
||||
let task_id = self.get_or_create_task_id(set_disk_id).await?;
|
||||
@@ -232,7 +231,6 @@ impl ErasureSetHealer {
|
||||
|
||||
/// heal single bucket with resume
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, _skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
async fn heal_bucket_with_resume(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -245,7 +243,7 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
) -> Result<()> {
|
||||
info!(target: "rustfs:ahm:heal_bucket_with_resume" ,"Starting heal for bucket from object index {}", current_object_index);
|
||||
info!(target: "rustfs:ahm:heal_bucket_with_resume" ,"Starting heal for bucket: {} from object index {}", bucket, current_object_index);
|
||||
|
||||
// 1. get bucket info
|
||||
let _bucket_info = match self.storage.get_bucket_info(bucket).await? {
|
||||
@@ -275,9 +273,7 @@ impl ErasureSetHealer {
|
||||
let object_exists = match self.storage.object_exists(bucket, object).await {
|
||||
Ok(exists) => exists,
|
||||
Err(e) => {
|
||||
warn!("Failed to check existence of {}/{}: {}, marking as failed", bucket, object, e);
|
||||
*failed_objects += 1;
|
||||
checkpoint_manager.add_failed_object(object.clone()).await?;
|
||||
warn!("Failed to check existence of {}/{}: {}, skipping", bucket, object, e);
|
||||
*current_object_index = obj_idx + 1;
|
||||
continue;
|
||||
}
|
||||
@@ -367,7 +363,7 @@ impl ErasureSetHealer {
|
||||
let _permit = semaphore
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|e| Error::other(format!("Failed to acquire semaphore for bucket heal: {e}")))?;
|
||||
.map_err(|e| Error::other(format!("Failed to acquire semaphore for bucket heal: {}", e)))?;
|
||||
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(Error::TaskCancelled);
|
||||
@@ -465,7 +461,7 @@ impl ErasureSetHealer {
|
||||
let _permit = semaphore
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|e| Error::other(format!("Failed to acquire semaphore for object heal: {e}")))?;
|
||||
.map_err(|e| Error::other(format!("Failed to acquire semaphore for object heal: {}", e)))?;
|
||||
|
||||
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
|
||||
Ok((_result, None)) => {
|
||||
|
||||
@@ -22,7 +22,7 @@ use rustfs_ecstore::disk::DiskAPI;
|
||||
use rustfs_ecstore::disk::error::DiskError;
|
||||
use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
|
||||
use std::{
|
||||
collections::{BinaryHeap, HashMap, HashSet},
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
@@ -33,151 +33,6 @@ use tokio::{
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// Priority queue wrapper for heal requests
|
||||
/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items
|
||||
#[derive(Debug)]
|
||||
struct PriorityHealQueue {
|
||||
/// Heap of (priority, sequence, request) tuples
|
||||
heap: BinaryHeap<PriorityQueueItem>,
|
||||
/// Sequence counter for FIFO ordering within same priority
|
||||
sequence: u64,
|
||||
/// Set of request keys to prevent duplicates
|
||||
dedup_keys: HashSet<String>,
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
#[derive(Debug)]
|
||||
struct PriorityQueueItem {
|
||||
priority: HealPriority,
|
||||
sequence: u64,
|
||||
request: HealRequest,
|
||||
}
|
||||
|
||||
impl Eq for PriorityQueueItem {}
|
||||
|
||||
impl PartialEq for PriorityQueueItem {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.priority == other.priority && self.sequence == other.sequence
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for PriorityQueueItem {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
// First compare by priority (higher priority first)
|
||||
match self.priority.cmp(&other.priority) {
|
||||
std::cmp::Ordering::Equal => {
|
||||
// If priorities are equal, use sequence for FIFO (lower sequence first)
|
||||
other.sequence.cmp(&self.sequence)
|
||||
}
|
||||
ordering => ordering,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for PriorityQueueItem {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl PriorityHealQueue {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.heap.is_empty()
|
||||
}
|
||||
|
||||
fn push(&mut self, request: HealRequest) -> bool {
|
||||
let key = Self::make_dedup_key(&request);
|
||||
|
||||
// Check for duplicates
|
||||
if self.dedup_keys.contains(&key) {
|
||||
return false; // Duplicate request, don't add
|
||||
}
|
||||
|
||||
self.dedup_keys.insert(key);
|
||||
self.sequence += 1;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
request,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Get statistics about queue contents by priority
|
||||
fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
|
||||
let mut stats = HashMap::new();
|
||||
for item in &self.heap {
|
||||
*stats.entry(item.priority).or_insert(0) += 1;
|
||||
}
|
||||
stats
|
||||
}
|
||||
|
||||
fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
let key = Self::make_dedup_key(&item.request);
|
||||
self.dedup_keys.remove(&key);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
fn make_dedup_key(request: &HealRequest) -> String {
|
||||
match &request.heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
format!("bucket:{}", bucket)
|
||||
}
|
||||
HealType::ErasureSet { set_disk_id, .. } => {
|
||||
format!("erasure_set:{}", set_disk_id)
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
format!("metadata:{}:{}", bucket, object)
|
||||
}
|
||||
HealType::MRF { meta_path } => {
|
||||
format!("mrf:{}", meta_path)
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a request with the same key already exists in the queue
|
||||
#[allow(dead_code)]
|
||||
fn contains_key(&self, request: &HealRequest) -> bool {
|
||||
let key = Self::make_dedup_key(request);
|
||||
self.dedup_keys.contains(&key)
|
||||
}
|
||||
|
||||
/// Check if an erasure set heal request for a specific set_disk_id exists
|
||||
fn contains_erasure_set(&self, set_disk_id: &str) -> bool {
|
||||
let key = format!("erasure_set:{}", set_disk_id);
|
||||
self.dedup_keys.contains(&key)
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal config
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealConfig {
|
||||
@@ -230,8 +85,8 @@ pub struct HealManager {
|
||||
state: Arc<RwLock<HealState>>,
|
||||
/// Active heal tasks
|
||||
active_heals: Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
|
||||
/// Heal queue (priority-based)
|
||||
heal_queue: Arc<Mutex<PriorityHealQueue>>,
|
||||
/// Heal queue
|
||||
heal_queue: Arc<Mutex<VecDeque<HealRequest>>>,
|
||||
/// Storage layer interface
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
/// Cancel token
|
||||
@@ -248,7 +103,7 @@ impl HealManager {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
state: Arc::new(RwLock::new(HealState::default())),
|
||||
active_heals: Arc::new(Mutex::new(HashMap::new())),
|
||||
heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())),
|
||||
heal_queue: Arc::new(Mutex::new(VecDeque::new())),
|
||||
storage,
|
||||
cancel_token: CancellationToken::new(),
|
||||
statistics: Arc::new(RwLock::new(HealStatistics::new())),
|
||||
@@ -306,54 +161,17 @@ impl HealManager {
|
||||
let config = self.config.read().await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
|
||||
let queue_len = queue.len();
|
||||
let queue_capacity = config.queue_size;
|
||||
|
||||
if queue_len >= queue_capacity {
|
||||
if queue.len() >= config.queue_size {
|
||||
return Err(Error::ConfigurationError {
|
||||
message: format!("Heal queue is full ({}/{})", queue_len, queue_capacity),
|
||||
message: "Heal queue is full".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Warn when queue is getting full (>80% capacity)
|
||||
let capacity_threshold = (queue_capacity as f64 * 0.8) as usize;
|
||||
if queue_len >= capacity_threshold {
|
||||
warn!(
|
||||
"Heal queue is {}% full ({}/{}). Consider increasing queue size or processing capacity.",
|
||||
(queue_len * 100) / queue_capacity,
|
||||
queue_len,
|
||||
queue_capacity
|
||||
);
|
||||
}
|
||||
|
||||
let request_id = request.id.clone();
|
||||
let priority = request.priority;
|
||||
|
||||
// Try to push the request; if it's a duplicate, still return the request_id
|
||||
let is_new = queue.push(request);
|
||||
|
||||
// Log queue statistics periodically (when adding high/urgent priority items)
|
||||
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
|
||||
let stats = queue.get_priority_stats();
|
||||
info!(
|
||||
"Heal queue stats after adding {:?} priority request: total={}, urgent={}, high={}, normal={}, low={}",
|
||||
priority,
|
||||
queue_len + 1,
|
||||
stats.get(&HealPriority::Urgent).unwrap_or(&0),
|
||||
stats.get(&HealPriority::High).unwrap_or(&0),
|
||||
stats.get(&HealPriority::Normal).unwrap_or(&0),
|
||||
stats.get(&HealPriority::Low).unwrap_or(&0)
|
||||
);
|
||||
}
|
||||
|
||||
queue.push_back(request);
|
||||
drop(queue);
|
||||
|
||||
if is_new {
|
||||
info!("Submitted heal request: {} with priority: {:?}", request_id, priority);
|
||||
} else {
|
||||
info!("Heal request already queued (duplicate): {}", request_id);
|
||||
}
|
||||
|
||||
info!("Submitted heal request: {}", request_id);
|
||||
Ok(request_id)
|
||||
}
|
||||
|
||||
@@ -503,7 +321,13 @@ impl HealManager {
|
||||
let mut skip = false;
|
||||
{
|
||||
let queue = heal_queue.lock().await;
|
||||
if queue.contains_erasure_set(&set_disk_id) {
|
||||
if queue.iter().any(|req| {
|
||||
matches!(
|
||||
&req.heal_type,
|
||||
crate::heal::task::HealType::ErasureSet { set_disk_id: queued_id, .. }
|
||||
if queued_id == &set_disk_id
|
||||
)
|
||||
}) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
@@ -534,7 +358,7 @@ impl HealManager {
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let mut queue = heal_queue.lock().await;
|
||||
queue.push(req);
|
||||
queue.push_back(req);
|
||||
info!("Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
|
||||
}
|
||||
}
|
||||
@@ -545,9 +369,8 @@ impl HealManager {
|
||||
}
|
||||
|
||||
/// Process heal queue
|
||||
/// Processes multiple tasks per cycle when capacity allows and queue has high-priority items
|
||||
async fn process_heal_queue(
|
||||
heal_queue: &Arc<Mutex<PriorityHealQueue>>,
|
||||
heal_queue: &Arc<Mutex<VecDeque<HealRequest>>>,
|
||||
active_heals: &Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
|
||||
config: &Arc<RwLock<HealConfig>>,
|
||||
statistics: &Arc<RwLock<HealStatistics>>,
|
||||
@@ -556,83 +379,51 @@ impl HealManager {
|
||||
let config = config.read().await;
|
||||
let mut active_heals_guard = active_heals.lock().await;
|
||||
|
||||
// Check if new heal tasks can be started
|
||||
let active_count = active_heals_guard.len();
|
||||
if active_count >= config.max_concurrent_heals {
|
||||
// check if new heal tasks can be started
|
||||
if active_heals_guard.len() >= config.max_concurrent_heals {
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate how many tasks we can start this cycle
|
||||
let available_slots = config.max_concurrent_heals - active_count;
|
||||
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let queue_len = queue.len();
|
||||
if let Some(request) = queue.pop_front() {
|
||||
let task = Arc::new(HealTask::from_request(request, storage.clone()));
|
||||
let task_id = task.id.clone();
|
||||
active_heals_guard.insert(task_id.clone(), task.clone());
|
||||
drop(active_heals_guard);
|
||||
let active_heals_clone = active_heals.clone();
|
||||
let statistics_clone = statistics.clone();
|
||||
|
||||
if queue_len == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Process multiple tasks if:
|
||||
// 1. We have available slots
|
||||
// 2. Queue is not empty
|
||||
// Prioritize urgent/high priority tasks by processing up to 2 tasks per cycle if available
|
||||
let tasks_to_process = if queue_len > 0 {
|
||||
std::cmp::min(available_slots, std::cmp::min(2, queue_len))
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
for _ in 0..tasks_to_process {
|
||||
if let Some(request) = queue.pop() {
|
||||
let task_priority = request.priority;
|
||||
let task = Arc::new(HealTask::from_request(request, storage.clone()));
|
||||
let task_id = task.id.clone();
|
||||
active_heals_guard.insert(task_id.clone(), task.clone());
|
||||
let active_heals_clone = active_heals.clone();
|
||||
let statistics_clone = statistics.clone();
|
||||
|
||||
// start heal task
|
||||
tokio::spawn(async move {
|
||||
info!("Starting heal task: {} with priority: {:?}", task_id, task_priority);
|
||||
let result = task.execute().await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Heal task completed successfully: {}", task_id);
|
||||
// start heal task
|
||||
tokio::spawn(async move {
|
||||
info!("Starting heal task: {}", task_id);
|
||||
let result = task.execute().await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Heal task completed successfully: {}", task_id);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Heal task failed: {} - {}", task_id, e);
|
||||
}
|
||||
}
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_task.get_status().await {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Heal task failed: {} - {}", task_id, e);
|
||||
_ => {
|
||||
stats.update_task_completion(false);
|
||||
}
|
||||
}
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_task.get_status().await {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
}
|
||||
_ => {
|
||||
stats.update_task_completion(false);
|
||||
}
|
||||
}
|
||||
stats.update_running_tasks(active_heals_guard.len() as u64);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stats.update_running_tasks(active_heals_guard.len() as u64);
|
||||
}
|
||||
});
|
||||
|
||||
// Update statistics for all started tasks
|
||||
let mut stats = statistics.write().await;
|
||||
stats.total_tasks += tasks_to_process as u64;
|
||||
|
||||
// Log queue status if items remain
|
||||
if !queue.is_empty() {
|
||||
let remaining = queue.len();
|
||||
if remaining > 10 {
|
||||
info!("Heal queue has {} pending requests, {} tasks active", remaining, active_heals_guard.len());
|
||||
}
|
||||
// update statistics
|
||||
let mut stats = statistics.write().await;
|
||||
stats.total_tasks += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -647,333 +438,3 @@ impl std::fmt::Debug for HealManager {
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_ordering() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
// Add requests with different priorities
|
||||
let low_req = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
|
||||
let normal_req = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket2".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let high_req = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket3".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
|
||||
let urgent_req = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket4".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Urgent,
|
||||
);
|
||||
|
||||
// Add in random order: low, high, normal, urgent
|
||||
assert!(queue.push(low_req));
|
||||
assert!(queue.push(high_req));
|
||||
assert!(queue.push(normal_req));
|
||||
assert!(queue.push(urgent_req));
|
||||
|
||||
assert_eq!(queue.len(), 4);
|
||||
|
||||
// Should pop in priority order: urgent, high, normal, low
|
||||
let popped1 = queue.pop().unwrap();
|
||||
assert_eq!(popped1.priority, HealPriority::Urgent);
|
||||
|
||||
let popped2 = queue.pop().unwrap();
|
||||
assert_eq!(popped2.priority, HealPriority::High);
|
||||
|
||||
let popped3 = queue.pop().unwrap();
|
||||
assert_eq!(popped3.priority, HealPriority::Normal);
|
||||
|
||||
let popped4 = queue.pop().unwrap();
|
||||
assert_eq!(popped4.priority, HealPriority::Low);
|
||||
|
||||
assert_eq!(queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fifo_same_priority() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
// Add multiple requests with same priority
|
||||
let req1 = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let req2 = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket2".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let req3 = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket3".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let id1 = req1.id.clone();
|
||||
let id2 = req2.id.clone();
|
||||
let id3 = req3.id.clone();
|
||||
|
||||
assert!(queue.push(req1));
|
||||
assert!(queue.push(req2));
|
||||
assert!(queue.push(req3));
|
||||
|
||||
// Should maintain FIFO order for same priority
|
||||
let popped1 = queue.pop().unwrap();
|
||||
assert_eq!(popped1.id, id1);
|
||||
|
||||
let popped2 = queue.pop().unwrap();
|
||||
assert_eq!(popped2.id, id2);
|
||||
|
||||
let popped3 = queue.pop().unwrap();
|
||||
assert_eq!(popped3.id, id3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_deduplication() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
let req1 = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket1".to_string(),
|
||||
object: "object1".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let req2 = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket1".to_string(),
|
||||
object: "object1".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
|
||||
// First request should be added
|
||||
assert!(queue.push(req1));
|
||||
assert_eq!(queue.len(), 1);
|
||||
|
||||
// Second request with same object should be rejected (duplicate)
|
||||
assert!(!queue.push(req2));
|
||||
assert_eq!(queue.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_contains_erasure_set() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
let req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket1".to_string()],
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
assert!(queue.push(req));
|
||||
assert!(queue.contains_erasure_set("pool_0_set_1"));
|
||||
assert!(!queue.contains_erasure_set("pool_0_set_2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_dedup_key_generation() {
|
||||
// Test different heal types generate different keys
|
||||
let obj_req = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket1".to_string(),
|
||||
object: "object1".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let bucket_req = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let erasure_req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket1".to_string()],
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let obj_key = PriorityHealQueue::make_dedup_key(&obj_req);
|
||||
let bucket_key = PriorityHealQueue::make_dedup_key(&bucket_req);
|
||||
let erasure_key = PriorityHealQueue::make_dedup_key(&erasure_req);
|
||||
|
||||
// All keys should be different
|
||||
assert_ne!(obj_key, bucket_key);
|
||||
assert_ne!(obj_key, erasure_key);
|
||||
assert_ne!(bucket_key, erasure_key);
|
||||
|
||||
assert!(obj_key.starts_with("object:"));
|
||||
assert!(bucket_key.starts_with("bucket:"));
|
||||
assert!(erasure_key.starts_with("erasure_set:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_mixed_priorities_and_types() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
// Add various requests
|
||||
let requests = vec![
|
||||
(
|
||||
HealType::Object {
|
||||
bucket: "b1".to_string(),
|
||||
object: "o1".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealPriority::Low,
|
||||
),
|
||||
(
|
||||
HealType::Bucket {
|
||||
bucket: "b2".to_string(),
|
||||
},
|
||||
HealPriority::Urgent,
|
||||
),
|
||||
(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["b3".to_string()],
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
(
|
||||
HealType::Object {
|
||||
bucket: "b4".to_string(),
|
||||
object: "o4".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealPriority::High,
|
||||
),
|
||||
];
|
||||
|
||||
for (heal_type, priority) in requests {
|
||||
let req = HealRequest::new(heal_type, HealOptions::default(), priority);
|
||||
queue.push(req);
|
||||
}
|
||||
|
||||
assert_eq!(queue.len(), 4);
|
||||
|
||||
// Check they come out in priority order
|
||||
let priorities: Vec<HealPriority> = (0..4).filter_map(|_| queue.pop().map(|r| r.priority)).collect();
|
||||
|
||||
assert_eq!(
|
||||
priorities,
|
||||
vec![
|
||||
HealPriority::Urgent,
|
||||
HealPriority::High,
|
||||
HealPriority::Normal,
|
||||
HealPriority::Low,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_stats() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
// Add requests with different priorities
|
||||
for _ in 0..3 {
|
||||
queue.push(HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: format!("bucket-low-{}", queue.len()),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
));
|
||||
}
|
||||
|
||||
for _ in 0..2 {
|
||||
queue.push(HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: format!("bucket-normal-{}", queue.len()),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
));
|
||||
}
|
||||
|
||||
queue.push(HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-high".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
));
|
||||
|
||||
let stats = queue.get_priority_stats();
|
||||
|
||||
assert_eq!(*stats.get(&HealPriority::Low).unwrap_or(&0), 3);
|
||||
assert_eq!(*stats.get(&HealPriority::Normal).unwrap_or(&0), 2);
|
||||
assert_eq!(*stats.get(&HealPriority::High).unwrap_or(&0), 1);
|
||||
assert_eq!(*stats.get(&HealPriority::Urgent).unwrap_or(&0), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_is_empty() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
|
||||
assert!(queue.is_empty());
|
||||
|
||||
queue.push(HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "test".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
));
|
||||
|
||||
assert!(!queue.is_empty());
|
||||
|
||||
queue.pop();
|
||||
|
||||
assert!(queue.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
|
||||
/// Helper function to convert Path to &str, returning an error if conversion fails
|
||||
fn path_to_str(path: &Path) -> Result<&str> {
|
||||
path.to_str()
|
||||
.ok_or_else(|| Error::other(format!("Invalid UTF-8 path: {path:?}")))
|
||||
.ok_or_else(|| Error::other(format!("Invalid UTF-8 path: {:?}", path)))
|
||||
}
|
||||
|
||||
/// resume state
|
||||
|
||||
@@ -180,7 +180,8 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
MAX_READ_BYTES, bucket, object
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"Object too large: {n_read} bytes (max: {MAX_READ_BYTES} bytes) for {bucket}/{object}"
|
||||
"Object too large: {} bytes (max: {} bytes) for {}/{}",
|
||||
n_read, MAX_READ_BYTES, bucket, object
|
||||
)));
|
||||
}
|
||||
}
|
||||
@@ -400,13 +401,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(_) => Ok(true), // Object exists
|
||||
Err(e) => {
|
||||
// Map ObjectNotFound to false, other errors must be propagated!
|
||||
// Map ObjectNotFound to false, other errors to false as well for safety
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
debug!("Object not found: {}/{}", bucket, object);
|
||||
Ok(false)
|
||||
} else {
|
||||
error!("Error checking object existence {}/{}: {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
debug!("Error checking object existence {}/{}: {}", bucket, object, e);
|
||||
Ok(false) // Treat errors as non-existence to be safe
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -498,7 +499,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self
|
||||
.ecstore
|
||||
.clone()
|
||||
.list_objects_v2(bucket, prefix, None, None, 1000, false, None, false)
|
||||
.list_objects_v2(bucket, prefix, None, None, 1000, false, None)
|
||||
.await
|
||||
{
|
||||
Ok(list_info) => {
|
||||
|
||||
@@ -51,7 +51,7 @@ pub enum HealType {
|
||||
}
|
||||
|
||||
/// Heal priority
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub enum HealPriority {
|
||||
/// Low priority
|
||||
Low = 0,
|
||||
@@ -272,7 +272,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
|
||||
pub async fn execute(&self) -> Result<()> {
|
||||
// update status and timestamps atomically to avoid race conditions
|
||||
let now = SystemTime::now();
|
||||
@@ -286,7 +285,7 @@ impl HealTask {
|
||||
*task_start_instant = Some(start_instant);
|
||||
}
|
||||
|
||||
info!("Task started");
|
||||
info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Object {
|
||||
@@ -316,7 +315,7 @@ impl HealTask {
|
||||
Ok(_) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Completed;
|
||||
info!("Task completed successfully");
|
||||
info!("Heal task completed successfully: {}", self.id);
|
||||
}
|
||||
Err(Error::TaskCancelled) => {
|
||||
let mut status = self.status.write().await;
|
||||
@@ -355,9 +354,8 @@ impl HealTask {
|
||||
}
|
||||
|
||||
// specific heal implementation method
|
||||
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
|
||||
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!("Starting object heal workflow");
|
||||
info!("Healing object: {}/{}", bucket, object);
|
||||
|
||||
// update progress
|
||||
{
|
||||
@@ -367,7 +365,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
// Step 1: Check if object exists and get metadata
|
||||
warn!("Step 1: Checking object existence and metadata");
|
||||
info!("Step 1: Checking object existence and metadata");
|
||||
self.check_control_flags().await?;
|
||||
let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
|
||||
if !object_exists {
|
||||
@@ -426,7 +424,7 @@ impl HealTask {
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
info!("Removing corrupted object: {}/{}", bucket, object);
|
||||
warn!("Removing corrupted object: {}/{}", bucket, object);
|
||||
if !self.options.dry_run {
|
||||
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
|
||||
info!("Successfully deleted corrupted object: {}/{}", bucket, object);
|
||||
@@ -449,9 +447,11 @@ impl HealTask {
|
||||
info!("Step 3: Verifying heal result");
|
||||
let object_size = result.object_size as u64;
|
||||
info!(
|
||||
object_size = object_size,
|
||||
drives_healed = result.after.drives.len(),
|
||||
"Heal completed successfully"
|
||||
"Heal completed successfully: {}/{} ({} bytes, {} drives healed)",
|
||||
bucket,
|
||||
object,
|
||||
object_size,
|
||||
result.after.drives.len()
|
||||
);
|
||||
|
||||
{
|
||||
@@ -481,7 +481,7 @@ impl HealTask {
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
info!("Removing corrupted object: {}/{}", bucket, object);
|
||||
warn!("Removing corrupted object: {}/{}", bucket, object);
|
||||
if !self.options.dry_run {
|
||||
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
|
||||
info!("Successfully deleted corrupted object: {}/{}", bucket, object);
|
||||
|
||||
@@ -1005,7 +1005,6 @@ impl Scanner {
|
||||
100, // max_keys - small limit for performance
|
||||
false, // fetch_owner
|
||||
None, // start_after
|
||||
false, // incl_deleted
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -144,7 +144,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
let mut wtxn = lmdb_env.write_txn().unwrap();
|
||||
let db = match lmdb_env
|
||||
.database_options()
|
||||
.name(&format!("bucket_{bucket_name}"))
|
||||
.name(&format!("bucket_{}", bucket_name))
|
||||
.types::<I64<BigEndian>, LifecycleContentCodec>()
|
||||
.flags(DatabaseFlags::DUP_SORT)
|
||||
//.dup_sort_comparator::<>()
|
||||
@@ -152,7 +152,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
{
|
||||
Ok(db) => db,
|
||||
Err(err) => {
|
||||
panic!("lmdb error: {err}");
|
||||
panic!("lmdb error: {}", err);
|
||||
}
|
||||
};
|
||||
let _ = wtxn.commit();
|
||||
@@ -199,7 +199,7 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
||||
.await
|
||||
.expect("Failed to upload test object");
|
||||
|
||||
println!("object_info1: {object_info:?}");
|
||||
println!("object_info1: {:?}", object_info);
|
||||
|
||||
info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
|
||||
}
|
||||
@@ -456,7 +456,7 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
let object_info = convert_record_to_object_info(record);
|
||||
println!("object_info2: {object_info:?}");
|
||||
println!("object_info2: {:?}", object_info);
|
||||
let mod_time = object_info.mod_time.unwrap_or(OffsetDateTime::now_utc());
|
||||
let expiry_time = rustfs_ecstore::bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, 1);
|
||||
|
||||
@@ -494,9 +494,9 @@ mod serial_tests {
|
||||
type_,
|
||||
object_name,
|
||||
} = &elm.1;
|
||||
println!("cache row:{ver_no} {ver_id} {mod_time} {type_:?} {object_name}");
|
||||
println!("cache row:{} {} {} {:?} {}", ver_no, ver_id, mod_time, type_, object_name);
|
||||
}
|
||||
println!("row:{row:?}");
|
||||
println!("row:{:?}", row);
|
||||
}
|
||||
//drop(iter);
|
||||
wtxn.commit().unwrap();
|
||||
|
||||
@@ -277,11 +277,11 @@ async fn create_test_tier(server: u32) {
|
||||
};
|
||||
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
|
||||
if let Err(err) = tier_config_mgr.add(args, false).await {
|
||||
println!("tier_config_mgr add failed, e: {err:?}");
|
||||
println!("tier_config_mgr add failed, e: {:?}", err);
|
||||
panic!("tier add failed. {err}");
|
||||
}
|
||||
if let Err(e) = tier_config_mgr.save().await {
|
||||
println!("tier_config_mgr save failed, e: {e:?}");
|
||||
println!("tier_config_mgr save failed, e: {:?}", e);
|
||||
panic!("tier save failed");
|
||||
}
|
||||
println!("Created test tier: COLDTIER44");
|
||||
@@ -299,7 +299,7 @@ async fn object_exists(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bo
|
||||
#[allow(dead_code)]
|
||||
async fn object_is_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
|
||||
if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
|
||||
println!("oi: {oi:?}");
|
||||
println!("oi: {:?}", oi);
|
||||
oi.delete_marker
|
||||
} else {
|
||||
println!("object_is_delete_marker is error");
|
||||
@@ -311,7 +311,7 @@ async fn object_is_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &
|
||||
#[allow(dead_code)]
|
||||
async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> bool {
|
||||
if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
|
||||
println!("oi: {oi:?}");
|
||||
println!("oi: {:?}", oi);
|
||||
!oi.transitioned_object.status.is_empty()
|
||||
} else {
|
||||
println!("object_is_transitioned is error");
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::{AuditEntry, AuditResult, AuditSystem};
|
||||
use rustfs_ecstore::config::Config;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{debug, error, trace, warn};
|
||||
use tracing::{error, trace, warn};
|
||||
|
||||
/// Global audit system instance
|
||||
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
|
||||
@@ -78,7 +78,7 @@ pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
|
||||
} else {
|
||||
// The system is not initialized at all. This is a more important state.
|
||||
// It might be better to return an error or log a warning.
|
||||
debug!("Audit system not initialized, dropping audit entry.");
|
||||
warn!("Audit system not initialized, dropping audit entry.");
|
||||
// If this should be a hard failure, you can return Err(AuditError::NotInitialized("..."))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ impl AuditSystem {
|
||||
|
||||
/// Starts the audit system with the given configuration
|
||||
pub async fn start(&self, config: Config) -> AuditResult<()> {
|
||||
let state = self.state.write().await;
|
||||
let mut state = self.state.write().await;
|
||||
|
||||
match *state {
|
||||
AuditSystemState::Running => {
|
||||
@@ -72,6 +72,7 @@ impl AuditSystem {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
*state = AuditSystemState::Starting;
|
||||
drop(state);
|
||||
|
||||
info!("Starting audit system");
|
||||
@@ -89,17 +90,6 @@ impl AuditSystem {
|
||||
let mut registry = self.registry.lock().await;
|
||||
match registry.create_targets_from_config(&config).await {
|
||||
Ok(targets) => {
|
||||
if targets.is_empty() {
|
||||
info!("No enabled audit targets found, keeping audit system stopped");
|
||||
drop(registry);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
{
|
||||
let mut state = self.state.write().await;
|
||||
*state = AuditSystemState::Starting;
|
||||
}
|
||||
|
||||
info!(target_count = targets.len(), "Created audit targets successfully");
|
||||
|
||||
// Initialize all targets
|
||||
|
||||
@@ -1105,17 +1105,10 @@ impl TargetClient {
|
||||
Err(e) => match e {
|
||||
SdkError::ServiceError(oe) => match oe.into_err() {
|
||||
HeadBucketError::NotFound(_) => Ok(false),
|
||||
other => Err(S3ClientError::new(format!(
|
||||
"failed to check bucket exists for bucket:{bucket} please check the bucket name and credentials, error:{other:?}"
|
||||
))),
|
||||
other => Err(other.into()),
|
||||
},
|
||||
SdkError::DispatchFailure(e) => Err(S3ClientError::new(format!(
|
||||
"failed to dispatch bucket exists for bucket:{bucket} error:{e:?}"
|
||||
))),
|
||||
|
||||
_ => Err(S3ClientError::new(format!(
|
||||
"failed to check bucket exists for bucket:{bucket} error:{e:?}"
|
||||
))),
|
||||
_ => Err(e.into()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,8 +103,6 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
if filter.test_tags(&object_tags) {
|
||||
rules.push(rule.clone());
|
||||
}
|
||||
} else {
|
||||
rules.push(rule.clone());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
|
||||
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE,
|
||||
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE,
|
||||
RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
@@ -2324,7 +2324,7 @@ async fn replicate_object_with_multipart(
|
||||
let mut user_metadata = HashMap::new();
|
||||
|
||||
user_metadata.insert(
|
||||
RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE.to_string(),
|
||||
RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE.to_string(),
|
||||
object_info
|
||||
.user_defined
|
||||
.get(&format!("{RESERVED_METADATA_PREFIX}actual-size"))
|
||||
|
||||
@@ -277,7 +277,6 @@ pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Res
|
||||
1000, // max_keys
|
||||
false, // fetch_owner
|
||||
None, // start_after
|
||||
false, // incl_deleted
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -140,12 +140,6 @@ pub enum DiskError {
|
||||
|
||||
#[error("io error {0}")]
|
||||
Io(io::Error),
|
||||
|
||||
#[error("source stalled")]
|
||||
SourceStalled,
|
||||
|
||||
#[error("timeout")]
|
||||
Timeout,
|
||||
}
|
||||
|
||||
impl DiskError {
|
||||
@@ -372,8 +366,6 @@ impl Clone for DiskError {
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
DiskError::SourceStalled => DiskError::SourceStalled,
|
||||
DiskError::Timeout => DiskError::Timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -420,8 +412,6 @@ impl DiskError {
|
||||
DiskError::ErasureWriteQuorum => 0x25,
|
||||
DiskError::ErasureReadQuorum => 0x26,
|
||||
DiskError::ShortWrite => 0x27,
|
||||
DiskError::SourceStalled => 0x28,
|
||||
DiskError::Timeout => 0x29,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -466,8 +456,6 @@ impl DiskError {
|
||||
0x25 => Some(DiskError::ErasureWriteQuorum),
|
||||
0x26 => Some(DiskError::ErasureReadQuorum),
|
||||
0x27 => Some(DiskError::ShortWrite),
|
||||
0x28 => Some(DiskError::SourceStalled),
|
||||
0x29 => Some(DiskError::Timeout),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -806,7 +806,7 @@ impl LocalDisk {
|
||||
Ok((bytes, modtime))
|
||||
}
|
||||
|
||||
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo]) -> Result<()> {
|
||||
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec<FileInfo>) -> Result<()> {
|
||||
let volume_dir = self.get_bucket_path(volume)?;
|
||||
let xlpath = self.get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
|
||||
@@ -820,7 +820,7 @@ impl LocalDisk {
|
||||
|
||||
fm.unmarshal_msg(&data)?;
|
||||
|
||||
for fi in fis.iter() {
|
||||
for fi in fis {
|
||||
let data_dir = match fm.delete_version(fi) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
@@ -2300,6 +2300,7 @@ impl DiskAPI for LocalDisk {
|
||||
let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
//
|
||||
if err != DiskError::FileNotFound {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -1085,7 +1085,7 @@ mod tests {
|
||||
|
||||
drop(listener);
|
||||
|
||||
let url = url::Url::parse(&format!("http://{ip}:{port}/data/rustfs0")).unwrap();
|
||||
let url = url::Url::parse(&format!("http://{}:{}/data/rustfs0", ip, port)).unwrap();
|
||||
let endpoint = Endpoint {
|
||||
url,
|
||||
is_local: false,
|
||||
|
||||
@@ -68,7 +68,6 @@ use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::{Rng, seq::SliceRandom};
|
||||
use regex::Regex;
|
||||
use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType, HealOpts, HealScanMode, send_heal_disk};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
|
||||
RawFileInfo, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
|
||||
@@ -112,7 +111,7 @@ use tracing::error;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024;
|
||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
||||
@@ -1316,26 +1315,9 @@ impl SetDisks {
|
||||
|
||||
for (i, meta) in metas.iter().enumerate() {
|
||||
if !meta.is_valid() {
|
||||
debug!(
|
||||
index = i,
|
||||
valid = false,
|
||||
version_id = ?meta.version_id,
|
||||
mod_time = ?meta.mod_time,
|
||||
"find_file_info_in_quorum: skipping invalid meta"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!(
|
||||
index = i,
|
||||
valid = true,
|
||||
version_id = ?meta.version_id,
|
||||
mod_time = ?meta.mod_time,
|
||||
deleted = meta.deleted,
|
||||
size = meta.size,
|
||||
"find_file_info_in_quorum: inspecting meta"
|
||||
);
|
||||
|
||||
let etag_only = mod_time.is_none() && etag.is_some() && meta.get_etag().is_some_and(|v| &v == etag.as_ref().unwrap());
|
||||
let mod_valid = mod_time == &meta.mod_time;
|
||||
|
||||
@@ -1361,13 +1343,6 @@ impl SetDisks {
|
||||
meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice()));
|
||||
|
||||
hasher.reset();
|
||||
} else {
|
||||
debug!(
|
||||
index = i,
|
||||
etag_only_match = etag_only,
|
||||
mod_valid_match = mod_valid,
|
||||
"find_file_info_in_quorum: meta does not match common etag or mod_time, skipping hash calculation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1516,7 +1491,7 @@ impl SetDisks {
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<Vec<FileInfo>> {
|
||||
) -> Result<Vec<rustfs_filemeta::FileInfo>> {
|
||||
// Use existing disk selection logic
|
||||
let disks = self.disks.read().await;
|
||||
let required_reads = self.format.erasure.sets.len();
|
||||
@@ -2205,11 +2180,11 @@ impl SetDisks {
|
||||
// TODO: replicatio
|
||||
|
||||
if fi.deleted {
|
||||
return if opts.version_id.is_none() || opts.delete_marker {
|
||||
Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]))
|
||||
if opts.version_id.is_none() || opts.delete_marker {
|
||||
return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
|
||||
} else {
|
||||
Err(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]))
|
||||
};
|
||||
return Err(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]));
|
||||
}
|
||||
}
|
||||
|
||||
Ok((oi, write_quorum))
|
||||
@@ -2237,7 +2212,7 @@ impl SetDisks {
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start");
|
||||
tracing::debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start");
|
||||
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi);
|
||||
|
||||
let total_size = fi.size as usize;
|
||||
@@ -2262,20 +2237,27 @@ impl SetDisks {
|
||||
|
||||
let (last_part_index, last_part_relative_offset) = fi.to_part_offset(end_offset)?;
|
||||
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
bucket,
|
||||
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
|
||||
object,
|
||||
offset,
|
||||
length,
|
||||
end_offset,
|
||||
part_index,
|
||||
last_part_index,
|
||||
last_part_relative_offset,
|
||||
"Multipart read bounds"
|
||||
);
|
||||
|
||||
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
|
||||
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
|
||||
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
|
||||
tracing::debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
|
||||
|
||||
let mut total_read = 0;
|
||||
for current_part in part_indices {
|
||||
if total_read == length {
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
bucket,
|
||||
object,
|
||||
total_read,
|
||||
@@ -2297,7 +2279,7 @@ impl SetDisks {
|
||||
|
||||
let read_offset = (part_offset / erasure.block_size) * erasure.shard_size();
|
||||
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
bucket,
|
||||
object,
|
||||
part_index = current_part,
|
||||
@@ -2352,65 +2334,12 @@ impl SetDisks {
|
||||
return Err(Error::other(format!("not enough disks to read: {errors:?}")));
|
||||
}
|
||||
|
||||
// Check if we have missing shards even though we can read successfully
|
||||
// This happens when a node was offline during write and comes back online
|
||||
let total_shards = erasure.data_shards + erasure.parity_shards;
|
||||
let available_shards = nil_count;
|
||||
let missing_shards = total_shards - available_shards;
|
||||
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
total_shards,
|
||||
available_shards,
|
||||
missing_shards,
|
||||
data_shards = erasure.data_shards,
|
||||
parity_shards = erasure.parity_shards,
|
||||
"Shard availability check"
|
||||
);
|
||||
|
||||
if missing_shards > 0 && available_shards >= erasure.data_shards {
|
||||
// We have missing shards but enough to read - trigger background heal
|
||||
info!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
missing_shards,
|
||||
available_shards,
|
||||
pool_index,
|
||||
set_index,
|
||||
"Detected missing shards during read, triggering background heal"
|
||||
);
|
||||
if let Err(e) =
|
||||
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(pool_index),
|
||||
Some(set_index),
|
||||
))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
error = %e,
|
||||
"Failed to enqueue heal request for missing shards"
|
||||
);
|
||||
} else {
|
||||
warn!(bucket, object, part_number, "Successfully enqueued heal request for missing shards");
|
||||
}
|
||||
}
|
||||
|
||||
// debug!(
|
||||
// "read part {} part_offset {},part_length {},part_size {} ",
|
||||
// part_number, part_offset, part_length, part_size
|
||||
// );
|
||||
let (written, err) = erasure.decode(writer, readers, part_offset, part_length, part_size).await;
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
bucket,
|
||||
object,
|
||||
part_index = current_part,
|
||||
@@ -2426,7 +2355,7 @@ impl SetDisks {
|
||||
match de_err {
|
||||
DiskError::FileNotFound | DiskError::FileCorrupt => {
|
||||
error!("erasure.decode err 111 {:?}", &de_err);
|
||||
if let Err(e) = rustfs_common::heal_channel::send_heal_request(
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(
|
||||
rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
@@ -2436,16 +2365,7 @@ impl SetDisks {
|
||||
Some(set_index),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
part_number,
|
||||
error = %e,
|
||||
"Failed to enqueue heal request after decode error"
|
||||
);
|
||||
}
|
||||
.await;
|
||||
has_err = false;
|
||||
}
|
||||
_ => {}
|
||||
@@ -2466,7 +2386,7 @@ impl SetDisks {
|
||||
|
||||
// debug!("read end");
|
||||
|
||||
debug!(bucket, object, total_read, expected_length = length, "Multipart read finished");
|
||||
tracing::debug!(bucket, object, total_read, expected_length = length, "Multipart read finished");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -2585,7 +2505,6 @@ impl SetDisks {
|
||||
Ok((new_disks, new_infos, healing))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2593,7 +2512,10 @@ impl SetDisks {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
info!(?opts, "Starting heal_object");
|
||||
info!(
|
||||
"SetDisks heal_object: bucket={}, object={}, version_id={}, opts={:?}",
|
||||
bucket, object, version_id, opts
|
||||
);
|
||||
let mut result = HealResultItem {
|
||||
heal_item_type: HealItemType::Object.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
@@ -2625,34 +2547,20 @@ impl SetDisks {
|
||||
if reuse_existing_lock {
|
||||
None
|
||||
} else {
|
||||
let mut lock_result = None;
|
||||
for i in 0..3 {
|
||||
let start_time = Instant::now();
|
||||
match self
|
||||
.fast_lock_manager
|
||||
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(duration = ?elapsed, attempt = i + 1, "Write lock acquired");
|
||||
lock_result = Some(res);
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(error = ?e, attempt = i + 1, duration = ?elapsed, "Lock acquisition failed, retrying");
|
||||
if i < 2 {
|
||||
tokio::time::sleep(Duration::from_millis(50 * (i as u64 + 1))).await;
|
||||
} else {
|
||||
let message = self.format_lock_error(bucket, object, "write", &e);
|
||||
error!("Failed to acquire write lock after retries: {}", message);
|
||||
return Err(DiskError::other(message));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lock_result
|
||||
let start_time = std::time::Instant::now();
|
||||
let lock_result = self
|
||||
.fast_lock_manager
|
||||
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let elapsed = start_time.elapsed();
|
||||
let message = self.format_lock_error(bucket, object, "write", &e);
|
||||
error!("Failed to acquire write lock for heal operation after {:?}: {}", elapsed, message);
|
||||
DiskError::other(message)
|
||||
})?;
|
||||
let elapsed = start_time.elapsed();
|
||||
info!("Successfully acquired write lock for object: {} in {:?}", object, elapsed);
|
||||
Some(lock_result)
|
||||
}
|
||||
} else {
|
||||
info!("Skipping lock acquisition (no_lock=true)");
|
||||
@@ -2669,37 +2577,8 @@ impl SetDisks {
|
||||
|
||||
let disks = { self.disks.read().await.clone() };
|
||||
|
||||
let (mut parts_metadata, errs) = {
|
||||
let mut retry_count = 0;
|
||||
loop {
|
||||
let (parts, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?;
|
||||
|
||||
// Check if we have enough valid metadata to proceed
|
||||
// If we have too many errors, and we haven't exhausted retries, try again
|
||||
let valid_count = errs.iter().filter(|e| e.is_none()).count();
|
||||
// Simple heuristic: if valid_count is less than expected quorum (e.g. half disks), retry
|
||||
// But we don't know the exact quorum yet. Let's just retry on high error rate if possible.
|
||||
// Actually, read_all_fileinfo shouldn't fail easily.
|
||||
// Let's just retry if we see ANY non-NotFound errors that might be transient (like timeouts)
|
||||
|
||||
let has_transient_error = errs
|
||||
.iter()
|
||||
.any(|e| matches!(e, Some(DiskError::SourceStalled) | Some(DiskError::Timeout)));
|
||||
|
||||
if !has_transient_error || retry_count >= 3 {
|
||||
break (parts, errs);
|
||||
}
|
||||
|
||||
info!(
|
||||
"read_all_fileinfo encountered transient errors, retrying (attempt {}/3). Errs: {:?}",
|
||||
retry_count + 1,
|
||||
errs
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(50 * (retry_count as u64 + 1))).await;
|
||||
retry_count += 1;
|
||||
}
|
||||
};
|
||||
info!(parts_count = parts_metadata.len(), ?errs, "File info read complete");
|
||||
let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?;
|
||||
info!("Read file info: parts_metadata.len()={}, errs={:?}", parts_metadata.len(), errs);
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
warn!(
|
||||
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
|
||||
@@ -2718,7 +2597,7 @@ impl SetDisks {
|
||||
));
|
||||
}
|
||||
|
||||
info!(parts_count = parts_metadata.len(), "Initiating quorum check");
|
||||
info!("About to call object_quorum_from_meta with parts_metadata.len()={}", parts_metadata.len());
|
||||
match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) {
|
||||
Ok((read_quorum, _)) => {
|
||||
result.parity_blocks = result.disk_count - read_quorum as usize;
|
||||
@@ -2743,12 +2622,10 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
"disks_with_all_parts results: available_disks count={}, total_disks={}",
|
||||
available_disks.iter().filter(|d| d.is_some()).count(),
|
||||
available_disks.len()
|
||||
);
|
||||
|
||||
// info!(
|
||||
// "disks_with_all_parts: got available_disks: {:?}, data_errs_by_disk: {:?}, data_errs_by_part: {:?}, latest_meta: {:?}",
|
||||
// available_disks, data_errs_by_disk, data_errs_by_part, latest_meta
|
||||
// );
|
||||
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
// Initialize erasure coding
|
||||
erasure_coding::Erasure::new(
|
||||
@@ -2768,7 +2645,10 @@ impl SetDisks {
|
||||
let mut outdate_disks = vec![None; disk_len];
|
||||
let mut disks_to_heal_count = 0;
|
||||
|
||||
info!("Checking {} disks for healing needs (bucket={}, object={})", disk_len, bucket, object);
|
||||
// info!(
|
||||
// "errs: {:?}, data_errs_by_disk: {:?}, latest_meta: {:?}",
|
||||
// errs, data_errs_by_disk, latest_meta
|
||||
// );
|
||||
for index in 0..available_disks.len() {
|
||||
let (yes, reason) = should_heal_object_on_disk(
|
||||
&errs[index],
|
||||
@@ -2776,16 +2656,9 @@ impl SetDisks {
|
||||
&parts_metadata[index],
|
||||
&latest_meta,
|
||||
);
|
||||
|
||||
info!(
|
||||
"Disk {} heal check: should_heal={}, reason={:?}, err={:?}, endpoint={}",
|
||||
index, yes, reason, errs[index], self.set_endpoints[index]
|
||||
);
|
||||
|
||||
if yes {
|
||||
outdate_disks[index] = disks[index].clone();
|
||||
disks_to_heal_count += 1;
|
||||
info!("Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]);
|
||||
}
|
||||
|
||||
let drive_state = match reason {
|
||||
@@ -2813,11 +2686,6 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
"Heal check complete: {} disks need healing out of {} total (bucket={}, object={})",
|
||||
disks_to_heal_count, disk_len, bucket, object
|
||||
);
|
||||
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
warn!(
|
||||
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
|
||||
@@ -2852,18 +2720,9 @@ impl SetDisks {
|
||||
);
|
||||
|
||||
if !latest_meta.deleted && disks_to_heal_count > latest_meta.erasure.parity_blocks {
|
||||
let total_disks = parts_metadata.len();
|
||||
let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
|
||||
let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks);
|
||||
|
||||
error!(
|
||||
"Data corruption detected for {}/{}: Insufficient healthy shards. Need at least {} data shards, but found only {} healthy disks. (Missing/Corrupt: {}, Parity: {})",
|
||||
bucket,
|
||||
object,
|
||||
required_data,
|
||||
healthy_count,
|
||||
disks_to_heal_count,
|
||||
latest_meta.erasure.parity_blocks
|
||||
"file({} : {}) part corrupt too much, can not to fix, disks_to_heal_count: {}, parity_blocks: {}",
|
||||
bucket, object, disks_to_heal_count, latest_meta.erasure.parity_blocks
|
||||
);
|
||||
|
||||
// Allow for dangling deletes, on versions that have DataDir missing etc.
|
||||
@@ -2895,7 +2754,7 @@ impl SetDisks {
|
||||
Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
|
||||
}
|
||||
Err(err) => {
|
||||
// t_errs = vec![Some(err.clone()]; errs.len());
|
||||
// t_errs = vec![Some(err.clone()); errs.len()];
|
||||
let mut t_errs = Vec::with_capacity(errs.len());
|
||||
for _ in 0..errs.len() {
|
||||
t_errs.push(Some(err.clone()));
|
||||
@@ -3050,7 +2909,7 @@ impl SetDisks {
|
||||
);
|
||||
for (index, disk) in latest_disks.iter().enumerate() {
|
||||
if let Some(outdated_disk) = &out_dated_disks[index] {
|
||||
info!(disk_index = index, "Creating writer for outdated disk");
|
||||
info!("Creating writer for index {} (outdated disk)", index);
|
||||
let writer = create_bitrot_writer(
|
||||
is_inline_buffer,
|
||||
Some(outdated_disk),
|
||||
@@ -3063,7 +2922,7 @@ impl SetDisks {
|
||||
.await?;
|
||||
writers.push(Some(writer));
|
||||
} else {
|
||||
info!(disk_index = index, "Skipping writer (disk not outdated)");
|
||||
info!("Skipping writer for index {} (not outdated)", index);
|
||||
writers.push(None);
|
||||
}
|
||||
|
||||
@@ -3073,7 +2932,7 @@ impl SetDisks {
|
||||
// // Box::new(Cursor::new(Vec::new()))
|
||||
// // } else {
|
||||
// // let disk = disk.clone();
|
||||
// // let part_path = format!("{}/{}/part.{}", object, src_data_dir, part.number);
|
||||
// // let part_path = format!("{}/{}/part.{}", tmp_id, dst_data_dir, part.number);
|
||||
// // disk.create_file("", RUSTFS_META_TMP_BUCKET, &part_path, 0).await?
|
||||
// // }
|
||||
// // };
|
||||
@@ -3169,12 +3028,6 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
// Rename from tmp location to the actual location.
|
||||
info!(
|
||||
"Starting rename phase: {} disks to process (bucket={}, object={})",
|
||||
out_dated_disks.iter().filter(|d| d.is_some()).count(),
|
||||
bucket,
|
||||
object
|
||||
);
|
||||
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(disk) = outdated_disk {
|
||||
// record the index of the updated disks
|
||||
@@ -3183,8 +3036,8 @@ impl SetDisks {
|
||||
parts_metadata[index].set_healing();
|
||||
|
||||
info!(
|
||||
"Renaming healed data for disk {} (endpoint={}): src_volume={}, src_path={}, dst_volume={}, dst_path={}",
|
||||
index, self.set_endpoints[index], RUSTFS_META_TMP_BUCKET, tmp_id, bucket, object
|
||||
"rename temp data, src_volume: {}, src_path: {}, dst_volume: {}, dst_path: {}",
|
||||
RUSTFS_META_TMP_BUCKET, tmp_id, bucket, object
|
||||
);
|
||||
let rename_result = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
|
||||
@@ -3192,15 +3045,10 @@ impl SetDisks {
|
||||
|
||||
if let Err(err) = &rename_result {
|
||||
info!(
|
||||
error = %err,
|
||||
disk_index = index,
|
||||
endpoint = %self.set_endpoints[index],
|
||||
"Rename failed, attempting fallback"
|
||||
"rename temp data err: {}. Try fallback to direct xl.meta overwrite...",
|
||||
err.to_string()
|
||||
);
|
||||
|
||||
// Preserve temp files for safety
|
||||
info!(temp_uuid = %tmp_id, "Rename failed, preserving temporary files for safety");
|
||||
|
||||
let healthy_index = latest_disks.iter().position(|d| d.is_some()).unwrap_or(0);
|
||||
|
||||
if let Some(healthy_disk) = &latest_disks[healthy_index] {
|
||||
@@ -3238,10 +3086,7 @@ impl SetDisks {
|
||||
));
|
||||
}
|
||||
} else {
|
||||
info!(
|
||||
"Successfully renamed healed data for disk {} (endpoint={}), removing temp files from volume={}, path={}",
|
||||
index, self.set_endpoints[index], RUSTFS_META_TMP_BUCKET, tmp_id
|
||||
);
|
||||
info!("remove temp object, volume: {}, path: {}", RUSTFS_META_TMP_BUCKET, tmp_id);
|
||||
|
||||
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id)
|
||||
.await
|
||||
@@ -3575,10 +3420,7 @@ impl SetDisks {
|
||||
}
|
||||
Ok(m)
|
||||
} else {
|
||||
error!(
|
||||
"Object {}/{} is corrupted but not dangling (some parts exist). Preserving data for potential manual recovery. Errors: {:?}",
|
||||
bucket, object, errs
|
||||
);
|
||||
error!("delete_if_dang_ling: is_object_dang_ling errs={:?}", errs);
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
}
|
||||
}
|
||||
@@ -3703,7 +3545,7 @@ impl ObjectIO for SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
// Acquire a shared read-lock early to protect read consistency
|
||||
let read_lock_guard = if !opts.no_lock {
|
||||
let _read_lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.fast_lock_manager
|
||||
.acquire_read_lock(bucket, object, self.locker_owner.as_str())
|
||||
@@ -3765,7 +3607,7 @@ impl ObjectIO for SetDisks {
|
||||
// Move the read-lock guard into the task so it lives for the duration of the read
|
||||
// let _guard_to_hold = _read_lock_guard; // moved into closure below
|
||||
tokio::spawn(async move {
|
||||
let _guard = read_lock_guard; // keep guard alive until task ends
|
||||
// let _guard = _guard_to_hold; // keep guard alive until task ends
|
||||
let mut writer = wd;
|
||||
if let Err(e) = Self::get_object_with_fileinfo(
|
||||
&bucket,
|
||||
@@ -4281,7 +4123,7 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
// Acquire locks in batch mode (best effort, matching previous behavior)
|
||||
let mut batch = rustfs_lock::BatchLockRequest::new(self.locker_owner.as_str()).with_all_or_nothing(false);
|
||||
let mut unique_objects: HashSet<String> = HashSet::new();
|
||||
let mut unique_objects: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
for dobj in &objects {
|
||||
if unique_objects.insert(dobj.object_name.clone()) {
|
||||
batch = batch.add_write_lock(bucket, dobj.object_name.clone());
|
||||
@@ -4296,7 +4138,7 @@ impl StorageAPI for SetDisks {
|
||||
.collect();
|
||||
let _lock_guards = batch_result.guards;
|
||||
|
||||
let failed_map: HashMap<(String, String), LockResult> = batch_result
|
||||
let failed_map: HashMap<(String, String), rustfs_lock::fast_lock::LockResult> = batch_result
|
||||
.failed_locks
|
||||
.into_iter()
|
||||
.map(|(key, err)| ((key.bucket.as_ref().to_string(), key.object.as_ref().to_string()), err))
|
||||
@@ -4384,6 +4226,7 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
for (_, mut fi_vers) in vers_map {
|
||||
fi_vers.versions.sort_by(|a, b| a.deleted.cmp(&b.deleted));
|
||||
fi_vers.versions.reverse();
|
||||
|
||||
if let Some(index) = fi_vers.versions.iter().position(|fi| fi.deleted) {
|
||||
fi_vers.versions.truncate(index + 1);
|
||||
@@ -4620,7 +4463,6 @@ impl StorageAPI for SetDisks {
|
||||
_max_keys: i32,
|
||||
_fetch_owner: bool,
|
||||
_start_after: Option<String>,
|
||||
_incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -4643,7 +4485,7 @@ impl StorageAPI for SetDisks {
|
||||
_rx: CancellationToken,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_result: Sender<ObjectInfoOrErr>,
|
||||
_result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
_opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
unimplemented!()
|
||||
@@ -4675,25 +4517,15 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||
if let Err(e) =
|
||||
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
error = %e,
|
||||
"Failed to enqueue heal request for partial object"
|
||||
);
|
||||
}
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
))
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4799,7 +4631,7 @@ impl StorageAPI for SetDisks {
|
||||
let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
return Err(Error::other(format!("remote tier error: {err}")));
|
||||
return Err(Error::other(format!("remote tier error: {}", err)));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -4979,11 +4811,11 @@ impl StorageAPI for SetDisks {
|
||||
false,
|
||||
)?;
|
||||
let mut p_reader = PutObjReader::new(hash_reader);
|
||||
return if let Err(err) = self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
|
||||
set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await
|
||||
if let Err(err) = self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
|
||||
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let res = self_.clone().new_multipart_upload(bucket, object, &ropts).await?;
|
||||
@@ -5440,6 +5272,7 @@ impl StorageAPI for SetDisks {
|
||||
if err == DiskError::DiskNotFound {
|
||||
None
|
||||
} else if err == DiskError::FileNotFound {
|
||||
warn!("list_multipart_uploads: FileNotFound");
|
||||
return Ok(ListMultipartsInfo {
|
||||
key_marker: key_marker.to_owned(),
|
||||
max_uploads,
|
||||
@@ -5702,17 +5535,6 @@ impl StorageAPI for SetDisks {
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let _object_lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.fast_lock_manager
|
||||
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
|
||||
.await
|
||||
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
|
||||
@@ -5831,7 +5653,7 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
|
||||
let ext_part = &curr_fi.parts[i];
|
||||
info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part");
|
||||
tracing::info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part");
|
||||
|
||||
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
|
||||
let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e));
|
||||
@@ -6087,7 +5909,7 @@ impl StorageAPI for SetDisks {
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(rustfs_common::heal_channel::HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
))
|
||||
@@ -6147,55 +5969,6 @@ impl StorageAPI for SetDisks {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let mut effective_object = object.to_string();
|
||||
|
||||
// Optimization: Only attempt correction if the name looks suspicious (quotes or URL encoded)
|
||||
// and the original object does NOT exist.
|
||||
let has_quotes = (effective_object.starts_with('\'') && effective_object.ends_with('\''))
|
||||
|| (effective_object.starts_with('"') && effective_object.ends_with('"'));
|
||||
let has_percent = effective_object.contains('%');
|
||||
|
||||
if has_quotes || has_percent {
|
||||
let disks = self.disks.read().await;
|
||||
// 1. Check if the original object exists (lightweight check)
|
||||
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, &effective_object, version_id, false, false).await?;
|
||||
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
// Original not found. Try candidates.
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
// Candidate 1: URL Decoded (Priority for web access issues)
|
||||
if has_percent {
|
||||
if let Ok(decoded) = urlencoding::decode(&effective_object) {
|
||||
if decoded != effective_object {
|
||||
candidates.push(decoded.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Candidate 2: Quote Stripped (For shell copy-paste issues)
|
||||
if has_quotes && effective_object.len() >= 2 {
|
||||
candidates.push(effective_object[1..effective_object.len() - 1].to_string());
|
||||
}
|
||||
|
||||
// Check candidates
|
||||
for candidate in candidates {
|
||||
let (_, errs_cand) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, &candidate, version_id, false, false).await?;
|
||||
|
||||
if !DiskError::is_all_not_found(&errs_cand) {
|
||||
info!(
|
||||
"Heal request for object '{}' failed (not found). Auto-corrected to '{}'.",
|
||||
effective_object, candidate
|
||||
);
|
||||
effective_object = candidate;
|
||||
break; // Found a match, stop searching
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let object = effective_object.as_str();
|
||||
|
||||
let _write_lock_guard = if !opts.no_lock {
|
||||
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
|
||||
let mut skip_lock = false;
|
||||
@@ -6210,10 +5983,10 @@ impl StorageAPI for SetDisks {
|
||||
skip_lock = true;
|
||||
}
|
||||
}
|
||||
|
||||
if skip_lock {
|
||||
None
|
||||
} else {
|
||||
info!(?opts, "Starting heal_object");
|
||||
Some(
|
||||
self.fast_lock_manager
|
||||
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
|
||||
|
||||
@@ -111,9 +111,6 @@ impl Sets {
|
||||
|
||||
let mut disk_set = Vec::with_capacity(set_count);
|
||||
|
||||
// Create fast lock manager for high performance
|
||||
let fast_lock_manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
|
||||
|
||||
for i in 0..set_count {
|
||||
let mut set_drive = Vec::with_capacity(set_drive_count);
|
||||
let mut set_endpoints = Vec::with_capacity(set_drive_count);
|
||||
@@ -167,9 +164,11 @@ impl Sets {
|
||||
|
||||
// Note: write_quorum was used for the old lock system, no longer needed with FastLock
|
||||
let _write_quorum = set_drive_count - parity_count;
|
||||
// Create fast lock manager for high performance
|
||||
let fast_lock_manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
fast_lock_manager.clone(),
|
||||
fast_lock_manager,
|
||||
GLOBAL_Local_Node_Name.read().await.to_string(),
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
set_drive_count,
|
||||
@@ -440,7 +439,6 @@ impl StorageAPI for Sets {
|
||||
_max_keys: i32,
|
||||
_fetch_owner: bool,
|
||||
_start_after: Option<String>,
|
||||
_incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
@@ -88,6 +88,8 @@ pub struct ECStore {
|
||||
pub pool_meta: RwLock<PoolMeta>,
|
||||
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
|
||||
pub decommission_cancelers: Vec<Option<usize>>,
|
||||
// mpCache: cache for MultipartUploadResult, key is "bucket/object"
|
||||
pub mp_cache: Arc<RwLock<HashMap<String, MultipartUploadResult>>>,
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -240,6 +242,7 @@ impl ECStore {
|
||||
pool_meta: RwLock::new(pool_meta),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers,
|
||||
mp_cache: Arc::new(RwLock::new(HashMap::new())),
|
||||
});
|
||||
|
||||
// Only set it when the global deployment ID is not yet configured
|
||||
@@ -1338,19 +1341,9 @@ impl StorageAPI for ECStore {
|
||||
max_keys: i32,
|
||||
fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info> {
|
||||
self.inner_list_objects_v2(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token,
|
||||
delimiter,
|
||||
max_keys,
|
||||
fetch_owner,
|
||||
start_after,
|
||||
incl_deleted,
|
||||
)
|
||||
.await
|
||||
self.inner_list_objects_v2(bucket, prefix, continuation_token, delimiter, max_keys, fetch_owner, start_after)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1773,8 +1766,58 @@ impl StorageAPI for ECStore {
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
check_list_multipart_args(bucket, prefix, &key_marker, &upload_id_marker, &delimiter)?;
|
||||
|
||||
// Return from cache if prefix is empty (list all multipart uploads for the bucket)
|
||||
if prefix.is_empty() {
|
||||
// TODO: return from cache
|
||||
let cache = self.mp_cache.read().await;
|
||||
let mut cached_uploads = Vec::new();
|
||||
let bucket_prefix = format!("{}/", bucket);
|
||||
|
||||
for (key, result) in cache.iter() {
|
||||
if key.starts_with(&bucket_prefix) {
|
||||
let object = key.strip_prefix(&bucket_prefix).unwrap_or("");
|
||||
if let Some(key_marker_val) = &key_marker {
|
||||
if object < key_marker_val.as_str() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(upload_id_marker_val) = &upload_id_marker {
|
||||
if object == key_marker.as_deref().unwrap_or("") && result.upload_id < *upload_id_marker_val {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
cached_uploads.push(MultipartInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
upload_id: result.upload_id.clone(),
|
||||
initiated: None,
|
||||
user_defined: HashMap::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by object name and upload_id
|
||||
cached_uploads.sort_by(|a, b| match a.object.cmp(&b.object) {
|
||||
Ordering::Equal => a.upload_id.cmp(&b.upload_id),
|
||||
other => other,
|
||||
});
|
||||
|
||||
// Apply max_uploads limit
|
||||
if cached_uploads.len() > max_uploads {
|
||||
cached_uploads.truncate(max_uploads);
|
||||
}
|
||||
|
||||
if !cached_uploads.is_empty() {
|
||||
return Ok(ListMultipartsInfo {
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
max_uploads,
|
||||
uploads: cached_uploads,
|
||||
prefix: prefix.to_owned(),
|
||||
delimiter: delimiter.to_owned(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -1817,8 +1860,15 @@ impl StorageAPI for ECStore {
|
||||
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
|
||||
check_new_multipart_args(bucket, object)?;
|
||||
|
||||
let encoded_object = encode_dir_object(object);
|
||||
let cache_key = format!("{}/{}", bucket, encoded_object);
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].new_multipart_upload(bucket, object, opts).await;
|
||||
let result = self.pools[0].new_multipart_upload(bucket, &encoded_object, opts).await?;
|
||||
// Cache the result
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.insert(cache_key, result.clone());
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
for (idx, pool) in self.pools.iter().enumerate() {
|
||||
@@ -1826,14 +1876,18 @@ impl StorageAPI for ECStore {
|
||||
continue;
|
||||
}
|
||||
let res = pool
|
||||
.list_multipart_uploads(bucket, object, None, None, None, MAX_UPLOADS_LIST)
|
||||
.list_multipart_uploads(bucket, &encoded_object, None, None, None, MAX_UPLOADS_LIST)
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
|
||||
let result = self.pools[idx].new_multipart_upload(bucket, &encoded_object, opts).await?;
|
||||
// Cache the result
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.insert(cache_key, result.clone());
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
let idx = self.get_pool_idx(bucket, object, -1).await?;
|
||||
let idx = self.get_pool_idx(bucket, &encoded_object, -1).await?;
|
||||
if opts.data_movement && idx == opts.src_pool_idx {
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
bucket.to_owned(),
|
||||
@@ -1842,7 +1896,11 @@ impl StorageAPI for ECStore {
|
||||
));
|
||||
}
|
||||
|
||||
self.pools[idx].new_multipart_upload(bucket, object, opts).await
|
||||
let result = self.pools[idx].new_multipart_upload(bucket, &encoded_object, opts).await?;
|
||||
// Cache the result
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.insert(cache_key, result.clone());
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1993,8 +2051,19 @@ impl StorageAPI for ECStore {
|
||||
|
||||
// TODO: defer DeleteUploadID
|
||||
|
||||
let encoded_object = encode_dir_object(object);
|
||||
let cache_key = format!("{}/{}", bucket, encoded_object);
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
|
||||
let result = self.pools[0]
|
||||
.abort_multipart_upload(bucket, &encoded_object, upload_id, opts)
|
||||
.await;
|
||||
// Remove from cache on success
|
||||
if result.is_ok() {
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.remove(&cache_key);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
@@ -2002,8 +2071,13 @@ impl StorageAPI for ECStore {
|
||||
continue;
|
||||
}
|
||||
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
|
||||
Ok(_) => return Ok(()),
|
||||
let err = match pool.abort_multipart_upload(bucket, &encoded_object, upload_id, opts).await {
|
||||
Ok(_) => {
|
||||
// Remove from cache on success
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.remove(&cache_key);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) => {
|
||||
//
|
||||
if is_err_invalid_upload_id(&err) { None } else { Some(err) }
|
||||
@@ -2029,11 +2103,20 @@ impl StorageAPI for ECStore {
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
|
||||
let encoded_object = encode_dir_object(object);
|
||||
let cache_key = format!("{}/{}", bucket, encoded_object);
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0]
|
||||
let result = self.pools[0]
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.complete_multipart_upload(bucket, &encoded_object, upload_id, uploaded_parts, opts)
|
||||
.await;
|
||||
// Remove from cache on success
|
||||
if result.is_ok() {
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.remove(&cache_key);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
@@ -2043,10 +2126,15 @@ impl StorageAPI for ECStore {
|
||||
|
||||
let pool = pool.clone();
|
||||
let err = match pool
|
||||
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
|
||||
.complete_multipart_upload(bucket, &encoded_object, upload_id, uploaded_parts.clone(), opts)
|
||||
.await
|
||||
{
|
||||
Ok(res) => return Ok(res),
|
||||
Ok(res) => {
|
||||
// Remove from cache on success
|
||||
let mut cache = self.mp_cache.write().await;
|
||||
cache.remove(&cache_key);
|
||||
return Ok(res);
|
||||
}
|
||||
Err(err) => {
|
||||
//
|
||||
if is_err_invalid_upload_id(&err) { None } else { Some(err) }
|
||||
|
||||
@@ -1224,7 +1224,6 @@ pub trait StorageAPI: ObjectIO + Debug {
|
||||
max_keys: i32,
|
||||
fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info>;
|
||||
// ListObjectVersions TODO: FIXME:
|
||||
async fn list_object_versions(
|
||||
|
||||
@@ -225,7 +225,6 @@ impl ECStore {
|
||||
max_keys: i32,
|
||||
_fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info> {
|
||||
let marker = {
|
||||
if continuation_token.is_none() {
|
||||
@@ -235,9 +234,7 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
|
||||
let loi = self
|
||||
.list_objects_generic(bucket, prefix, marker, delimiter, max_keys, incl_deleted)
|
||||
.await?;
|
||||
let loi = self.list_objects_generic(bucket, prefix, marker, delimiter, max_keys).await?;
|
||||
Ok(ListObjectsV2Info {
|
||||
is_truncated: loi.is_truncated,
|
||||
continuation_token,
|
||||
@@ -254,7 +251,6 @@ impl ECStore {
|
||||
marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
incl_deleted: bool,
|
||||
) -> Result<ListObjectsInfo> {
|
||||
let opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
@@ -262,7 +258,7 @@ impl ECStore {
|
||||
separator: delimiter.clone(),
|
||||
limit: max_keys_plus_one(max_keys, marker.is_some()),
|
||||
marker,
|
||||
incl_deleted,
|
||||
incl_deleted: false,
|
||||
ask_disks: "strict".to_owned(), //TODO: from config
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -26,7 +26,7 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/"
|
||||
|
||||
[dependencies]
|
||||
crc-fast = { workspace = true }
|
||||
crc32fast = { workspace = true }
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
@@ -220,11 +220,7 @@ impl FileInfo {
|
||||
let indices = {
|
||||
let cardinality = data_blocks + parity_blocks;
|
||||
let mut nums = vec![0; cardinality];
|
||||
let key_crc = {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(object.as_bytes());
|
||||
hasher.finalize() as u32
|
||||
};
|
||||
let key_crc = crc32fast::hash(object.as_bytes());
|
||||
|
||||
let start = key_crc as usize % cardinality;
|
||||
for i in 1..=cardinality {
|
||||
|
||||
@@ -427,19 +427,11 @@ impl FileMeta {
|
||||
return;
|
||||
}
|
||||
|
||||
self.versions.sort_by(|a, b| {
|
||||
if a.header.mod_time != b.header.mod_time {
|
||||
b.header.mod_time.cmp(&a.header.mod_time)
|
||||
} else if a.header.version_type != b.header.version_type {
|
||||
b.header.version_type.cmp(&a.header.version_type)
|
||||
} else if a.header.version_id != b.header.version_id {
|
||||
b.header.version_id.cmp(&a.header.version_id)
|
||||
} else if a.header.flags != b.header.flags {
|
||||
b.header.flags.cmp(&a.header.flags)
|
||||
} else {
|
||||
b.cmp(a)
|
||||
}
|
||||
});
|
||||
self.versions.reverse();
|
||||
|
||||
// for _v in self.versions.iter() {
|
||||
// // warn!("sort {} {:?}", i, v);
|
||||
// }
|
||||
}
|
||||
|
||||
// Find version
|
||||
@@ -497,27 +489,25 @@ impl FileMeta {
|
||||
|
||||
self.versions.sort_by(|a, b| {
|
||||
if a.header.mod_time != b.header.mod_time {
|
||||
b.header.mod_time.cmp(&a.header.mod_time)
|
||||
a.header.mod_time.cmp(&b.header.mod_time)
|
||||
} else if a.header.version_type != b.header.version_type {
|
||||
b.header.version_type.cmp(&a.header.version_type)
|
||||
a.header.version_type.cmp(&b.header.version_type)
|
||||
} else if a.header.version_id != b.header.version_id {
|
||||
b.header.version_id.cmp(&a.header.version_id)
|
||||
a.header.version_id.cmp(&b.header.version_id)
|
||||
} else if a.header.flags != b.header.flags {
|
||||
b.header.flags.cmp(&a.header.flags)
|
||||
a.header.flags.cmp(&b.header.flags)
|
||||
} else {
|
||||
b.cmp(a)
|
||||
a.cmp(b)
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> {
|
||||
if fi.version_id.is_none() {
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
}
|
||||
pub fn add_version(&mut self, fi: FileInfo) -> Result<()> {
|
||||
let vid = fi.version_id;
|
||||
|
||||
if let Some(ref data) = fi.data {
|
||||
let key = fi.version_id.unwrap_or_default().to_string();
|
||||
let key = vid.unwrap_or_default().to_string();
|
||||
self.data.replace(&key, data.to_vec())?;
|
||||
}
|
||||
|
||||
@@ -531,13 +521,12 @@ impl FileMeta {
|
||||
return Err(Error::other("file meta version invalid"));
|
||||
}
|
||||
|
||||
// TODO: make it configurable
|
||||
// 1000 is the limit of versions
|
||||
// if self.versions.len() + 1 > 1000 {
|
||||
// return Err(Error::other(
|
||||
// "You've exceeded the limit on the number of versions you can create on this object",
|
||||
// ));
|
||||
// }
|
||||
// 1000 is the limit of versions TODO: make it configurable
|
||||
if self.versions.len() + 1 > 1000 {
|
||||
return Err(Error::other(
|
||||
"You've exceeded the limit on the number of versions you can create on this object",
|
||||
));
|
||||
}
|
||||
|
||||
if self.versions.is_empty() {
|
||||
self.versions.push(FileMetaShallowVersion::try_from(version)?);
|
||||
@@ -562,6 +551,7 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::other("add_version failed"))
|
||||
|
||||
// if !ver.valid() {
|
||||
@@ -593,19 +583,12 @@ impl FileMeta {
|
||||
}
|
||||
|
||||
// delete_version deletes version, returns data_dir
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
|
||||
let vid = if fi.version_id.is_none() {
|
||||
Some(Uuid::nil())
|
||||
} else {
|
||||
Some(fi.version_id.unwrap())
|
||||
};
|
||||
|
||||
let mut ventry = FileMetaVersion::default();
|
||||
if fi.deleted {
|
||||
ventry.version_type = VersionType::Delete;
|
||||
ventry.delete_marker = Some(MetaDeleteMarker {
|
||||
version_id: vid,
|
||||
version_id: fi.version_id,
|
||||
mod_time: fi.mod_time,
|
||||
..Default::default()
|
||||
});
|
||||
@@ -706,10 +689,8 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
let mut found_index = None;
|
||||
|
||||
for (i, ver) in self.versions.iter().enumerate() {
|
||||
if ver.header.version_id != vid {
|
||||
if ver.header.version_id != fi.version_id {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -720,7 +701,7 @@ impl FileMeta {
|
||||
let mut v = self.get_idx(i)?;
|
||||
if v.delete_marker.is_none() {
|
||||
v.delete_marker = Some(MetaDeleteMarker {
|
||||
version_id: vid,
|
||||
version_id: fi.version_id,
|
||||
mod_time: fi.mod_time,
|
||||
meta_sys: HashMap::new(),
|
||||
});
|
||||
@@ -786,7 +767,7 @@ impl FileMeta {
|
||||
self.versions.remove(i);
|
||||
|
||||
if (fi.mark_deleted && fi.version_purge_status() != VersionPurgeStatusType::Complete)
|
||||
|| (fi.deleted && vid == Some(Uuid::nil()))
|
||||
|| (fi.deleted && fi.version_id.is_none())
|
||||
{
|
||||
self.add_version_filemata(ventry)?;
|
||||
}
|
||||
@@ -822,11 +803,18 @@ impl FileMeta {
|
||||
|
||||
return Ok(old_dir);
|
||||
}
|
||||
found_index = Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut found_index = None;
|
||||
for (i, version) in self.versions.iter().enumerate() {
|
||||
if version.header.version_type == VersionType::Object && version.header.version_id == fi.version_id {
|
||||
found_index = Some(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(i) = found_index else {
|
||||
if fi.deleted {
|
||||
self.add_version_filemata(ventry)?;
|
||||
@@ -1533,8 +1521,7 @@ impl FileMetaVersionHeader {
|
||||
cur.read_exact(&mut buf)?;
|
||||
self.version_id = {
|
||||
let id = Uuid::from_bytes(buf);
|
||||
// if id.is_nil() { None } else { Some(id) }
|
||||
Some(id)
|
||||
if id.is_nil() { None } else { Some(id) }
|
||||
};
|
||||
|
||||
// mod_time
|
||||
@@ -1708,7 +1695,7 @@ impl MetaObject {
|
||||
}
|
||||
|
||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> FileInfo {
|
||||
// let version_id = self.version_id.filter(|&vid| !vid.is_nil());
|
||||
let version_id = self.version_id.filter(|&vid| !vid.is_nil());
|
||||
|
||||
let parts = if all_parts {
|
||||
let mut parts = vec![ObjectPartInfo::default(); self.part_numbers.len()];
|
||||
@@ -1812,7 +1799,7 @@ impl MetaObject {
|
||||
.unwrap_or_default();
|
||||
|
||||
FileInfo {
|
||||
version_id: self.version_id,
|
||||
version_id,
|
||||
erasure,
|
||||
data_dir: self.data_dir,
|
||||
mod_time: self.mod_time,
|
||||
|
||||
@@ -37,6 +37,7 @@ serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
|
||||
# Cryptography
|
||||
aes-gcm = { workspace = true }
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! API types for KMS dynamic configuration
|
||||
|
||||
use crate::config::{BackendConfig, CacheConfig, KmsBackend, KmsConfig, LocalConfig, TlsConfig, VaultAuthMethod, VaultConfig};
|
||||
use crate::config::{KmsBackend, KmsConfig, VaultAuthMethod};
|
||||
use crate::service_manager::KmsServiceStatus;
|
||||
use crate::types::{KeyMetadata, KeyUsage};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -212,12 +212,12 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
};
|
||||
|
||||
let backend_summary = match &config.backend_config {
|
||||
BackendConfig::Local(local_config) => BackendSummary::Local {
|
||||
crate::config::BackendConfig::Local(local_config) => BackendSummary::Local {
|
||||
key_dir: local_config.key_dir.clone(),
|
||||
has_master_key: local_config.master_key.is_some(),
|
||||
file_permissions: local_config.file_permissions,
|
||||
},
|
||||
BackendConfig::Vault(vault_config) => BackendSummary::Vault {
|
||||
crate::config::BackendConfig::Vault(vault_config) => BackendSummary::Vault {
|
||||
address: vault_config.address.clone(),
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
@@ -248,7 +248,7 @@ impl ConfigureLocalKmsRequest {
|
||||
KmsConfig {
|
||||
backend: KmsBackend::Local,
|
||||
default_key_id: self.default_key_id.clone(),
|
||||
backend_config: BackendConfig::Local(LocalConfig {
|
||||
backend_config: crate::config::BackendConfig::Local(crate::config::LocalConfig {
|
||||
key_dir: self.key_dir.clone(),
|
||||
master_key: self.master_key.clone(),
|
||||
file_permissions: self.file_permissions,
|
||||
@@ -256,7 +256,7 @@ impl ConfigureLocalKmsRequest {
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
cache_config: CacheConfig {
|
||||
cache_config: crate::config::CacheConfig {
|
||||
max_keys: self.max_cached_keys.unwrap_or(1000),
|
||||
ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
|
||||
enable_metrics: true,
|
||||
@@ -271,7 +271,7 @@ impl ConfigureVaultKmsRequest {
|
||||
KmsConfig {
|
||||
backend: KmsBackend::Vault,
|
||||
default_key_id: self.default_key_id.clone(),
|
||||
backend_config: BackendConfig::Vault(VaultConfig {
|
||||
backend_config: crate::config::BackendConfig::Vault(crate::config::VaultConfig {
|
||||
address: self.address.clone(),
|
||||
auth_method: self.auth_method.clone(),
|
||||
namespace: self.namespace.clone(),
|
||||
@@ -279,7 +279,7 @@ impl ConfigureVaultKmsRequest {
|
||||
kv_mount: self.kv_mount.clone().unwrap_or_else(|| "secret".to_string()),
|
||||
key_path_prefix: self.key_path_prefix.clone().unwrap_or_else(|| "rustfs/kms/keys".to_string()),
|
||||
tls: if self.skip_tls_verify.unwrap_or(false) {
|
||||
Some(TlsConfig {
|
||||
Some(crate::config::TlsConfig {
|
||||
ca_cert_path: None,
|
||||
client_cert_path: None,
|
||||
client_key_path: None,
|
||||
@@ -292,7 +292,7 @@ impl ConfigureVaultKmsRequest {
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
cache_config: CacheConfig {
|
||||
cache_config: crate::config::CacheConfig {
|
||||
max_keys: self.max_cached_keys.unwrap_or(1000),
|
||||
ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
|
||||
enable_metrics: true,
|
||||
|
||||
@@ -200,16 +200,6 @@ pub struct BackendInfo {
|
||||
|
||||
impl BackendInfo {
|
||||
/// Create a new backend info
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `backend_type` - The type of the backend
|
||||
/// * `version` - The version of the backend
|
||||
/// * `endpoint` - The endpoint or location of the backend
|
||||
/// * `healthy` - Whether the backend is healthy
|
||||
///
|
||||
/// # Returns
|
||||
/// A new BackendInfo instance
|
||||
///
|
||||
pub fn new(backend_type: String, version: String, endpoint: String, healthy: bool) -> Self {
|
||||
Self {
|
||||
backend_type,
|
||||
@@ -221,14 +211,6 @@ impl BackendInfo {
|
||||
}
|
||||
|
||||
/// Add metadata to the backend info
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Metadata key
|
||||
/// * `value` - Metadata value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated BackendInfo instance
|
||||
///
|
||||
pub fn with_metadata(mut self, key: String, value: String) -> Self {
|
||||
self.metadata.insert(key, value);
|
||||
self
|
||||
|
||||
@@ -34,13 +34,6 @@ pub struct KmsCache {
|
||||
|
||||
impl KmsCache {
|
||||
/// Create a new KMS cache with the specified capacity
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `capacity` - Maximum number of entries in the cache
|
||||
///
|
||||
/// # Returns
|
||||
/// A new instance of `KmsCache`
|
||||
///
|
||||
pub fn new(capacity: u64) -> Self {
|
||||
Self {
|
||||
key_metadata_cache: Cache::builder()
|
||||
@@ -55,47 +48,22 @@ impl KmsCache {
|
||||
}
|
||||
|
||||
/// Get key metadata from cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to retrieve metadata for
|
||||
///
|
||||
/// # Returns
|
||||
/// An `Option` containing the `KeyMetadata` if found, or `None` if not found
|
||||
///
|
||||
pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
|
||||
self.key_metadata_cache.get(key_id).await
|
||||
}
|
||||
|
||||
/// Put key metadata into cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to store metadata for
|
||||
/// * `metadata` - The `KeyMetadata` to store in the cache
|
||||
///
|
||||
pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
|
||||
self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
|
||||
self.key_metadata_cache.run_pending_tasks().await;
|
||||
}
|
||||
|
||||
/// Get data key from cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to retrieve the data key for
|
||||
///
|
||||
/// # Returns
|
||||
/// An `Option` containing the `CachedDataKey` if found, or `None` if not found
|
||||
///
|
||||
pub async fn get_data_key(&self, key_id: &str) -> Option<CachedDataKey> {
|
||||
self.data_key_cache.get(key_id).await
|
||||
}
|
||||
|
||||
/// Put data key into cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to store the data key for
|
||||
/// * `plaintext` - The plaintext data key bytes
|
||||
/// * `ciphertext` - The ciphertext data key bytes
|
||||
///
|
||||
pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) {
|
||||
let cached_key = CachedDataKey {
|
||||
plaintext: plaintext.to_vec(),
|
||||
@@ -107,19 +75,11 @@ impl KmsCache {
|
||||
}
|
||||
|
||||
/// Remove key metadata from cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to remove metadata for
|
||||
///
|
||||
pub async fn remove_key_metadata(&mut self, key_id: &str) {
|
||||
self.key_metadata_cache.remove(key_id).await;
|
||||
}
|
||||
|
||||
/// Remove data key from cache
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - The ID of the key to remove the data key for
|
||||
///
|
||||
pub async fn remove_data_key(&mut self, key_id: &str) {
|
||||
self.data_key_cache.remove(key_id).await;
|
||||
}
|
||||
@@ -135,10 +95,6 @@ impl KmsCache {
|
||||
}
|
||||
|
||||
/// Get cache statistics (hit count, miss count)
|
||||
///
|
||||
/// # Returns
|
||||
/// A tuple containing total entries and total misses
|
||||
///
|
||||
pub fn stats(&self) -> (u64, u64) {
|
||||
let metadata_stats = (
|
||||
self.key_metadata_cache.entry_count(),
|
||||
|
||||
@@ -52,16 +52,6 @@ pub struct AesCipher {
|
||||
|
||||
impl AesCipher {
|
||||
/// Create a new AES cipher with the given key
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `key` - A byte slice representing the AES-256 key (32 bytes)
|
||||
///
|
||||
/// #Errors
|
||||
/// Returns `KmsError` if the key size is invalid
|
||||
///
|
||||
/// #Returns
|
||||
/// A Result containing the AesCipher instance
|
||||
///
|
||||
pub fn new(key: &[u8]) -> Result<Self> {
|
||||
if key.len() != 32 {
|
||||
return Err(KmsError::invalid_key_size(32, key.len()));
|
||||
@@ -152,16 +142,6 @@ pub struct ChaCha20Cipher {
|
||||
|
||||
impl ChaCha20Cipher {
|
||||
/// Create a new ChaCha20 cipher with the given key
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `key` - A byte slice representing the ChaCha20-Poly1305 key (32 bytes)
|
||||
///
|
||||
/// #Errors
|
||||
/// Returns `KmsError` if the key size is invalid
|
||||
///
|
||||
/// #Returns
|
||||
/// A Result containing the ChaCha20Cipher instance
|
||||
///
|
||||
pub fn new(key: &[u8]) -> Result<Self> {
|
||||
if key.len() != 32 {
|
||||
return Err(KmsError::invalid_key_size(32, key.len()));
|
||||
@@ -248,14 +228,6 @@ impl ObjectCipher for ChaCha20Cipher {
|
||||
}
|
||||
|
||||
/// Create a cipher instance for the given algorithm and key
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `algorithm` - The encryption algorithm to use
|
||||
/// * `key` - A byte slice representing the encryption key
|
||||
///
|
||||
/// #Returns
|
||||
/// A Result containing a boxed ObjectCipher instance
|
||||
///
|
||||
pub fn create_cipher(algorithm: &EncryptionAlgorithm, key: &[u8]) -> Result<Box<dyn ObjectCipher>> {
|
||||
match algorithm {
|
||||
EncryptionAlgorithm::Aes256 | EncryptionAlgorithm::AwsKms => Ok(Box::new(AesCipher::new(key)?)),
|
||||
@@ -264,13 +236,6 @@ pub fn create_cipher(algorithm: &EncryptionAlgorithm, key: &[u8]) -> Result<Box<
|
||||
}
|
||||
|
||||
/// Generate a random IV for the given algorithm
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `algorithm` - The encryption algorithm for which to generate the IV
|
||||
///
|
||||
/// #Returns
|
||||
/// A vector containing the generated IV bytes
|
||||
///
|
||||
pub fn generate_iv(algorithm: &EncryptionAlgorithm) -> Vec<u8> {
|
||||
let iv_size = match algorithm {
|
||||
EncryptionAlgorithm::Aes256 | EncryptionAlgorithm::AwsKms => 12,
|
||||
|
||||
@@ -18,12 +18,6 @@ use crate::encryption::ciphers::{create_cipher, generate_iv};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use base64::Engine;
|
||||
use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, info};
|
||||
use zeroize::Zeroize;
|
||||
|
||||
/// Data key for object encryption
|
||||
@@ -42,6 +36,12 @@ impl Drop for DataKey {
|
||||
self.plaintext_key.zeroize();
|
||||
}
|
||||
}
|
||||
use base64::Engine;
|
||||
use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Service for encrypting and decrypting S3 objects with KMS integration
|
||||
pub struct ObjectEncryptionService {
|
||||
@@ -59,110 +59,51 @@ pub struct EncryptionResult {
|
||||
|
||||
impl ObjectEncryptionService {
|
||||
/// Create a new object encryption service
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `kms_manager` - KMS manager to use for key operations
|
||||
///
|
||||
/// # Returns
|
||||
/// New ObjectEncryptionService instance
|
||||
///
|
||||
pub fn new(kms_manager: KmsManager) -> Self {
|
||||
Self { kms_manager }
|
||||
}
|
||||
|
||||
/// Create a new master key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - CreateKeyRequest with key parameters
|
||||
///
|
||||
/// # Returns
|
||||
/// CreateKeyResponse with created key details
|
||||
///
|
||||
pub async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
self.kms_manager.create_key(request).await
|
||||
}
|
||||
|
||||
/// Describe a master key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - DescribeKeyRequest with key ID
|
||||
///
|
||||
/// # Returns
|
||||
/// DescribeKeyResponse with key metadata
|
||||
///
|
||||
pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
|
||||
self.kms_manager.describe_key(request).await
|
||||
}
|
||||
|
||||
/// List master keys (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - ListKeysRequest with listing parameters
|
||||
///
|
||||
/// # Returns
|
||||
/// ListKeysResponse with list of keys
|
||||
///
|
||||
pub async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
|
||||
self.kms_manager.list_keys(request).await
|
||||
}
|
||||
|
||||
/// Generate a data encryption key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - GenerateDataKeyRequest with key parameters
|
||||
///
|
||||
/// # Returns
|
||||
/// GenerateDataKeyResponse with generated key details
|
||||
///
|
||||
pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
|
||||
self.kms_manager.generate_data_key(request).await
|
||||
}
|
||||
|
||||
/// Get the default key ID
|
||||
///
|
||||
/// # Returns
|
||||
/// Option with default key ID if configured
|
||||
///
|
||||
pub fn get_default_key_id(&self) -> Option<&String> {
|
||||
self.kms_manager.get_default_key_id()
|
||||
}
|
||||
|
||||
/// Get cache statistics
|
||||
///
|
||||
/// # Returns
|
||||
/// Option with (hits, misses) if caching is enabled
|
||||
///
|
||||
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
|
||||
self.kms_manager.cache_stats().await
|
||||
}
|
||||
|
||||
/// Clear the cache
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating success or failure
|
||||
///
|
||||
pub async fn clear_cache(&self) -> Result<()> {
|
||||
self.kms_manager.clear_cache().await
|
||||
}
|
||||
|
||||
/// Get backend health status
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating if backend is healthy
|
||||
///
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
self.kms_manager.health_check().await
|
||||
}
|
||||
|
||||
/// Create a data encryption key for object encryption
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `kms_key_id` - Optional KMS key ID to use (uses default if None)
|
||||
/// * `context` - ObjectEncryptionContext with bucket and object key
|
||||
///
|
||||
/// # Returns
|
||||
/// Tuple with DataKey and encrypted key blob
|
||||
///
|
||||
pub async fn create_data_key(
|
||||
&self,
|
||||
kms_key_id: &Option<String>,
|
||||
@@ -205,14 +146,6 @@ impl ObjectEncryptionService {
|
||||
}
|
||||
|
||||
/// Decrypt a data encryption key
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `encrypted_key` - Encrypted data key blob
|
||||
/// * `context` - ObjectEncryptionContext with bucket and object key
|
||||
///
|
||||
/// # Returns
|
||||
/// DataKey with decrypted key
|
||||
///
|
||||
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], _context: &ObjectEncryptionContext) -> Result<DataKey> {
|
||||
let decrypt_request = DecryptRequest {
|
||||
ciphertext: encrypted_key.to_vec(),
|
||||
@@ -496,17 +429,6 @@ impl ObjectEncryptionService {
|
||||
}
|
||||
|
||||
/// Decrypt object with customer-provided key (SSE-C)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - S3 bucket name
|
||||
/// * `object_key` - S3 object key
|
||||
/// * `ciphertext` - Encrypted data
|
||||
/// * `metadata` - Encryption metadata
|
||||
/// * `customer_key` - Customer-provided 256-bit key
|
||||
///
|
||||
/// # Returns
|
||||
/// Decrypted data as a reader
|
||||
///
|
||||
pub async fn decrypt_object_with_customer_key(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -559,14 +481,6 @@ impl ObjectEncryptionService {
|
||||
}
|
||||
|
||||
/// Validate encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `actual` - Actual encryption context from metadata
|
||||
/// * `expected` - Expected encryption context to validate against
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating success or context mismatch
|
||||
///
|
||||
fn validate_encryption_context(&self, actual: &HashMap<String, String>, expected: &HashMap<String, String>) -> Result<()> {
|
||||
for (key, expected_value) in expected {
|
||||
match actual.get(key) {
|
||||
@@ -585,13 +499,6 @@ impl ObjectEncryptionService {
|
||||
}
|
||||
|
||||
/// Convert encryption metadata to HTTP headers for S3 compatibility
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `metadata` - EncryptionMetadata to convert
|
||||
///
|
||||
/// # Returns
|
||||
/// HashMap of HTTP headers
|
||||
///
|
||||
pub fn metadata_to_headers(&self, metadata: &EncryptionMetadata) -> HashMap<String, String> {
|
||||
let mut headers = HashMap::new();
|
||||
|
||||
@@ -635,13 +542,6 @@ impl ObjectEncryptionService {
|
||||
}
|
||||
|
||||
/// Parse encryption metadata from HTTP headers
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `headers` - HashMap of HTTP headers
|
||||
///
|
||||
/// # Returns
|
||||
/// EncryptionMetadata parsed from headers
|
||||
///
|
||||
pub fn headers_to_metadata(&self, headers: &HashMap<String, String>) -> Result<EncryptionMetadata> {
|
||||
let algorithm = headers
|
||||
.get("x-amz-server-side-encryption")
|
||||
|
||||
@@ -116,7 +116,7 @@ impl KmsError {
|
||||
Self::BackendError { message: message.into() }
|
||||
}
|
||||
|
||||
/// Create access denied error
|
||||
/// Create an access denied error
|
||||
pub fn access_denied<S: Into<String>>(message: S) -> Self {
|
||||
Self::AccessDenied { message: message.into() }
|
||||
}
|
||||
@@ -184,7 +184,7 @@ impl KmsError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert from standard library errors
|
||||
// Convert from standard library errors
|
||||
impl From<std::io::Error> for KmsError {
|
||||
fn from(error: std::io::Error) -> Self {
|
||||
Self::IoError {
|
||||
@@ -206,13 +206,6 @@ impl From<serde_json::Error> for KmsError {
|
||||
|
||||
impl KmsError {
|
||||
/// Create a KMS error from AES-GCM error
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `error` - The AES-GCM error to convert
|
||||
///
|
||||
/// #Returns
|
||||
/// * `KmsError` - The corresponding KMS error
|
||||
///
|
||||
pub fn from_aes_gcm_error(error: aes_gcm::Error) -> Self {
|
||||
Self::CryptographicError {
|
||||
operation: "AES-GCM".to_string(),
|
||||
@@ -221,13 +214,6 @@ impl KmsError {
|
||||
}
|
||||
|
||||
/// Create a KMS error from ChaCha20-Poly1305 error
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `error` - The ChaCha20-Poly1305 error to convert
|
||||
///
|
||||
/// #Returns
|
||||
/// * `KmsError` - The corresponding KMS error
|
||||
///
|
||||
pub fn from_chacha20_error(error: chacha20poly1305::Error) -> Self {
|
||||
Self::CryptographicError {
|
||||
operation: "ChaCha20-Poly1305".to_string(),
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::encryption::service::ObjectEncryptionService;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
@@ -71,7 +71,7 @@ impl KmsServiceManager {
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
info!("CLAUDE DEBUG: configure() called with backend: {:?}", new_config.backend);
|
||||
tracing::info!("CLAUDE DEBUG: configure() called with backend: {:?}", new_config.backend);
|
||||
info!("Configuring KMS with backend: {:?}", new_config.backend);
|
||||
|
||||
// Update configuration
|
||||
@@ -92,7 +92,7 @@ impl KmsServiceManager {
|
||||
|
||||
/// Start KMS service with current configuration
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
info!("CLAUDE DEBUG: start() called");
|
||||
tracing::info!("CLAUDE DEBUG: start() called");
|
||||
let config = {
|
||||
let config_guard = self.config.read().await;
|
||||
match config_guard.as_ref() {
|
||||
@@ -254,7 +254,7 @@ impl Default for KmsServiceManager {
|
||||
}
|
||||
|
||||
/// Global KMS service manager instance
|
||||
static GLOBAL_KMS_SERVICE_MANAGER: OnceLock<Arc<KmsServiceManager>> = OnceLock::new();
|
||||
static GLOBAL_KMS_SERVICE_MANAGER: once_cell::sync::OnceCell<Arc<KmsServiceManager>> = once_cell::sync::OnceCell::new();
|
||||
|
||||
/// Initialize global KMS service manager
|
||||
pub fn init_global_kms_service_manager() -> Arc<KmsServiceManager> {
|
||||
@@ -270,12 +270,12 @@ pub fn get_global_kms_service_manager() -> Option<Arc<KmsServiceManager>> {
|
||||
|
||||
/// Get global encryption service (if KMS is running)
|
||||
pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
|
||||
info!("CLAUDE DEBUG: get_global_encryption_service called");
|
||||
tracing::info!("CLAUDE DEBUG: get_global_encryption_service called");
|
||||
let manager = get_global_kms_service_manager().unwrap_or_else(|| {
|
||||
warn!("CLAUDE DEBUG: KMS service manager not initialized, initializing now as fallback");
|
||||
tracing::warn!("CLAUDE DEBUG: KMS service manager not initialized, initializing now as fallback");
|
||||
init_global_kms_service_manager()
|
||||
});
|
||||
let service = manager.get_encryption_service().await;
|
||||
info!("CLAUDE DEBUG: get_encryption_service returned: {}", service.is_some());
|
||||
tracing::info!("CLAUDE DEBUG: get_encryption_service returned: {}", service.is_some());
|
||||
service
|
||||
}
|
||||
|
||||
@@ -42,17 +42,6 @@ pub struct DataKey {
|
||||
|
||||
impl DataKey {
|
||||
/// Create a new data key
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - Unique identifier for the key
|
||||
/// * `version` - Key version number
|
||||
/// * `plaintext` - Optional plaintext key material
|
||||
/// * `ciphertext` - Encrypted key material
|
||||
/// * `key_spec` - Key specification (e.g., "AES_256")
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `DataKey` instance
|
||||
///
|
||||
pub fn new(key_id: String, version: u32, plaintext: Option<Vec<u8>>, ciphertext: Vec<u8>, key_spec: String) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
@@ -66,11 +55,6 @@ impl DataKey {
|
||||
}
|
||||
|
||||
/// Clear the plaintext key material from memory for security
|
||||
///
|
||||
/// # Security
|
||||
/// This method zeroes out the plaintext key material before dropping it
|
||||
/// to prevent sensitive data from lingering in memory.
|
||||
///
|
||||
pub fn clear_plaintext(&mut self) {
|
||||
if let Some(ref mut plaintext) = self.plaintext {
|
||||
// Zero out the memory before dropping
|
||||
@@ -80,14 +64,6 @@ impl DataKey {
|
||||
}
|
||||
|
||||
/// Add metadata to the data key
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Metadata key
|
||||
/// * `value` - Metadata value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `DataKey` instance with added metadata
|
||||
///
|
||||
pub fn with_metadata(mut self, key: String, value: String) -> Self {
|
||||
self.metadata.insert(key, value);
|
||||
self
|
||||
@@ -121,15 +97,6 @@ pub struct MasterKey {
|
||||
|
||||
impl MasterKey {
|
||||
/// Create a new master key
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - Unique identifier for the key
|
||||
/// * `algorithm` - Key algorithm (e.g., "AES-256")
|
||||
/// * `created_by` - Optional creator/owner of the key
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `MasterKey` instance
|
||||
///
|
||||
pub fn new(key_id: String, algorithm: String, created_by: Option<String>) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
@@ -146,16 +113,6 @@ impl MasterKey {
|
||||
}
|
||||
|
||||
/// Create a new master key with description
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - Unique identifier for the key
|
||||
/// * `algorithm` - Key algorithm (e.g., "AES-256")
|
||||
/// * `created_by` - Optional creator/owner of the key
|
||||
/// * `description` - Optional key description
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `MasterKey` instance with description
|
||||
///
|
||||
pub fn new_with_description(
|
||||
key_id: String,
|
||||
algorithm: String,
|
||||
@@ -261,14 +218,6 @@ pub struct GenerateKeyRequest {
|
||||
|
||||
impl GenerateKeyRequest {
|
||||
/// Create a new generate key request
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `master_key_id` - Master key ID to use for encryption
|
||||
/// * `key_spec` - Key specification (e.g., "AES_256")
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `GenerateKeyRequest` instance
|
||||
///
|
||||
pub fn new(master_key_id: String, key_spec: String) -> Self {
|
||||
Self {
|
||||
master_key_id,
|
||||
@@ -280,27 +229,12 @@ impl GenerateKeyRequest {
|
||||
}
|
||||
|
||||
/// Add encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Context key
|
||||
/// * `value` - Context value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `GenerateKeyRequest` instance with added context
|
||||
///
|
||||
pub fn with_context(mut self, key: String, value: String) -> Self {
|
||||
self.encryption_context.insert(key, value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set key length explicitly
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `length` - Key length in bytes
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `GenerateKeyRequest` instance with specified key length
|
||||
///
|
||||
pub fn with_length(mut self, length: u32) -> Self {
|
||||
self.key_length = Some(length);
|
||||
self
|
||||
@@ -322,14 +256,6 @@ pub struct EncryptRequest {
|
||||
|
||||
impl EncryptRequest {
|
||||
/// Create a new encrypt request
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - Key ID to use for encryption
|
||||
/// * `plaintext` - Plaintext data to encrypt
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `EncryptRequest` instance
|
||||
///
|
||||
pub fn new(key_id: String, plaintext: Vec<u8>) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
@@ -340,14 +266,6 @@ impl EncryptRequest {
|
||||
}
|
||||
|
||||
/// Add encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Context key
|
||||
/// * `value` - Context value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `EncryptRequest` instance with added context
|
||||
///
|
||||
pub fn with_context(mut self, key: String, value: String) -> Self {
|
||||
self.encryption_context.insert(key, value);
|
||||
self
|
||||
@@ -380,13 +298,6 @@ pub struct DecryptRequest {
|
||||
|
||||
impl DecryptRequest {
|
||||
/// Create a new decrypt request
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `ciphertext` - Ciphertext to decrypt
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `DecryptRequest` instance
|
||||
///
|
||||
pub fn new(ciphertext: Vec<u8>) -> Self {
|
||||
Self {
|
||||
ciphertext,
|
||||
@@ -396,14 +307,6 @@ impl DecryptRequest {
|
||||
}
|
||||
|
||||
/// Add encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Context key
|
||||
/// * `value` - Context value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `DecryptRequest` instance with added context
|
||||
///
|
||||
pub fn with_context(mut self, key: String, value: String) -> Self {
|
||||
self.encryption_context.insert(key, value);
|
||||
self
|
||||
@@ -462,13 +365,6 @@ pub struct OperationContext {
|
||||
|
||||
impl OperationContext {
|
||||
/// Create a new operation context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `principal` - User or service performing the operation
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `OperationContext` instance
|
||||
///
|
||||
pub fn new(principal: String) -> Self {
|
||||
Self {
|
||||
operation_id: Uuid::new_v4(),
|
||||
@@ -480,40 +376,18 @@ impl OperationContext {
|
||||
}
|
||||
|
||||
/// Add additional context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Context key
|
||||
/// * `value` - Context value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `OperationContext` instance with added context
|
||||
///
|
||||
pub fn with_context(mut self, key: String, value: String) -> Self {
|
||||
self.additional_context.insert(key, value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set source IP
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `ip` - Source IP address
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `OperationContext` instance with source IP
|
||||
///
|
||||
pub fn with_source_ip(mut self, ip: String) -> Self {
|
||||
self.source_ip = Some(ip);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set user agent
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `agent` - User agent string
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `OperationContext` instance with user agent
|
||||
///
|
||||
pub fn with_user_agent(mut self, agent: String) -> Self {
|
||||
self.user_agent = Some(agent);
|
||||
self
|
||||
@@ -537,14 +411,6 @@ pub struct ObjectEncryptionContext {
|
||||
|
||||
impl ObjectEncryptionContext {
|
||||
/// Create a new object encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - Bucket name
|
||||
/// * `object_key` - Object key
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `ObjectEncryptionContext` instance
|
||||
///
|
||||
pub fn new(bucket: String, object_key: String) -> Self {
|
||||
Self {
|
||||
bucket,
|
||||
@@ -556,40 +422,18 @@ impl ObjectEncryptionContext {
|
||||
}
|
||||
|
||||
/// Set content type
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `content_type` - Content type string
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `ObjectEncryptionContext` instance with content type
|
||||
///
|
||||
pub fn with_content_type(mut self, content_type: String) -> Self {
|
||||
self.content_type = Some(content_type);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set object size
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `size` - Object size in bytes
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `ObjectEncryptionContext` instance with size
|
||||
///
|
||||
pub fn with_size(mut self, size: u64) -> Self {
|
||||
self.size = Some(size);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add encryption context
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - Context key
|
||||
/// * `value` - Context value
|
||||
///
|
||||
/// # Returns
|
||||
/// Updated `ObjectEncryptionContext` instance with added context
|
||||
///
|
||||
pub fn with_encryption_context(mut self, key: String, value: String) -> Self {
|
||||
self.encryption_context.insert(key, value);
|
||||
self
|
||||
@@ -659,10 +503,6 @@ pub enum KeySpec {
|
||||
|
||||
impl KeySpec {
|
||||
/// Get the key size in bytes
|
||||
///
|
||||
/// # Returns
|
||||
/// Key size in bytes
|
||||
///
|
||||
pub fn key_size(&self) -> usize {
|
||||
match self {
|
||||
Self::Aes256 => 32,
|
||||
@@ -672,10 +512,6 @@ impl KeySpec {
|
||||
}
|
||||
|
||||
/// Get the string representation for backends
|
||||
///
|
||||
/// # Returns
|
||||
/// Key specification as a string
|
||||
///
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Aes256 => "AES_256",
|
||||
@@ -800,14 +636,6 @@ pub struct GenerateDataKeyRequest {
|
||||
|
||||
impl GenerateDataKeyRequest {
|
||||
/// Create a new generate data key request
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_id` - Key ID to use for encryption
|
||||
/// * `key_spec` - Key specification
|
||||
///
|
||||
/// # Returns
|
||||
/// A new `GenerateDataKeyRequest` instance
|
||||
///
|
||||
pub fn new(key_id: String, key_spec: KeySpec) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
@@ -830,10 +658,6 @@ pub struct GenerateDataKeyResponse {
|
||||
|
||||
impl EncryptionAlgorithm {
|
||||
/// Get the algorithm name as a string
|
||||
///
|
||||
/// # Returns
|
||||
/// Algorithm name as a string
|
||||
///
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Aes256 => "AES256",
|
||||
|
||||
@@ -70,7 +70,7 @@ pub async fn init_obs(endpoint: Option<String>) -> Result<OtelGuard, GlobalError
|
||||
Ok(otel_guard)
|
||||
}
|
||||
|
||||
/// Set the global guard for OtelGuard
|
||||
/// Set the global guard for OpenTelemetry
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `guard` - The OtelGuard instance to set globally
|
||||
@@ -93,11 +93,11 @@ pub async fn init_obs(endpoint: Option<String>) -> Result<OtelGuard, GlobalError
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
|
||||
info!("Initializing global guard");
|
||||
info!("Initializing global OpenTelemetry guard");
|
||||
GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
|
||||
}
|
||||
|
||||
/// Get the global guard for OtelGuard
|
||||
/// Get the global guard for OpenTelemetry
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(Arc<Mutex<OtelGuard>>)` if guard exists
|
||||
|
||||
@@ -272,7 +272,8 @@ fn init_file_logging(config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
if (current & !desired) != 0 {
|
||||
if let Err(e) = fs::set_permissions(log_directory, Permissions::from_mode(desired)) {
|
||||
return Err(TelemetryError::SetPermissions(format!(
|
||||
"dir='{log_directory}', want={desired:#o}, have={current:#o}, err={e}"
|
||||
"dir='{}', want={:#o}, have={:#o}, err={}",
|
||||
log_directory, desired, current, e
|
||||
)));
|
||||
}
|
||||
// Second verification
|
||||
@@ -280,14 +281,15 @@ fn init_file_logging(config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
let after = meta2.permissions().mode() & 0o777;
|
||||
if after != desired {
|
||||
return Err(TelemetryError::SetPermissions(format!(
|
||||
"dir='{log_directory}', want={desired:#o}, after={after:#o}"
|
||||
"dir='{}', want={:#o}, after={:#o}",
|
||||
log_directory, desired, after
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(TelemetryError::Io(format!("stat '{log_directory}' failed: {e}")));
|
||||
return Err(TelemetryError::Io(format!("stat '{}' failed: {}", log_directory, e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@ use std::{cmp, env, fs, io::Write, path::Path, process::Command};
|
||||
|
||||
type AnyError = Box<dyn std::error::Error>;
|
||||
|
||||
const VERSION_PROTOBUF: Version = Version(33, 1, 0); // 31.1.0
|
||||
const VERSION_FLATBUFFERS: Version = Version(25, 9, 23); // 25.9.23
|
||||
const VERSION_PROTOBUF: Version = Version(27, 2, 0); // 27.2.0
|
||||
const VERSION_FLATBUFFERS: Version = Version(24, 3, 25); // 24.3.25
|
||||
/// Build protos if the major version of `flatc` or `protoc` is greater
|
||||
/// or lesser than the expected version.
|
||||
const ENV_BUILD_PROTOS: &str = "BUILD_PROTOS";
|
||||
|
||||
@@ -33,7 +33,7 @@ tokio = { workspace = true, features = ["full"] }
|
||||
rand = { workspace = true }
|
||||
http.workspace = true
|
||||
aes-gcm = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
crc32fast = { workspace = true }
|
||||
pin-project-lite.workspace = true
|
||||
serde = { workspace = true }
|
||||
bytes.workspace = true
|
||||
@@ -49,8 +49,10 @@ thiserror.workspace = true
|
||||
base64.workspace = true
|
||||
sha1.workspace = true
|
||||
sha2.workspace = true
|
||||
crc64fast-nvme.workspace = true
|
||||
s3s.workspace = true
|
||||
hex-simd.workspace = true
|
||||
crc32c.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = { workspace = true }
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::errors::ChecksumMismatch;
|
||||
use base64::{Engine as _, engine::general_purpose};
|
||||
use bytes::Bytes;
|
||||
use crc32fast::Hasher as Crc32Hasher;
|
||||
use http::HeaderMap;
|
||||
use sha1::Sha1;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -611,7 +612,7 @@ pub trait ChecksumHasher: Write + Send + Sync {
|
||||
|
||||
/// CRC32 IEEE hasher
|
||||
pub struct Crc32IeeeHasher {
|
||||
hasher: crc_fast::Digest,
|
||||
hasher: Crc32Hasher,
|
||||
}
|
||||
|
||||
impl Default for Crc32IeeeHasher {
|
||||
@@ -623,7 +624,7 @@ impl Default for Crc32IeeeHasher {
|
||||
impl Crc32IeeeHasher {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc),
|
||||
hasher: Crc32Hasher::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -641,36 +642,27 @@ impl Write for Crc32IeeeHasher {
|
||||
|
||||
impl ChecksumHasher for Crc32IeeeHasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
(self.hasher.clone().finalize() as u32).to_be_bytes().to_vec()
|
||||
self.hasher.clone().finalize().to_be_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
self.hasher = Crc32Hasher::new();
|
||||
}
|
||||
}
|
||||
|
||||
/// CRC32 Castagnoli hasher
|
||||
pub struct Crc32CastagnoliHasher {
|
||||
hasher: crc_fast::Digest,
|
||||
}
|
||||
|
||||
impl Default for Crc32CastagnoliHasher {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
#[derive(Default)]
|
||||
pub struct Crc32CastagnoliHasher(u32);
|
||||
|
||||
impl Crc32CastagnoliHasher {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi),
|
||||
}
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Crc32CastagnoliHasher {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.hasher.update(buf);
|
||||
self.0 = crc32c::crc32c_append(self.0, buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
@@ -681,11 +673,11 @@ impl Write for Crc32CastagnoliHasher {
|
||||
|
||||
impl ChecksumHasher for Crc32CastagnoliHasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
(self.hasher.clone().finalize() as u32).to_be_bytes().to_vec()
|
||||
self.0.to_be_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi);
|
||||
self.0 = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -766,27 +758,22 @@ impl ChecksumHasher for Sha256Hasher {
|
||||
}
|
||||
|
||||
/// CRC64 NVME hasher
|
||||
#[derive(Default)]
|
||||
pub struct Crc64NvmeHasher {
|
||||
hasher: crc_fast::Digest,
|
||||
}
|
||||
|
||||
impl Default for Crc64NvmeHasher {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
hasher: crc64fast_nvme::Digest,
|
||||
}
|
||||
|
||||
impl Crc64NvmeHasher {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme),
|
||||
hasher: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Crc64NvmeHasher {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.hasher.update(buf);
|
||||
self.hasher.write(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
@@ -797,11 +784,11 @@ impl Write for Crc64NvmeHasher {
|
||||
|
||||
impl ChecksumHasher for Crc64NvmeHasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
self.hasher.clone().finalize().to_be_bytes().to_vec()
|
||||
self.hasher.sum64().to_be_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme);
|
||||
self.hasher = Default::default();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -356,11 +356,7 @@ where
|
||||
*this.compressed_len = 0;
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Decompressed length mismatch")));
|
||||
}
|
||||
let actual_crc = {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&decompressed);
|
||||
hasher.finalize() as u32
|
||||
};
|
||||
let actual_crc = crc32fast::hash(&decompressed);
|
||||
if actual_crc != crc {
|
||||
// error!("DecompressReader CRC32 mismatch: actual {actual_crc} != expected {crc}");
|
||||
this.compressed_buf.take();
|
||||
@@ -408,11 +404,7 @@ where
|
||||
|
||||
/// Build compressed block with header + uvarint + compressed data
|
||||
fn build_compressed_block(uncompressed_data: &[u8], compression_algorithm: CompressionAlgorithm) -> Vec<u8> {
|
||||
let crc = {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(uncompressed_data);
|
||||
hasher.finalize() as u32
|
||||
};
|
||||
let crc = crc32fast::hash(uncompressed_data);
|
||||
let compressed_data = compress_block(uncompressed_data, compression_algorithm);
|
||||
let uncompressed_len = uncompressed_data.len();
|
||||
let mut uncompressed_len_buf = [0u8; 10];
|
||||
|
||||
@@ -102,11 +102,7 @@ where
|
||||
let nonce = Nonce::try_from(this.nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?;
|
||||
let plaintext = &temp_buf.filled()[..n];
|
||||
let plaintext_len = plaintext.len();
|
||||
let crc = {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(plaintext);
|
||||
hasher.finalize() as u32
|
||||
};
|
||||
let crc = crc32fast::hash(plaintext);
|
||||
let ciphertext = cipher
|
||||
.encrypt(&nonce, plaintext)
|
||||
.map_err(|e| Error::other(format!("encrypt error: {e}")))?;
|
||||
@@ -413,11 +409,7 @@ where
|
||||
return Poll::Ready(Err(Error::other("Plaintext length mismatch")));
|
||||
}
|
||||
|
||||
let actual_crc = {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&plaintext);
|
||||
hasher.finalize() as u32
|
||||
};
|
||||
let actual_crc = crc32fast::hash(&plaintext);
|
||||
if actual_crc != crc {
|
||||
this.ciphertext_buf.take();
|
||||
*this.ciphertext_read = 0;
|
||||
|
||||
@@ -543,7 +543,7 @@ mod tests {
|
||||
*req.uri_mut() = Uri::from_parts(parts).unwrap();
|
||||
|
||||
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
|
||||
println!("canonical_request: \n{canonical_request}\n");
|
||||
println!("canonical_request: \n{}\n", canonical_request);
|
||||
assert_eq!(
|
||||
canonical_request,
|
||||
concat!(
|
||||
@@ -562,7 +562,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
|
||||
println!("string_to_sign: \n{string_to_sign}\n");
|
||||
println!("string_to_sign: \n{}\n", string_to_sign);
|
||||
assert_eq!(
|
||||
string_to_sign,
|
||||
concat!(
|
||||
@@ -576,7 +576,7 @@ mod tests {
|
||||
|
||||
let signing_key = get_signing_key(secret_access_key, region, t, service);
|
||||
let signature = get_signature(signing_key, &string_to_sign);
|
||||
println!("signature: \n{signature}\n");
|
||||
println!("signature: \n{}\n", signature);
|
||||
assert_eq!(signature, "73fad2dfea0727e10a7179bf49150360a56f2e6b519c53999fd6e011152187d0");
|
||||
}
|
||||
|
||||
@@ -608,7 +608,7 @@ mod tests {
|
||||
|
||||
println!("{:?}", req.uri().query());
|
||||
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
|
||||
println!("canonical_request: \n{canonical_request}\n");
|
||||
println!("canonical_request: \n{}\n", canonical_request);
|
||||
assert_eq!(
|
||||
canonical_request,
|
||||
concat!(
|
||||
@@ -627,7 +627,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
|
||||
println!("string_to_sign: \n{string_to_sign}\n");
|
||||
println!("string_to_sign: \n{}\n", string_to_sign);
|
||||
assert_eq!(
|
||||
string_to_sign,
|
||||
concat!(
|
||||
@@ -641,7 +641,7 @@ mod tests {
|
||||
|
||||
let signing_key = get_signing_key(secret_access_key, region, t, service);
|
||||
let signature = get_signature(signing_key, &string_to_sign);
|
||||
println!("signature: \n{signature}\n");
|
||||
println!("signature: \n{}\n", signature);
|
||||
assert_eq!(signature, "dfbed913d1982428f6224ee506431fc133dbcad184194c0cbf01bc517435788a");
|
||||
}
|
||||
|
||||
@@ -673,7 +673,7 @@ mod tests {
|
||||
|
||||
println!("{:?}", req.uri().query());
|
||||
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
|
||||
println!("canonical_request: \n{canonical_request}\n");
|
||||
println!("canonical_request: \n{}\n", canonical_request);
|
||||
assert_eq!(
|
||||
canonical_request,
|
||||
concat!(
|
||||
@@ -692,7 +692,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
|
||||
println!("string_to_sign: \n{string_to_sign}\n");
|
||||
println!("string_to_sign: \n{}\n", string_to_sign);
|
||||
assert_eq!(
|
||||
string_to_sign,
|
||||
concat!(
|
||||
@@ -706,7 +706,7 @@ mod tests {
|
||||
|
||||
let signing_key = get_signing_key(secret_access_key, region, t, service);
|
||||
let signature = get_signature(signing_key, &string_to_sign);
|
||||
println!("signature: \n{signature}\n");
|
||||
println!("signature: \n{}\n", signature);
|
||||
assert_eq!(signature, "c7c7c6e12e5709c0c2ffc4707600a86c3cd261dd1de7409126a17f5b08c58dfa");
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
/// Check if MQTT Broker is available
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `broker_url` - URL of MQTT Broker, for example `mqtt://localhost:1883`
|
||||
/// * `topic` - Topic for testing connections
|
||||
@@ -41,7 +40,6 @@
|
||||
/// url = "2.5.7"
|
||||
/// tokio = { version = "1", features = ["full"] }
|
||||
/// ```
|
||||
///
|
||||
pub async fn check_mqtt_broker_available(broker_url: &str, topic: &str) -> Result<(), String> {
|
||||
use rumqttc::{AsyncClient, MqttOptions, QoS};
|
||||
let url = rustfs_utils::parse_url(broker_url).map_err(|e| format!("Broker URL parsing failed:{e}"))?;
|
||||
|
||||
@@ -29,7 +29,7 @@ base64-simd = { workspace = true, optional = true }
|
||||
blake3 = { workspace = true, optional = true }
|
||||
brotli = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, optional = true }
|
||||
crc-fast = { workspace = true, optional = true }
|
||||
crc32fast = { workspace = true, optional = true }
|
||||
flate2 = { workspace = true, optional = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
hashbrown = { workspace = true, optional = true }
|
||||
@@ -88,9 +88,9 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde",
|
||||
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
|
||||
string = ["dep:regex", "dep:rand"]
|
||||
crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"]
|
||||
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd", "dep:crc-fast"]
|
||||
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd", "dep:crc32fast"]
|
||||
os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities
|
||||
integration = [] # integration test features
|
||||
sys = ["dep:sysinfo"] # system information features
|
||||
http = ["dep:convert_case", "dep:http", "dep:regex"]
|
||||
http = ["dep:convert_case", "dep:http"]
|
||||
full = ["ip", "tls", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "sys", "notify", "http"] # all features
|
||||
|
||||
@@ -26,13 +26,6 @@ use tracing::{debug, warn};
|
||||
|
||||
/// Load public certificate from file.
|
||||
/// This function loads a public certificate from the specified file.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `filename` - A string slice that holds the name of the file containing the public certificate.
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Result containing a vector of CertificateDer if successful, or an io::Error if an error occurs during loading.
|
||||
///
|
||||
pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
|
||||
// Open certificate file.
|
||||
let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
|
||||
@@ -50,13 +43,6 @@ pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
|
||||
|
||||
/// Load private key from file.
|
||||
/// This function loads a private key from the specified file.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `filename` - A string slice that holds the name of the file containing the private key.
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Result containing the PrivateKeyDer if successful, or an io::Error if an error occurs during loading.
|
||||
///
|
||||
pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
|
||||
// Open keyfile.
|
||||
let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
|
||||
@@ -67,14 +53,6 @@ pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
|
||||
}
|
||||
|
||||
/// error function
|
||||
/// This function creates a new io::Error with the provided error message.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `err` - A string containing the error message.
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Error instance with the specified error message.
|
||||
///
|
||||
pub fn certs_error(err: String) -> Error {
|
||||
Error::other(err)
|
||||
}
|
||||
@@ -83,13 +61,6 @@ pub fn certs_error(err: String) -> Error {
|
||||
/// This function loads all certificate and private key pairs from the specified directory.
|
||||
/// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory.
|
||||
/// The root directory can also contain a default certificate/private key pair.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `dir_path` - A string slice that holds the path to the directory containing the certificates and private keys.
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Result containing a HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer). If no valid certificate/private key pairs are found, an io::Error is returned.
|
||||
///
|
||||
pub fn load_all_certs_from_directory(
|
||||
dir_path: &str,
|
||||
) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
|
||||
@@ -166,14 +137,6 @@ pub fn load_all_certs_from_directory(
|
||||
/// loading a single certificate private key pair
|
||||
/// This function loads a certificate and private key from the specified paths.
|
||||
/// It returns a tuple containing the certificate and private key.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `cert_path` - A string slice that holds the path to the certificate file.
|
||||
/// * `key_path` - A string slice that holds the path to the private key file
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Result containing a tuple of (Vec<CertificateDer>, PrivateKeyDer) if successful, or an io::Error if an error occurs during loading.
|
||||
///
|
||||
fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
|
||||
let certs = load_certs(cert_path)?;
|
||||
let key = load_private_key(key_path)?;
|
||||
@@ -185,12 +148,6 @@ fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<Certif
|
||||
/// It uses the first certificate/private key pair found in the root directory as the default certificate.
|
||||
/// The rest of the certificates/private keys are used for SNI resolution.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `cert_key_pairs` - A HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer).
|
||||
///
|
||||
/// # Returns
|
||||
/// * An io::Result containing an implementation of ResolvesServerCert if successful, or an io::Error if an error occurs during loading.
|
||||
///
|
||||
pub fn create_multi_cert_resolver(
|
||||
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
|
||||
) -> io::Result<impl ResolvesServerCert> {
|
||||
@@ -238,10 +195,6 @@ pub fn create_multi_cert_resolver(
|
||||
}
|
||||
|
||||
/// Checks if TLS key logging is enabled.
|
||||
///
|
||||
/// # Returns
|
||||
/// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable.
|
||||
///
|
||||
pub fn tls_key_log() -> bool {
|
||||
env::var("RUSTFS_TLS_KEYLOG")
|
||||
.map(|v| {
|
||||
@@ -250,8 +203,6 @@ pub fn tls_key_log() -> bool {
|
||||
|| v.eq_ignore_ascii_case("on")
|
||||
|| v.eq_ignore_ascii_case("true")
|
||||
|| v.eq_ignore_ascii_case("yes")
|
||||
|| v.eq_ignore_ascii_case("enabled")
|
||||
|| v.eq_ignore_ascii_case("t")
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::io::Write;
|
||||
use std::{fmt, str};
|
||||
use tokio::io;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
@@ -42,13 +41,13 @@ impl CompressionAlgorithm {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CompressionAlgorithm {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
impl std::fmt::Display for CompressionAlgorithm {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_str())
|
||||
}
|
||||
}
|
||||
impl str::FromStr for CompressionAlgorithm {
|
||||
type Err = io::Error;
|
||||
impl std::str::FromStr for CompressionAlgorithm {
|
||||
type Err = std::io::Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
@@ -64,16 +63,6 @@ impl str::FromStr for CompressionAlgorithm {
|
||||
}
|
||||
}
|
||||
|
||||
/// Compress a block of data using the specified compression algorithm.
|
||||
/// Returns the compressed data as a Vec<u8>.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input` - The input data to be compressed.
|
||||
/// * `algorithm` - The compression algorithm to use.
|
||||
///
|
||||
/// # Returns
|
||||
/// * A Vec<u8> containing the compressed data.
|
||||
///
|
||||
pub fn compress_block(input: &[u8], algorithm: CompressionAlgorithm) -> Vec<u8> {
|
||||
match algorithm {
|
||||
CompressionAlgorithm::Gzip => {
|
||||
@@ -116,16 +105,6 @@ pub fn compress_block(input: &[u8], algorithm: CompressionAlgorithm) -> Vec<u8>
|
||||
}
|
||||
}
|
||||
|
||||
/// Decompress a block of data using the specified compression algorithm.
|
||||
/// Returns the decompressed data as a Vec<u8>.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `compressed` - The compressed data to be decompressed.
|
||||
/// * `algorithm` - The compression algorithm used for compression.
|
||||
///
|
||||
/// # Returns
|
||||
/// * A Result containing a Vec<u8> with the decompressed data, or an io::Error.
|
||||
///
|
||||
pub fn decompress_block(compressed: &[u8], algorithm: CompressionAlgorithm) -> io::Result<Vec<u8>> {
|
||||
match algorithm {
|
||||
CompressionAlgorithm::Gzip => {
|
||||
|
||||
@@ -12,95 +12,45 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use hex_simd::{AsOut, AsciiCase};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use hyper::body::Bytes;
|
||||
use sha1::Sha1;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::mem::MaybeUninit;
|
||||
|
||||
/// Base64 URL safe encoding without padding
|
||||
/// `base64_encode_url_safe_no_pad(input)`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input` - A byte slice to be encoded
|
||||
///
|
||||
/// # Returns
|
||||
/// A `String` containing the Base64 URL safe encoded representation of the input data
|
||||
use hex_simd::{AsOut, AsciiCase};
|
||||
use hyper::body::Bytes;
|
||||
|
||||
pub fn base64_encode_url_safe_no_pad(input: &[u8]) -> String {
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
|
||||
}
|
||||
|
||||
/// Base64 URL safe decoding without padding
|
||||
/// `base64_decode_url_safe_no_pad(input)`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input` - A byte slice containing the Base64 URL safe encoded data
|
||||
///
|
||||
/// # Returns
|
||||
/// A `Result` containing a `Vec<u8>` with the decoded data or a `base64_simd::Error` if decoding fails
|
||||
///
|
||||
/// # Errors
|
||||
/// This function will return an error if the input data is not valid Base64 URL safe encoded data
|
||||
///
|
||||
pub fn base64_decode_url_safe_no_pad(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
|
||||
base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
|
||||
}
|
||||
|
||||
/// encode to hex string (lowercase)
|
||||
/// `hex(data)`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `data` - A byte slice to be encoded
|
||||
///
|
||||
/// # Returns
|
||||
/// A `String` containing the hexadecimal representation of the input data in lowercase
|
||||
///
|
||||
pub fn hex(data: impl AsRef<[u8]>) -> String {
|
||||
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
/// verify sha256 checksum string
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `s` - A string slice to be verified
|
||||
///
|
||||
/// # Returns
|
||||
/// A `bool` indicating whether the input string is a valid SHA-256 checksum (64
|
||||
///
|
||||
pub fn is_sha256_checksum(s: &str) -> bool {
|
||||
// TODO: optimize
|
||||
let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f');
|
||||
s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex)
|
||||
}
|
||||
|
||||
/// HMAC-SHA1 hashing
|
||||
/// `hmac_sha1(key, data)`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - A byte slice representing the HMAC key
|
||||
/// * `data` - A byte slice representing the data to be hashed
|
||||
///
|
||||
/// # Returns
|
||||
/// A 20-byte array containing the HMAC-SHA1 hash of the input data using the provided key
|
||||
///
|
||||
pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use sha1::Sha1;
|
||||
|
||||
let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).unwrap();
|
||||
m.update(data.as_ref());
|
||||
m.finalize().into_bytes().into()
|
||||
}
|
||||
|
||||
/// HMAC-SHA256 hashing
|
||||
/// `hmac_sha256(key, data)`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - A byte slice representing the HMAC key
|
||||
/// * `data` - A byte slice representing the data to be hashed
|
||||
///
|
||||
/// # Returns
|
||||
/// A 32-byte array containing the HMAC-SHA256 hash of the input data using the provided key
|
||||
///
|
||||
pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use sha2::Sha256;
|
||||
|
||||
let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).unwrap();
|
||||
m.update(data.as_ref());
|
||||
m.finalize().into_bytes().into()
|
||||
@@ -114,25 +64,18 @@ fn hex_bytes32<R>(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R {
|
||||
}
|
||||
|
||||
fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> {
|
||||
use sha2::{Digest, Sha256};
|
||||
<Sha256 as Digest>::digest(data)
|
||||
}
|
||||
|
||||
fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut h = <Sha256 as Digest>::new();
|
||||
chunk.iter().for_each(|data| h.update(data));
|
||||
h.finalize()
|
||||
}
|
||||
|
||||
/// hex of sha256 `f(hex(sha256(data)))`
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `data` - A byte slice representing the data to be hashed
|
||||
/// * `f` - A closure that takes a string slice and returns a value of type `R`
|
||||
///
|
||||
/// # Returns
|
||||
/// A value of type `R` returned by the closure `f` after processing the hexadecimal
|
||||
/// representation of the SHA-256 hash of the input data
|
||||
///
|
||||
/// `f(hex(sha256(data)))`
|
||||
pub fn hex_sha256<R>(data: &[u8], f: impl FnOnce(&str) -> R) -> R {
|
||||
hex_bytes32(sha256(data).as_ref(), f)
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ use rustfs_config::{DEFAULT_LOG_DIR, DEFAULT_LOG_FILENAME};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::debug;
|
||||
|
||||
/// Get the absolute path to the current project
|
||||
///
|
||||
@@ -30,12 +29,11 @@ use tracing::debug;
|
||||
/// # Returns
|
||||
/// - `Ok(PathBuf)`: The absolute path of the project that was successfully obtained.
|
||||
/// - `Err(String)`: Error message for the failed path.
|
||||
///
|
||||
pub fn get_project_root() -> Result<PathBuf, String> {
|
||||
// Try to get the project root directory through the CARGO_MANIFEST_DIR environment variable
|
||||
if let Ok(manifest_dir) = env::var("CARGO_MANIFEST_DIR") {
|
||||
let project_root = Path::new(&manifest_dir).to_path_buf();
|
||||
debug!("Get the project root directory with CARGO_MANIFEST_DIR:{}", project_root.display());
|
||||
println!("Get the project root directory with CARGO_MANIFEST_DIR:{}", project_root.display());
|
||||
return Ok(project_root);
|
||||
}
|
||||
|
||||
@@ -45,7 +43,7 @@ pub fn get_project_root() -> Result<PathBuf, String> {
|
||||
// Assume that the project root directory is in the parent directory of the parent directory of the executable path (usually target/debug or target/release)
|
||||
project_root.pop(); // Remove the executable file name
|
||||
project_root.pop(); // Remove target/debug or target/release
|
||||
debug!("Deduce the project root directory through current_exe:{}", project_root.display());
|
||||
println!("Deduce the project root directory through current_exe:{}", project_root.display());
|
||||
return Ok(project_root);
|
||||
}
|
||||
|
||||
@@ -53,7 +51,7 @@ pub fn get_project_root() -> Result<PathBuf, String> {
|
||||
if let Ok(mut current_dir) = env::current_dir() {
|
||||
// Assume that the project root directory is in the parent directory of the current working directory
|
||||
current_dir.pop();
|
||||
debug!("Deduce the project root directory through current_dir:{}", current_dir.display());
|
||||
println!("Deduce the project root directory through current_dir:{}", current_dir.display());
|
||||
return Ok(current_dir);
|
||||
}
|
||||
|
||||
@@ -63,38 +61,12 @@ pub fn get_project_root() -> Result<PathBuf, String> {
|
||||
|
||||
/// Get the log directory as a string
|
||||
/// This function will try to find a writable log directory in the following order:
|
||||
///
|
||||
/// 1. Environment variables are specified
|
||||
/// 2. System temporary directory
|
||||
/// 3. User home directory
|
||||
/// 4. Current working directory
|
||||
/// 5. Relative path
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - The environment variable key to check for log directory
|
||||
///
|
||||
/// # Returns
|
||||
/// * `String` - The log directory path as a string
|
||||
///
|
||||
pub fn get_log_directory_to_string(key: &str) -> String {
|
||||
get_log_directory(key).to_string_lossy().to_string()
|
||||
}
|
||||
|
||||
/// Get the log directory
|
||||
/// This function will try to find a writable log directory in the following order:
|
||||
///
|
||||
/// 1. Environment variables are specified
|
||||
/// 2. System temporary directory
|
||||
/// 3. User home directory
|
||||
/// 4. Current working directory
|
||||
/// 5. Relative path
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - The environment variable key to check for log directory
|
||||
///
|
||||
/// # Returns
|
||||
/// * `PathBuf` - The log directory path
|
||||
///
|
||||
pub fn get_log_directory(key: &str) -> PathBuf {
|
||||
// Environment variables are specified
|
||||
if let Ok(log_dir) = env::var(key) {
|
||||
|
||||
476
crates/utils/src/dns_resolver.rs
Normal file
476
crates/utils/src/dns_resolver.rs
Normal file
@@ -0,0 +1,476 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// #![allow(dead_code)]
|
||||
//
|
||||
// //! Layered DNS resolution utility for Kubernetes environments
|
||||
// //!
|
||||
// //! This module provides robust DNS resolution with multiple fallback layers:
|
||||
// //! 1. Local cache (Moka) for previously resolved results
|
||||
// //! 2. System DNS resolver (container/host adaptive) using hickory-resolver
|
||||
// //! 3. Public DNS servers as final fallback (8.8.8.8, 1.1.1.1) using hickory-resolver with TLS
|
||||
// //!
|
||||
// //! The resolver is designed to handle 5-level or deeper domain names that may fail
|
||||
// //! in Kubernetes environments due to CoreDNS configuration, DNS recursion limits,
|
||||
// //! or network-related issues. Uses hickory-resolver for actual DNS queries with TLS support.
|
||||
//
|
||||
// use hickory_resolver::Resolver;
|
||||
// use hickory_resolver::config::ResolverConfig;
|
||||
// use hickory_resolver::name_server::TokioConnectionProvider;
|
||||
// use moka::future::Cache;
|
||||
// use std::net::IpAddr;
|
||||
// use std::sync::OnceLock;
|
||||
// use std::time::Duration;
|
||||
// use tracing::{debug, error, info, instrument, warn};
|
||||
//
|
||||
// /// Maximum FQDN length according to RFC standards
|
||||
// const MAX_FQDN_LENGTH: usize = 253;
|
||||
// /// Maximum DNS label length according to RFC standards
|
||||
// const MAX_LABEL_LENGTH: usize = 63;
|
||||
// /// Cache entry TTL in seconds
|
||||
// const CACHE_TTL_SECONDS: u64 = 300; // 5 minutes
|
||||
// /// Maximum cache size (number of entries)
|
||||
// const MAX_CACHE_SIZE: u64 = 10000;
|
||||
//
|
||||
// /// DNS resolution error types with detailed context and tracing information
|
||||
// #[derive(Debug, thiserror::Error)]
|
||||
// pub enum DnsError {
|
||||
// #[error("Invalid domain format: {reason}")]
|
||||
// InvalidFormat { reason: String },
|
||||
//
|
||||
// #[error("Local cache miss for domain: {domain}")]
|
||||
// CacheMiss { domain: String },
|
||||
//
|
||||
// #[error("System DNS resolution failed for domain: {domain} - {source}")]
|
||||
// SystemDnsFailed {
|
||||
// domain: String,
|
||||
// #[source]
|
||||
// source: Box<dyn std::error::Error + Send + Sync>,
|
||||
// },
|
||||
//
|
||||
// #[error("Public DNS resolution failed for domain: {domain} - {source}")]
|
||||
// PublicDnsFailed {
|
||||
// domain: String,
|
||||
// #[source]
|
||||
// source: Box<dyn std::error::Error + Send + Sync>,
|
||||
// },
|
||||
//
|
||||
// #[error(
|
||||
// "All DNS resolution attempts failed for domain: {domain}. Please check your domain spelling, network connectivity, or DNS configuration"
|
||||
// )]
|
||||
// AllAttemptsFailed { domain: String },
|
||||
//
|
||||
// #[error("DNS resolver initialization failed: {source}")]
|
||||
// InitializationFailed {
|
||||
// #[source]
|
||||
// source: Box<dyn std::error::Error + Send + Sync>,
|
||||
// },
|
||||
//
|
||||
// #[error("DNS configuration error: {source}")]
|
||||
// ConfigurationError {
|
||||
// #[source]
|
||||
// source: Box<dyn std::error::Error + Send + Sync>,
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// /// Layered DNS resolver with caching and multiple fallback strategies
|
||||
// pub struct LayeredDnsResolver {
|
||||
// /// Local cache for resolved domains using Moka for high performance
|
||||
// cache: Cache<String, Vec<IpAddr>>,
|
||||
// /// System DNS resolver using hickory-resolver with default configuration
|
||||
// system_resolver: Resolver<TokioConnectionProvider>,
|
||||
// /// Public DNS resolver using hickory-resolver with Cloudflare DNS servers
|
||||
// public_resolver: Resolver<TokioConnectionProvider>,
|
||||
// }
|
||||
//
|
||||
// impl LayeredDnsResolver {
|
||||
// /// Create a new layered DNS resolver with automatic DNS configuration detection
|
||||
// #[instrument(skip_all)]
|
||||
// pub async fn new() -> Result<Self, DnsError> {
|
||||
// info!("Initializing layered DNS resolver with hickory-resolver, Moka cache and public DNS fallback");
|
||||
//
|
||||
// // Create Moka cache with TTL and size limits
|
||||
// let cache = Cache::builder()
|
||||
// .time_to_live(Duration::from_secs(CACHE_TTL_SECONDS))
|
||||
// .max_capacity(MAX_CACHE_SIZE)
|
||||
// .build();
|
||||
//
|
||||
// // Create system DNS resolver with default configuration (auto-detects container/host DNS)
|
||||
// let system_resolver =
|
||||
// Resolver::builder_with_config(ResolverConfig::default(), TokioConnectionProvider::default()).build();
|
||||
//
|
||||
// let mut config = ResolverConfig::cloudflare_tls();
|
||||
// for ns in ResolverConfig::google_tls().name_servers() {
|
||||
// config.add_name_server(ns.clone())
|
||||
// }
|
||||
// // Create public DNS resolver using Cloudflare DNS with TLS support
|
||||
// let public_resolver = Resolver::builder_with_config(config, TokioConnectionProvider::default()).build();
|
||||
//
|
||||
// info!("DNS resolver initialized successfully with hickory-resolver system and Cloudflare TLS public fallback");
|
||||
//
|
||||
// Ok(Self {
|
||||
// cache,
|
||||
// system_resolver,
|
||||
// public_resolver,
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// /// Validate domain format according to RFC standards
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// fn validate_domain_format(domain: &str) -> Result<(), DnsError> {
|
||||
// info!("Validating domain format start");
|
||||
// // Check FQDN length
|
||||
// if domain.len() > MAX_FQDN_LENGTH {
|
||||
// return Err(DnsError::InvalidFormat {
|
||||
// reason: format!("FQDN must not exceed {} bytes, got {} bytes", MAX_FQDN_LENGTH, domain.len()),
|
||||
// });
|
||||
// }
|
||||
//
|
||||
// // Check each label length
|
||||
// for label in domain.split('.') {
|
||||
// if label.len() > MAX_LABEL_LENGTH {
|
||||
// return Err(DnsError::InvalidFormat {
|
||||
// reason: format!(
|
||||
// "Each label must not exceed {} bytes, label '{}' has {} bytes",
|
||||
// MAX_LABEL_LENGTH,
|
||||
// label,
|
||||
// label.len()
|
||||
// ),
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // Check for empty labels (except trailing dot)
|
||||
// let labels: Vec<&str> = domain.trim_end_matches('.').split('.').collect();
|
||||
// for label in &labels {
|
||||
// if label.is_empty() {
|
||||
// return Err(DnsError::InvalidFormat {
|
||||
// reason: "Domain contains empty labels".to_string(),
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// info!("DNS resolver validated successfully");
|
||||
// Ok(())
|
||||
// }
|
||||
//
|
||||
// /// Check local cache for resolved domain
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// async fn check_cache(&self, domain: &str) -> Option<Vec<IpAddr>> {
|
||||
// match self.cache.get(domain).await {
|
||||
// Some(ips) => {
|
||||
// debug!("DNS cache hit for domain: {}, found {} IPs", domain, ips.len());
|
||||
// Some(ips)
|
||||
// }
|
||||
// None => {
|
||||
// debug!("DNS cache miss for domain: {}", domain);
|
||||
// None
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /// Update local cache with resolved IPs
|
||||
// #[instrument(skip_all, fields(domain = %domain, ip_count = ips.len()))]
|
||||
// async fn update_cache(&self, domain: &str, ips: Vec<IpAddr>) {
|
||||
// self.cache.insert(domain.to_string(), ips.clone()).await;
|
||||
// debug!("DNS cache updated for domain: {} with {} IPs", domain, ips.len());
|
||||
// }
|
||||
//
|
||||
// /// Get cache statistics for monitoring
|
||||
// #[instrument(skip_all)]
|
||||
// pub async fn cache_stats(&self) -> (u64, u64) {
|
||||
// let entry_count = self.cache.entry_count();
|
||||
// let weighted_size = self.cache.weighted_size();
|
||||
// debug!("DNS cache stats - entries: {}, weighted_size: {}", entry_count, weighted_size);
|
||||
// (entry_count, weighted_size)
|
||||
// }
|
||||
//
|
||||
// /// Manually invalidate cache entries (useful for testing or forced refresh)
|
||||
// #[instrument(skip_all)]
|
||||
// pub async fn invalidate_cache(&self) {
|
||||
// self.cache.invalidate_all();
|
||||
// info!("DNS cache invalidated");
|
||||
// }
|
||||
//
|
||||
// /// Resolve domain using system DNS (cluster/host DNS configuration) with hickory-resolver
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// async fn resolve_with_system_dns(&self, domain: &str) -> Result<Vec<IpAddr>, DnsError> {
|
||||
// debug!("Attempting system DNS resolution for domain: {} using hickory-resolver", domain);
|
||||
//
|
||||
// match self.system_resolver.lookup_ip(domain).await {
|
||||
// Ok(lookup) => {
|
||||
// let ips: Vec<IpAddr> = lookup.iter().collect();
|
||||
// if !ips.is_empty() {
|
||||
// info!("System DNS resolution successful for domain: {} -> {} IPs", domain, ips.len());
|
||||
// Ok(ips)
|
||||
// } else {
|
||||
// warn!("System DNS returned empty result for domain: {}", domain);
|
||||
// Err(DnsError::SystemDnsFailed {
|
||||
// domain: domain.to_string(),
|
||||
// source: "No IP addresses found".to_string().into(),
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// Err(e) => {
|
||||
// warn!("System DNS resolution failed for domain: {} - {}", domain, e);
|
||||
// Err(DnsError::SystemDnsFailed {
|
||||
// domain: domain.to_string(),
|
||||
// source: Box::new(e),
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /// Resolve domain using public DNS servers (Cloudflare TLS DNS) with hickory-resolver
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// async fn resolve_with_public_dns(&self, domain: &str) -> Result<Vec<IpAddr>, DnsError> {
|
||||
// debug!(
|
||||
// "Attempting public DNS resolution for domain: {} using hickory-resolver with TLS-enabled Cloudflare DNS",
|
||||
// domain
|
||||
// );
|
||||
//
|
||||
// match self.public_resolver.lookup_ip(domain).await {
|
||||
// Ok(lookup) => {
|
||||
// let ips: Vec<IpAddr> = lookup.iter().collect();
|
||||
// if !ips.is_empty() {
|
||||
// info!("Public DNS resolution successful for domain: {} -> {} IPs", domain, ips.len());
|
||||
// Ok(ips)
|
||||
// } else {
|
||||
// warn!("Public DNS returned empty result for domain: {}", domain);
|
||||
// Err(DnsError::PublicDnsFailed {
|
||||
// domain: domain.to_string(),
|
||||
// source: "No IP addresses found".to_string().into(),
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// Err(e) => {
|
||||
// error!("Public DNS resolution failed for domain: {} - {}", domain, e);
|
||||
// Err(DnsError::PublicDnsFailed {
|
||||
// domain: domain.to_string(),
|
||||
// source: Box::new(e),
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /// Resolve domain with layered fallback strategy using hickory-resolver
|
||||
// ///
|
||||
// /// Resolution order with detailed tracing:
|
||||
// /// 1. Local cache (Moka with TTL)
|
||||
// /// 2. System DNS (hickory-resolver with host/container adaptive configuration)
|
||||
// /// 3. Public DNS (hickory-resolver with TLS-enabled Cloudflare DNS fallback)
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// pub async fn resolve(&self, domain: &str) -> Result<Vec<IpAddr>, DnsError> {
|
||||
// info!("Starting DNS resolution process for domain: {} start", domain);
|
||||
// // Validate domain format first
|
||||
// Self::validate_domain_format(domain)?;
|
||||
//
|
||||
// info!("Starting DNS resolution for domain: {}", domain);
|
||||
//
|
||||
// // Step 1: Check local cache
|
||||
// if let Some(ips) = self.check_cache(domain).await {
|
||||
// info!("DNS resolution completed from cache for domain: {} -> {} IPs", domain, ips.len());
|
||||
// return Ok(ips);
|
||||
// }
|
||||
//
|
||||
// debug!("Local cache miss for domain: {}, attempting system DNS", domain);
|
||||
//
|
||||
// // Step 2: Try system DNS (cluster/host adaptive)
|
||||
// match self.resolve_with_system_dns(domain).await {
|
||||
// Ok(ips) => {
|
||||
// self.update_cache(domain, ips.clone()).await;
|
||||
// info!("DNS resolution completed via system DNS for domain: {} -> {} IPs", domain, ips.len());
|
||||
// return Ok(ips);
|
||||
// }
|
||||
// Err(system_err) => {
|
||||
// warn!("System DNS failed for domain: {} - {}", domain, system_err);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // Step 3: Fallback to public DNS
|
||||
// info!("Falling back to public DNS for domain: {}", domain);
|
||||
// match self.resolve_with_public_dns(domain).await {
|
||||
// Ok(ips) => {
|
||||
// self.update_cache(domain, ips.clone()).await;
|
||||
// info!("DNS resolution completed via public DNS for domain: {} -> {} IPs", domain, ips.len());
|
||||
// Ok(ips)
|
||||
// }
|
||||
// Err(public_err) => {
|
||||
// error!(
|
||||
// "All DNS resolution attempts failed for domain:` {}`. System DNS: failed, Public DNS: {}",
|
||||
// domain, public_err
|
||||
// );
|
||||
// Err(DnsError::AllAttemptsFailed {
|
||||
// domain: domain.to_string(),
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /// Global DNS resolver instance
|
||||
// static GLOBAL_DNS_RESOLVER: OnceLock<LayeredDnsResolver> = OnceLock::new();
|
||||
//
|
||||
// /// Initialize the global DNS resolver
|
||||
// #[instrument]
|
||||
// pub async fn init_global_dns_resolver() -> Result<(), DnsError> {
|
||||
// info!("Initializing global DNS resolver");
|
||||
// let resolver = LayeredDnsResolver::new().await?;
|
||||
//
|
||||
// match GLOBAL_DNS_RESOLVER.set(resolver) {
|
||||
// Ok(()) => {
|
||||
// info!("Global DNS resolver initialized successfully");
|
||||
// Ok(())
|
||||
// }
|
||||
// Err(_) => {
|
||||
// warn!("Global DNS resolver was already initialized");
|
||||
// Ok(())
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /// Get the global DNS resolver instance
|
||||
// pub fn get_global_dns_resolver() -> Option<&'static LayeredDnsResolver> {
|
||||
// GLOBAL_DNS_RESOLVER.get()
|
||||
// }
|
||||
//
|
||||
// /// Resolve domain using the global DNS resolver with comprehensive tracing
|
||||
// #[instrument(skip_all, fields(domain = %domain))]
|
||||
// pub async fn resolve_domain(domain: &str) -> Result<Vec<IpAddr>, DnsError> {
|
||||
// info!("resolving domain for: {}", domain);
|
||||
// match get_global_dns_resolver() {
|
||||
// Some(resolver) => resolver.resolve(domain).await,
|
||||
// None => Err(DnsError::InitializationFailed {
|
||||
// source: "Global DNS resolver not initialized. Call init_global_dns_resolver() first."
|
||||
// .to_string()
|
||||
// .into(),
|
||||
// }),
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// #[cfg(test)]
|
||||
// mod tests {
|
||||
// use super::*;
|
||||
//
|
||||
// #[test]
|
||||
// fn test_domain_validation() {
|
||||
// // Valid domains
|
||||
// assert!(LayeredDnsResolver::validate_domain_format("example.com").is_ok());
|
||||
// assert!(LayeredDnsResolver::validate_domain_format("sub.example.com").is_ok());
|
||||
// assert!(LayeredDnsResolver::validate_domain_format("very.deep.sub.domain.example.com").is_ok());
|
||||
//
|
||||
// // Invalid domains - too long FQDN
|
||||
// let long_domain = "a".repeat(254);
|
||||
// assert!(LayeredDnsResolver::validate_domain_format(&long_domain).is_err());
|
||||
//
|
||||
// // Invalid domains - label too long
|
||||
// let long_label = format!("{}.com", "a".repeat(64));
|
||||
// assert!(LayeredDnsResolver::validate_domain_format(&long_label).is_err());
|
||||
//
|
||||
// // Invalid domains - empty label
|
||||
// assert!(LayeredDnsResolver::validate_domain_format("example..com").is_err());
|
||||
// }
|
||||
//
|
||||
// #[tokio::test]
|
||||
// async fn test_cache_functionality() {
|
||||
// let resolver = LayeredDnsResolver::new().await.unwrap();
|
||||
//
|
||||
// // Test cache miss
|
||||
// assert!(resolver.check_cache("example.com").await.is_none());
|
||||
//
|
||||
// // Update cache
|
||||
// let test_ips = vec![IpAddr::from([192, 0, 2, 1])];
|
||||
// resolver.update_cache("example.com", test_ips.clone()).await;
|
||||
//
|
||||
// // Test cache hit
|
||||
// assert_eq!(resolver.check_cache("example.com").await, Some(test_ips));
|
||||
//
|
||||
// // Test cache stats (note: moka cache might not immediately reflect changes)
|
||||
// let (total, _weighted_size) = resolver.cache_stats().await;
|
||||
// // Cache should have at least the entry we just added (might be 0 due to async nature)
|
||||
// assert!(total <= 1, "Cache should have at most 1 entry, got {total}");
|
||||
// }
|
||||
//
|
||||
// #[tokio::test]
|
||||
// async fn test_dns_resolution() {
|
||||
// let resolver = LayeredDnsResolver::new().await.unwrap();
|
||||
//
|
||||
// // Test resolution of a known domain (localhost should always resolve)
|
||||
// match resolver.resolve("localhost").await {
|
||||
// Ok(ips) => {
|
||||
// assert!(!ips.is_empty());
|
||||
// println!("Resolved localhost to: {ips:?}");
|
||||
// }
|
||||
// Err(e) => {
|
||||
// // In some test environments, even localhost might fail
|
||||
// // This is acceptable as long as our error handling works
|
||||
// println!("DNS resolution failed (might be expected in test environments): {e}");
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// #[tokio::test]
|
||||
// async fn test_invalid_domain_resolution() {
|
||||
// let resolver = LayeredDnsResolver::new().await.unwrap();
|
||||
//
|
||||
// // Test resolution of invalid domain
|
||||
// let result = resolver
|
||||
// .resolve("nonexistent.invalid.domain.example.thisdefinitelydoesnotexist")
|
||||
// .await;
|
||||
// assert!(result.is_err());
|
||||
//
|
||||
// if let Err(e) = result {
|
||||
// println!("Expected error for invalid domain: {e}");
|
||||
// // Should be AllAttemptsFailed since both system and public DNS should fail
|
||||
// assert!(matches!(e, DnsError::AllAttemptsFailed { .. }));
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// #[tokio::test]
|
||||
// async fn test_cache_invalidation() {
|
||||
// let resolver = LayeredDnsResolver::new().await.unwrap();
|
||||
//
|
||||
// // Add entry to cache
|
||||
// let test_ips = vec![IpAddr::from([192, 0, 2, 1])];
|
||||
// resolver.update_cache("test.example.com", test_ips.clone()).await;
|
||||
//
|
||||
// // Verify cache hit
|
||||
// assert_eq!(resolver.check_cache("test.example.com").await, Some(test_ips));
|
||||
//
|
||||
// // Invalidate cache
|
||||
// resolver.invalidate_cache().await;
|
||||
//
|
||||
// // Verify cache miss after invalidation
|
||||
// assert!(resolver.check_cache("test.example.com").await.is_none());
|
||||
// }
|
||||
//
|
||||
// #[tokio::test]
|
||||
// async fn test_global_resolver_initialization() {
|
||||
// // Test initialization
|
||||
// assert!(init_global_dns_resolver().await.is_ok());
|
||||
//
|
||||
// // Test that resolver is available
|
||||
// assert!(get_global_dns_resolver().is_some());
|
||||
//
|
||||
// // Test domain resolution through global resolver
|
||||
// match resolve_domain("localhost").await {
|
||||
// Ok(ips) => {
|
||||
// assert!(!ips.is_empty());
|
||||
// println!("Global resolver resolved localhost to: {ips:?}");
|
||||
// }
|
||||
// Err(e) => {
|
||||
// println!("Global resolver DNS resolution failed (might be expected in test environments): {e}");
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -366,8 +366,8 @@ pub fn get_env_bool(key: &str, default: bool) -> bool {
|
||||
env::var(key)
|
||||
.ok()
|
||||
.and_then(|v| match v.to_lowercase().as_str() {
|
||||
"1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Some(true),
|
||||
"0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Some(false),
|
||||
"1" | "true" | "yes" => Some(true),
|
||||
"0" | "false" | "no" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(default)
|
||||
@@ -383,8 +383,8 @@ pub fn get_env_bool(key: &str, default: bool) -> bool {
|
||||
///
|
||||
pub fn get_env_opt_bool(key: &str) -> Option<bool> {
|
||||
env::var(key).ok().and_then(|v| match v.to_lowercase().as_str() {
|
||||
"1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Some(true),
|
||||
"0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Some(false),
|
||||
"1" | "true" | "yes" => Some(true),
|
||||
"0" | "false" | "no" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -72,13 +72,6 @@ fn u8x32_from_u64x4(input: [u64; 4]) -> [u8; 32] {
|
||||
|
||||
impl HashAlgorithm {
|
||||
/// Hash the input data and return the hash result as Vec<u8>.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `data` - A byte slice representing the data to be hashed
|
||||
///
|
||||
/// # Returns
|
||||
/// A byte slice containing the hash of the input data
|
||||
///
|
||||
pub fn hash_encode(&self, data: &[u8]) -> impl AsRef<[u8]> {
|
||||
match self {
|
||||
HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
|
||||
@@ -99,10 +92,6 @@ impl HashAlgorithm {
|
||||
}
|
||||
|
||||
/// Return the output size in bytes for the hash algorithm.
|
||||
///
|
||||
/// # Returns
|
||||
/// The size in bytes of the hash output
|
||||
///
|
||||
pub fn size(&self) -> usize {
|
||||
match self {
|
||||
HashAlgorithm::SHA256 => 32,
|
||||
@@ -115,22 +104,13 @@ impl HashAlgorithm {
|
||||
}
|
||||
}
|
||||
|
||||
use crc32fast::Hasher;
|
||||
use siphasher::sip::SipHasher;
|
||||
|
||||
pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
pub const DEFAULT_SIP_HASH_KEY: [u8; 16] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
|
||||
/// SipHash function to hash a string key into a bucket index.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - The input string to be hashed
|
||||
/// * `cardinality` - The number of buckets
|
||||
/// * `id` - A 16-byte array used as the SipHash key
|
||||
///
|
||||
/// # Returns
|
||||
/// A usize representing the bucket index
|
||||
///
|
||||
pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
|
||||
// Your key, must be 16 bytes
|
||||
|
||||
@@ -140,19 +120,12 @@ pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
|
||||
(result as usize) % cardinality
|
||||
}
|
||||
|
||||
/// CRC32 hash function to hash a string key into a bucket index.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key` - The input string to be hashed
|
||||
/// * `cardinality` - The number of buckets
|
||||
///
|
||||
/// # Returns
|
||||
/// A usize representing the bucket index
|
||||
///
|
||||
pub fn crc_hash(key: &str, cardinality: usize) -> usize {
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(key.as_bytes());
|
||||
let checksum = hasher.finalize() as u32;
|
||||
let mut hasher = Hasher::new(); // Create a new hasher
|
||||
|
||||
hasher.update(key.as_bytes()); // Update hash state, add data
|
||||
|
||||
let checksum = hasher.finalize();
|
||||
|
||||
checksum as usize % cardinality
|
||||
}
|
||||
|
||||
@@ -163,10 +163,9 @@ pub const AMZ_TAGGING_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||
pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
|
||||
|
||||
pub const RUSTFS_FORCE_DELETE: &str = "X-Rustfs-Force-Delete";
|
||||
pub const RUSTFS_INCLUDE_DELETED: &str = "X-Rustfs-Include-Deleted";
|
||||
|
||||
pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-Status";
|
||||
pub const RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size";
|
||||
pub const RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size";
|
||||
|
||||
pub const RUSTFS_BUCKET_SOURCE_VERSION_ID: &str = "X-Rustfs-Source-Version-Id";
|
||||
pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-Rustfs-Source-Mtime";
|
||||
|
||||
@@ -34,10 +34,6 @@ static FOR_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)(?:for=)([
|
||||
static PROTO_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)^(;|,| )+(?:proto=)(https|http)").unwrap());
|
||||
|
||||
/// Used to disable all processing of the X-Forwarded-For header in source IP discovery.
|
||||
///
|
||||
/// # Returns
|
||||
/// A `bool` indicating whether the X-Forwarded-For header is enabled
|
||||
///
|
||||
fn is_xff_header_enabled() -> bool {
|
||||
env::var("_RUSTFS_API_XFF_HEADER")
|
||||
.unwrap_or_else(|_| "on".to_string())
|
||||
@@ -47,13 +43,6 @@ fn is_xff_header_enabled() -> bool {
|
||||
|
||||
/// GetSourceScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239
|
||||
/// Forwarded headers (in that order).
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `headers` - HTTP headers from the request
|
||||
///
|
||||
/// # Returns
|
||||
/// An `Option<String>` containing the source scheme if found
|
||||
///
|
||||
pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
||||
// Retrieve the scheme from X-Forwarded-Proto.
|
||||
if let Some(proto) = headers.get(X_FORWARDED_PROTO) {
|
||||
@@ -95,13 +84,6 @@ pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
||||
|
||||
/// GetSourceIPFromHeaders retrieves the IP from the X-Forwarded-For, X-Real-IP
|
||||
/// and RFC7239 Forwarded headers (in that order)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `headers` - HTTP headers from the request
|
||||
///
|
||||
/// # Returns
|
||||
/// An `Option<String>` containing the source IP address if found
|
||||
///
|
||||
pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
||||
let mut addr = None;
|
||||
|
||||
@@ -150,14 +132,6 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
||||
/// GetSourceIPRaw retrieves the IP from the request headers
|
||||
/// and falls back to remote_addr when necessary.
|
||||
/// however returns without bracketing.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `headers` - HTTP headers from the request
|
||||
/// * `remote_addr` - Remote address as a string
|
||||
///
|
||||
/// # Returns
|
||||
/// A `String` containing the source IP address
|
||||
///
|
||||
pub fn get_source_ip_raw(headers: &HeaderMap, remote_addr: &str) -> String {
|
||||
let addr = get_source_ip_from_headers(headers).unwrap_or_else(|| remote_addr.to_string());
|
||||
|
||||
@@ -171,15 +145,6 @@ pub fn get_source_ip_raw(headers: &HeaderMap, remote_addr: &str) -> String {
|
||||
|
||||
/// GetSourceIP retrieves the IP from the request headers
|
||||
/// and falls back to remote_addr when necessary.
|
||||
/// It brackets IPv6 addresses.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `headers` - HTTP headers from the request
|
||||
/// * `remote_addr` - Remote address as a string
|
||||
///
|
||||
/// # Returns
|
||||
/// A `String` containing the source IP address, with IPv6 addresses bracketed
|
||||
///
|
||||
pub fn get_source_ip(headers: &HeaderMap, remote_addr: &str) -> String {
|
||||
let addr = get_source_ip_raw(headers, remote_addr);
|
||||
if addr.contains(':') { format!("[{addr}]") } else { addr }
|
||||
|
||||
@@ -83,7 +83,7 @@ pub fn put_uvarint_len(x: u64) -> usize {
|
||||
i + 1
|
||||
}
|
||||
|
||||
/// Decodes an u64 from buf and returns (value, number of bytes read).
|
||||
/// Decodes a u64 from buf and returns (value, number of bytes read).
|
||||
/// If buf is too small, returns (0, 0).
|
||||
/// If overflow, returns (0, -(n as isize)), where n is the number of bytes read.
|
||||
pub fn uvarint(buf: &[u8]) -> (u64, isize) {
|
||||
|
||||
@@ -20,9 +20,9 @@ use std::net::{IpAddr, Ipv4Addr};
|
||||
/// If both fail to retrieve, None is returned.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6)
|
||||
/// * `None` - Unable to obtain any native IP address
|
||||
///
|
||||
pub fn get_local_ip() -> Option<IpAddr> {
|
||||
local_ip_address::local_ip()
|
||||
.ok()
|
||||
@@ -34,8 +34,8 @@ pub fn get_local_ip() -> Option<IpAddr> {
|
||||
/// If the IP address cannot be obtained, returns "127.0.0.1" as the default value.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `String` - Native IP address (IPv4 or IPv6) as a string, or the default value
|
||||
///
|
||||
/// * `String` - Native IP address (IPv4 or IPv6) as a string, or the default value
|
||||
pub fn get_local_ip_with_default() -> String {
|
||||
get_local_ip()
|
||||
.unwrap_or_else(|| IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))) // Provide a safe default value
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
#[cfg(feature = "tls")]
|
||||
pub mod certs;
|
||||
#[cfg(feature = "net")]
|
||||
pub mod dns_resolver;
|
||||
#[cfg(feature = "ip")]
|
||||
pub mod ip;
|
||||
#[cfg(feature = "net")]
|
||||
|
||||
@@ -40,8 +40,7 @@ pub enum NetError {
|
||||
SchemeWithEmptyHost,
|
||||
}
|
||||
|
||||
/// Host represents a network host with IP/name and port.
|
||||
/// Similar to Go's net.Host structure.
|
||||
// Host represents a network host with IP/name and port.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Host {
|
||||
pub name: String,
|
||||
@@ -131,38 +130,22 @@ fn trim_ipv6(host: &str) -> Result<String, NetError> {
|
||||
}
|
||||
}
|
||||
|
||||
/// URL is a wrapper around url::Url for custom handling.
|
||||
/// Provides methods similar to Go's URL struct.
|
||||
// URL is a wrapper around url::Url for custom handling.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParsedURL(pub Url);
|
||||
|
||||
impl ParsedURL {
|
||||
/// is_empty returns true if the URL is empty or "about:blank".
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `&self` - Reference to the ParsedURL instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the URL is empty or "about:blank", false otherwise.
|
||||
///
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank")
|
||||
}
|
||||
|
||||
/// hostname returns the hostname of the URL.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `String` - The hostname of the URL, or an empty string if not set.
|
||||
///
|
||||
pub fn hostname(&self) -> String {
|
||||
self.0.host_str().unwrap_or("").to_string()
|
||||
}
|
||||
|
||||
/// port returns the port of the URL as a string, defaulting to "80" for http and "443" for https if not set.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `String` - The port of the URL as a string.
|
||||
///
|
||||
pub fn port(&self) -> String {
|
||||
match self.0.port() {
|
||||
Some(p) => p.to_string(),
|
||||
@@ -175,19 +158,11 @@ impl ParsedURL {
|
||||
}
|
||||
|
||||
/// scheme returns the scheme of the URL.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `&str` - The scheme of the URL.
|
||||
///
|
||||
pub fn scheme(&self) -> &str {
|
||||
self.0.scheme()
|
||||
}
|
||||
|
||||
/// url returns a reference to the underlying Url.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `&Url` - Reference to the underlying Url.
|
||||
///
|
||||
pub fn url(&self) -> &Url {
|
||||
&self.0
|
||||
}
|
||||
@@ -238,18 +213,7 @@ impl<'de> serde::Deserialize<'de> for ParsedURL {
|
||||
}
|
||||
}
|
||||
|
||||
/// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `s` - The URL string to parse.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(ParsedURL)` - If parsing is successful.
|
||||
/// * `Err(NetError)` - If parsing fails or host is invalid.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns NetError if parsing fails or host is invalid.
|
||||
///
|
||||
// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
|
||||
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
if let Some(scheme_end) = s.find("://") {
|
||||
if s[scheme_end + 3..].starts_with('/') {
|
||||
@@ -309,14 +273,6 @@ pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// parse_http_url parses a string into a ParsedURL, ensuring the scheme is http or https.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `s` - The URL string to parse.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(ParsedURL)` - If parsing is successful and scheme is http/https.
|
||||
/// * `Err(NetError)` - If parsing fails or scheme is not http/https.
|
||||
///
|
||||
pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
let u = parse_url(s)?;
|
||||
match u.0.scheme() {
|
||||
@@ -327,14 +283,6 @@ pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// is_network_or_host_down checks if an error indicates network or host down, considering timeouts.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `err` - The std::io::Error to check.
|
||||
/// * `expect_timeouts` - Whether timeouts are expected.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the error indicates network or host down, false otherwise.
|
||||
///
|
||||
pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return !expect_timeouts;
|
||||
@@ -349,26 +297,12 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// is_conn_reset_err checks if an error indicates a connection reset by peer.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `err` - The std::io::Error to check.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the error indicates connection reset, false otherwise.
|
||||
///
|
||||
pub fn is_conn_reset_err(err: &std::io::Error) -> bool {
|
||||
err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// is_conn_refused_err checks if an error indicates a connection refused.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `err` - The std::io::Error to check.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the error indicates connection refused, false otherwise.
|
||||
///
|
||||
pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
|
||||
err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED))
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ pub struct RetryTimer {
|
||||
|
||||
impl RetryTimer {
|
||||
pub fn new(max_retry: i64, base_sleep: Duration, max_sleep: Duration, jitter: f64, random: u64) -> Self {
|
||||
//println!("time1: {:?}", std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis());
|
||||
Self {
|
||||
base_sleep,
|
||||
max_sleep,
|
||||
@@ -69,6 +70,9 @@ impl Stream for RetryTimer {
|
||||
sleep = self.max_sleep;
|
||||
}
|
||||
if (jitter - NO_JITTER).abs() > 1e-9 {
|
||||
//println!("\njitter: {:?}", jitter);
|
||||
//println!("sleep: {sleep:?}");
|
||||
//println!("0000: {:?}", self.random as f64 * jitter / 100_f64);
|
||||
let sleep_ms = sleep.as_millis();
|
||||
let reduction = ((sleep_ms as f64) * (self.random as f64 * jitter / 100_f64)).round() as u128;
|
||||
let jittered_ms = sleep_ms.saturating_sub(reduction);
|
||||
@@ -81,21 +85,29 @@ impl Stream for RetryTimer {
|
||||
let mut timer = interval(sleep);
|
||||
timer.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
self.timer = Some(timer);
|
||||
//println!("time1: {:?}", std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis());
|
||||
}
|
||||
|
||||
let mut timer = self.timer.as_mut().unwrap();
|
||||
match Pin::new(&mut timer).poll_tick(cx) {
|
||||
Poll::Ready(_) => {
|
||||
//println!("ready");
|
||||
//println!("time2: {:?}", std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis());
|
||||
self.rem -= 1;
|
||||
if self.rem > 0 {
|
||||
let mut new_timer = interval(sleep);
|
||||
new_timer.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
new_timer.reset();
|
||||
self.timer = Some(new_timer);
|
||||
//println!("time1: {:?}", std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis());
|
||||
}
|
||||
Poll::Ready(Some(()))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Pending => {
|
||||
//println!("pending");
|
||||
//println!("time2: {:?}", std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis());
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -164,7 +176,7 @@ pub fn is_request_error_retryable(_err: std::io::Error) -> bool {
|
||||
#[allow(unused_imports)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::StreamExt;
|
||||
use futures::{Future, StreamExt};
|
||||
use rand::Rng;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
|
||||
@@ -17,29 +17,6 @@ use regex::Regex;
|
||||
use std::io::{Error, Result};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// Parses a boolean value from a string.
|
||||
///
|
||||
/// # Arguments
|
||||
/// `str` - A string slice representing the boolean value.
|
||||
///
|
||||
/// # Returns
|
||||
/// A `Result` containing the parsed boolean value or an error if parsing fails.
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::parse_bool;
|
||||
///
|
||||
/// let true_values = ["1", "t", "T", "true", "TRUE", "True", "on", "ON", "On", "enabled"];
|
||||
/// let false_values = ["0", "f", "F", "false", "FALSE", "False", "off", "OFF", "Off", "disabled"];
|
||||
///
|
||||
/// for val in true_values.iter() {
|
||||
/// assert_eq!(parse_bool(val).unwrap(), true);
|
||||
/// }
|
||||
/// for val in false_values.iter() {
|
||||
/// assert_eq!(parse_bool(val).unwrap(), false);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
pub fn parse_bool(str: &str) -> Result<bool> {
|
||||
match str {
|
||||
"1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true),
|
||||
@@ -48,23 +25,6 @@ pub fn parse_bool(str: &str) -> Result<bool> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Matches a simple pattern against a name using wildcards.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `pattern` - The pattern to match, which may include wildcards '*' and '?'
|
||||
/// * `name` - The name to match against the pattern
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if the name matches the pattern, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::match_simple;
|
||||
/// assert!(match_simple("file*", "file123"));
|
||||
/// assert!(match_simple("file?", "file1"));
|
||||
/// assert!(!match_simple("file?", "file12"));
|
||||
/// ```
|
||||
///
|
||||
pub fn match_simple(pattern: &str, name: &str) -> bool {
|
||||
if pattern.is_empty() {
|
||||
return name == pattern;
|
||||
@@ -76,24 +36,6 @@ pub fn match_simple(pattern: &str, name: &str) -> bool {
|
||||
deep_match_rune(name.as_bytes(), pattern.as_bytes(), true)
|
||||
}
|
||||
|
||||
/// Matches a pattern against a name using wildcards.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `pattern` - The pattern to match, which may include wildcards '*' and '?'
|
||||
/// * `name` - The name to match against the pattern
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if the name matches the pattern, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::match_pattern;
|
||||
///
|
||||
/// assert!(match_pattern("file*", "file123"));
|
||||
/// assert!(match_pattern("file?", "file1"));
|
||||
/// assert!(!match_pattern("file?", "file12"));
|
||||
/// ```
|
||||
///
|
||||
pub fn match_pattern(pattern: &str, name: &str) -> bool {
|
||||
if pattern.is_empty() {
|
||||
return name == pattern;
|
||||
@@ -105,25 +47,6 @@ pub fn match_pattern(pattern: &str, name: &str) -> bool {
|
||||
deep_match_rune(name.as_bytes(), pattern.as_bytes(), false)
|
||||
}
|
||||
|
||||
/// Checks if any pattern in the list matches the given string.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `patterns` - A slice of patterns to match against
|
||||
/// * `match_str` - The string to match against the patterns
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if any pattern matches the string, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::has_pattern;
|
||||
///
|
||||
/// let patterns = vec!["file*", "data?", "image*"];
|
||||
/// assert!(has_pattern(&patterns, "file123"));
|
||||
/// assert!(has_pattern(&patterns, "data1"));
|
||||
/// assert!(!has_pattern(&patterns, "video1"));
|
||||
/// ```
|
||||
///
|
||||
pub fn has_pattern(patterns: &[&str], match_str: &str) -> bool {
|
||||
for pattern in patterns {
|
||||
if match_simple(pattern, match_str) {
|
||||
@@ -133,23 +56,6 @@ pub fn has_pattern(patterns: &[&str], match_str: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Checks if the given string has any suffix from the provided list, ignoring case.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `str` - The string to check
|
||||
/// * `list` - A slice of suffixes to check against
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if the string ends with any of the suffixes in the list, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::has_string_suffix_in_slice;
|
||||
///
|
||||
/// let suffixes = vec![".txt", ".md", ".rs"];
|
||||
/// assert!(has_string_suffix_in_slice("document.TXT", &suffixes));
|
||||
/// assert!(!has_string_suffix_in_slice("image.png", &suffixes));
|
||||
/// ```
|
||||
pub fn has_string_suffix_in_slice(str: &str, list: &[&str]) -> bool {
|
||||
let str = str.to_lowercase();
|
||||
for v in list {
|
||||
@@ -193,24 +99,6 @@ fn deep_match_rune(str_: &[u8], pattern: &[u8], simple: bool) -> bool {
|
||||
str_.is_empty() && pattern.is_empty()
|
||||
}
|
||||
|
||||
/// Matches a pattern as a prefix against the given text.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `pattern` - The pattern to match, which may include wildcards '*' and '?'
|
||||
/// * `text` - The text to match against the pattern
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if the text matches the pattern as a prefix, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::match_as_pattern_prefix;
|
||||
///
|
||||
/// assert!(match_as_pattern_prefix("file*", "file123"));
|
||||
/// assert!(match_as_pattern_prefix("file?", "file1"));
|
||||
/// assert!(!match_as_pattern_prefix("file?", "file12"));
|
||||
/// ```
|
||||
///
|
||||
pub fn match_as_pattern_prefix(pattern: &str, text: &str) -> bool {
|
||||
let mut i = 0;
|
||||
while i < text.len() && i < pattern.len() {
|
||||
@@ -327,24 +215,6 @@ impl ArgPattern {
|
||||
}
|
||||
|
||||
/// finds all ellipses patterns, recursively and parses the ranges numerically.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `arg` - The argument string to search for ellipses patterns
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Result<ArgPattern>` - A result containing the parsed ArgPattern or an error if parsing fails
|
||||
///
|
||||
/// # Errors
|
||||
/// This function will return an error if the ellipses pattern format is invalid.
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::find_ellipses_patterns;
|
||||
///
|
||||
/// let pattern = "http://rustfs{2...3}/export/set{1...64}";
|
||||
/// let arg_pattern = find_ellipses_patterns(pattern).unwrap();
|
||||
/// assert_eq!(arg_pattern.total_sizes(), 128);
|
||||
/// ```
|
||||
pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
|
||||
let mut parts = match ELLIPSES_RE.captures(arg) {
|
||||
Some(caps) => caps,
|
||||
@@ -398,21 +268,6 @@ pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
|
||||
}
|
||||
|
||||
/// returns true if input arg has ellipses type pattern.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `s` - A slice of strings to check for ellipses patterns
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if any string contains ellipses patterns, `false` otherwise
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::has_ellipses;
|
||||
///
|
||||
/// let args = vec!["http://rustfs{2...3}/export/set{1...64}", "mydisk-{a...z}{1...20}"];
|
||||
/// assert!(has_ellipses(&args));
|
||||
/// ```
|
||||
///
|
||||
pub fn has_ellipses<T: AsRef<str>>(s: &[T]) -> bool {
|
||||
let pattern = [ELLIPSES, OPEN_BRACES, CLOSE_BRACES];
|
||||
|
||||
@@ -424,24 +279,6 @@ pub fn has_ellipses<T: AsRef<str>>(s: &[T]) -> bool {
|
||||
/// example:
|
||||
/// {1...64}
|
||||
/// {33...64}
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `pattern` - A string slice representing the ellipses range pattern
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Result<Vec<String>>` - A result containing a vector of strings representing the expanded range or an error if parsing fails
|
||||
///
|
||||
/// # Errors
|
||||
/// This function will return an error if the ellipses range format is invalid.
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::parse_ellipses_range;
|
||||
///
|
||||
/// let range = parse_ellipses_range("{1...5}").unwrap();
|
||||
/// assert_eq!(range, vec!["1", "2", "3", "4", "5"]);
|
||||
/// ```
|
||||
///
|
||||
pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
|
||||
if !pattern.contains(OPEN_BRACES) {
|
||||
return Err(Error::other("Invalid argument"));
|
||||
@@ -480,25 +317,6 @@ pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
/// Generates a random access key of the specified length.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `length` - The length of the access key to generate
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Result<String>` - A result containing the generated access key or an error if the length is too short
|
||||
///
|
||||
/// # Errors
|
||||
/// This function will return an error if the specified length is less than 3.
|
||||
///
|
||||
/// Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::gen_access_key;
|
||||
///
|
||||
/// let access_key = gen_access_key(16).unwrap();
|
||||
/// println!("Generated access key: {}", access_key);
|
||||
/// ```
|
||||
///
|
||||
pub fn gen_access_key(length: usize) -> Result<String> {
|
||||
const ALPHA_NUMERIC_TABLE: [char; 36] = [
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
|
||||
@@ -519,25 +337,6 @@ pub fn gen_access_key(length: usize) -> Result<String> {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Generates a random secret key of the specified length.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `length` - The length of the secret key to generate
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Result<String>` - A result containing the generated secret key or an error if the length is too short
|
||||
///
|
||||
/// # Errors
|
||||
/// This function will return an error if the specified length is less than 8.
|
||||
///
|
||||
/// # Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::gen_secret_key;
|
||||
///
|
||||
/// let secret_key = gen_secret_key(32).unwrap();
|
||||
/// println!("Generated secret key: {}", secret_key);
|
||||
/// ```
|
||||
///
|
||||
pub fn gen_secret_key(length: usize) -> Result<String> {
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
|
||||
@@ -556,22 +355,6 @@ pub fn gen_secret_key(length: usize) -> Result<String> {
|
||||
}
|
||||
|
||||
/// Tests whether the string s begins with prefix ignoring case
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `s` - The string to test
|
||||
/// * `prefix` - The prefix to look for
|
||||
///
|
||||
/// # Returns
|
||||
/// * `true` if s starts with prefix (case-insensitive), `false` otherwise
|
||||
///
|
||||
/// # Examples
|
||||
/// ```no_run
|
||||
/// use rustfs_utils::string::strings_has_prefix_fold;
|
||||
///
|
||||
/// assert!(strings_has_prefix_fold("HelloWorld", "hello"));
|
||||
/// assert!(!strings_has_prefix_fold("HelloWorld", "world"));
|
||||
/// ```
|
||||
///
|
||||
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
|
||||
if s.len() < prefix.len() {
|
||||
return false;
|
||||
|
||||
@@ -39,17 +39,7 @@ impl ServiceType {
|
||||
}
|
||||
}
|
||||
|
||||
/// UserAgent structure to hold User-Agent information
|
||||
/// including OS platform, architecture, version, and service type.
|
||||
/// It provides methods to generate a formatted User-Agent string.
|
||||
/// # Examples
|
||||
/// ```
|
||||
/// use rustfs_utils::{get_user_agent, ServiceType};
|
||||
///
|
||||
/// let ua = get_user_agent(ServiceType::Core);
|
||||
/// println!("User-Agent: {}", ua);
|
||||
/// ```
|
||||
#[derive(Debug)]
|
||||
// UserAgent structure
|
||||
struct UserAgent {
|
||||
os_platform: String,
|
||||
arch: String,
|
||||
@@ -156,14 +146,7 @@ impl fmt::Display for UserAgent {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the User-Agent string and accept business type parameters
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `service` - The type of service for which the User-Agent is being created.
|
||||
///
|
||||
/// # Returns
|
||||
/// A formatted User-Agent string.
|
||||
///
|
||||
// Get the User-Agent string and accept business type parameters
|
||||
pub fn get_user_agent(service: ServiceType) -> String {
|
||||
UserAgent::new(service).to_string()
|
||||
}
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
# Adaptive Buffer Sizing - Complete Implementation Summary
|
||||
|
||||
## English Version
|
||||
|
||||
### Overview
|
||||
This implementation provides a comprehensive adaptive buffer sizing optimization system for RustFS, enabling intelligent buffer size selection based on file size and workload characteristics. The complete migration path (Phases 1-4) has been successfully implemented with full backward compatibility.
|
||||
|
||||
### Key Features
|
||||
|
||||
#### 1. Workload Profile System
|
||||
- **6 Predefined Profiles**: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage
|
||||
- **Custom Configuration Support**: Flexible buffer size configuration with validation
|
||||
- **OS Environment Detection**: Automatic detection of secure Chinese OS environments (Kylin, NeoKylin, UOS, OpenKylin)
|
||||
- **Thread-Safe Global Configuration**: Atomic flags and immutable configuration structures
|
||||
|
||||
#### 2. Intelligent Buffer Sizing
|
||||
- **File Size Aware**: Automatically adjusts buffer sizes from 32KB to 4MB based on file size
|
||||
- **Profile-Based Optimization**: Different buffer strategies for different workload types
|
||||
- **Unknown Size Handling**: Special handling for streaming and chunked uploads
|
||||
- **Performance Metrics**: Optional metrics collection via feature flag
|
||||
|
||||
#### 3. Integration Points
|
||||
- **put_object**: Optimized buffer sizing for object uploads
|
||||
- **put_object_extract**: Special handling for archive extraction
|
||||
- **upload_part**: Multipart upload optimization
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
#### Phase 1: Infrastructure (Completed)
|
||||
- Created workload profile module (`rustfs/src/config/workload_profiles.rs`)
|
||||
- Implemented core data structures (WorkloadProfile, BufferConfig, RustFSBufferConfig)
|
||||
- Added configuration validation and testing framework
|
||||
|
||||
#### Phase 2: Opt-In Usage (Completed)
|
||||
- Added global configuration management
|
||||
- Implemented `RUSTFS_BUFFER_PROFILE_ENABLE` and `RUSTFS_BUFFER_PROFILE` configuration
|
||||
- Integrated buffer sizing into core upload functions
|
||||
- Maintained backward compatibility with legacy behavior
|
||||
|
||||
#### Phase 3: Default Enablement (Completed)
|
||||
- Changed default to enabled with GeneralPurpose profile
|
||||
- Replaced opt-in with opt-out mechanism (`--buffer-profile-disable`)
|
||||
- Created comprehensive migration guide (MIGRATION_PHASE3.md)
|
||||
- Ensured zero-impact migration for existing deployments
|
||||
|
||||
#### Phase 4: Full Integration (Completed)
|
||||
- Unified profile-only implementation
|
||||
- Removed hardcoded buffer values
|
||||
- Added optional performance metrics collection
|
||||
- Cleaned up deprecated code and improved documentation
|
||||
|
||||
### Technical Details
|
||||
|
||||
#### Buffer Size Ranges by Profile
|
||||
|
||||
| Profile | Min Buffer | Max Buffer | Optimal For |
|
||||
|---------|-----------|-----------|-------------|
|
||||
| GeneralPurpose | 64KB | 1MB | Mixed workloads |
|
||||
| AiTraining | 512KB | 4MB | Large files, sequential I/O |
|
||||
| DataAnalytics | 128KB | 2MB | Mixed read-write patterns |
|
||||
| WebWorkload | 32KB | 256KB | Small files, high concurrency |
|
||||
| IndustrialIoT | 64KB | 512KB | Real-time streaming |
|
||||
| SecureStorage | 32KB | 256KB | Compliance environments |
|
||||
|
||||
#### Configuration Options
|
||||
|
||||
**Environment Variables:**
|
||||
- `RUSTFS_BUFFER_PROFILE`: Select workload profile (default: GeneralPurpose)
|
||||
- `RUSTFS_BUFFER_PROFILE_DISABLE`: Disable profiling (opt-out)
|
||||
|
||||
**Command-Line Flags:**
|
||||
- `--buffer-profile <PROFILE>`: Set workload profile
|
||||
- `--buffer-profile-disable`: Disable workload profiling
|
||||
|
||||
### Performance Impact
|
||||
|
||||
- **Default (GeneralPurpose)**: Same performance as original implementation
|
||||
- **AiTraining**: Up to 4x throughput improvement for large files (>500MB)
|
||||
- **WebWorkload**: Lower memory usage, better concurrency for small files
|
||||
- **Metrics Collection**: < 1% CPU overhead when enabled
|
||||
|
||||
### Code Quality
|
||||
|
||||
- **30+ Unit Tests**: Comprehensive test coverage for all profiles and scenarios
|
||||
- **1200+ Lines of Documentation**: Complete usage guides, migration guides, and API documentation
|
||||
- **Thread-Safe Design**: Atomic flags, immutable configurations, zero data races
|
||||
- **Memory Safe**: All configurations validated, bounded buffer sizes
|
||||
|
||||
### Files Changed
|
||||
|
||||
```
|
||||
rustfs/src/config/mod.rs | 10 +
|
||||
rustfs/src/config/workload_profiles.rs | 650 +++++++++++++++++
|
||||
rustfs/src/storage/ecfs.rs | 200 ++++++
|
||||
rustfs/src/main.rs | 40 ++
|
||||
docs/adaptive-buffer-sizing.md | 550 ++++++++++++++
|
||||
docs/IMPLEMENTATION_SUMMARY.md | 380 ++++++++++
|
||||
docs/MIGRATION_PHASE3.md | 380 ++++++++++
|
||||
docs/PHASE4_GUIDE.md | 425 +++++++++++
|
||||
docs/README.md | 3 +
|
||||
```
|
||||
|
||||
### Backward Compatibility
|
||||
|
||||
- ✅ Zero breaking changes
|
||||
- ✅ Default behavior matches original implementation
|
||||
- ✅ Opt-out mechanism available
|
||||
- ✅ All existing tests pass
|
||||
- ✅ No configuration required for migration
|
||||
|
||||
### Usage Examples
|
||||
|
||||
**Default (Recommended):**
|
||||
```bash
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Custom Profile:**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Opt-Out:**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**With Metrics:**
|
||||
```bash
|
||||
cargo build --features metrics --release
|
||||
./target/release/rustfs /data
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 中文版本
|
||||
|
||||
### 概述
|
||||
本实现为 RustFS 提供了全面的自适应缓冲区大小优化系统，能够根据文件大小和工作负载特性智能选择缓冲区大小。完整的迁移路径（阶段 1-4）已成功实现，完全向后兼容。
|
||||
|
||||
### 核心功能
|
||||
|
||||
#### 1. 工作负载配置文件系统
|
||||
- **6 种预定义配置文件**：通用、AI训练、数据分析、Web工作负载、工业物联网、安全存储
|
||||
- **自定义配置支持**：灵活的缓冲区大小配置和验证
|
||||
- **操作系统环境检测**：自动检测中国安全操作系统环境（麒麟、中标麒麟、统信、开放麒麟）
|
||||
- **线程安全的全局配置**：原子标志和不可变配置结构
|
||||
|
||||
#### 2. 智能缓冲区大小调整
|
||||
- **文件大小感知**：根据文件大小自动调整 32KB 到 4MB 的缓冲区
|
||||
- **基于配置文件的优化**：不同工作负载类型的不同缓冲区策略
|
||||
- **未知大小处理**：流式传输和分块上传的特殊处理
|
||||
- **性能指标**：通过功能标志可选的指标收集
|
||||
|
||||
#### 3. 集成点
|
||||
- **put_object**：对象上传的优化缓冲区大小
|
||||
- **put_object_extract**：存档提取的特殊处理
|
||||
- **upload_part**：多部分上传优化
|
||||
|
||||
### 实现阶段
|
||||
|
||||
#### 阶段 1：基础设施（已完成）
|
||||
- 创建工作负载配置文件模块（`rustfs/src/config/workload_profiles.rs`）
|
||||
- 实现核心数据结构（WorkloadProfile、BufferConfig、RustFSBufferConfig）
|
||||
- 添加配置验证和测试框架
|
||||
|
||||
#### 阶段 2：选择性启用（已完成）
|
||||
- 添加全局配置管理
|
||||
- 实现 `RUSTFS_BUFFER_PROFILE_ENABLE` 和 `RUSTFS_BUFFER_PROFILE` 配置
|
||||
- 将缓冲区大小调整集成到核心上传函数中
|
||||
- 保持与旧版行为的向后兼容性
|
||||
|
||||
#### 阶段 3：默认启用（已完成）
|
||||
- 将默认值更改为使用通用配置文件启用
|
||||
- 将选择性启用替换为选择性退出机制（`--buffer-profile-disable`）
|
||||
- 创建全面的迁移指南（MIGRATION_PHASE3.md）
|
||||
- 确保现有部署的零影响迁移
|
||||
|
||||
#### 阶段 4：完全集成（已完成）
|
||||
- 统一的纯配置文件实现
|
||||
- 移除硬编码的缓冲区值
|
||||
- 添加可选的性能指标收集
|
||||
- 清理弃用代码并改进文档
|
||||
|
||||
### 技术细节
|
||||
|
||||
#### 按配置文件划分的缓冲区大小范围
|
||||
|
||||
| 配置文件 | 最小缓冲 | 最大缓冲 | 最适合 |
|
||||
|---------|---------|---------|--------|
|
||||
| 通用 | 64KB | 1MB | 混合工作负载 |
|
||||
| AI训练 | 512KB | 4MB | 大文件、顺序I/O |
|
||||
| 数据分析 | 128KB | 2MB | 混合读写模式 |
|
||||
| Web工作负载 | 32KB | 256KB | 小文件、高并发 |
|
||||
| 工业物联网 | 64KB | 512KB | 实时流式传输 |
|
||||
| 安全存储 | 32KB | 256KB | 合规环境 |
|
||||
|
||||
#### 配置选项
|
||||
|
||||
**环境变量：**
|
||||
- `RUSTFS_BUFFER_PROFILE`：选择工作负载配置文件（默认：通用）
|
||||
- `RUSTFS_BUFFER_PROFILE_DISABLE`：禁用配置文件（选择性退出）
|
||||
|
||||
**命令行标志：**
|
||||
- `--buffer-profile <配置文件>`：设置工作负载配置文件
|
||||
- `--buffer-profile-disable`：禁用工作负载配置文件
|
||||
|
||||
### 性能影响
|
||||
|
||||
- **默认（通用）**：与原始实现性能相同
|
||||
- **AI训练**：大文件（>500MB）吞吐量提升最多 4倍
|
||||
- **Web工作负载**：小文件的内存使用更低、并发性更好
|
||||
- **指标收集**：启用时 CPU 开销 < 1%
|
||||
|
||||
### 代码质量
|
||||
|
||||
- **30+ 单元测试**：全面覆盖所有配置文件和场景
|
||||
- **1200+ 行文档**：完整的使用指南、迁移指南和 API 文档
|
||||
- **线程安全设计**：原子标志、不可变配置、零数据竞争
|
||||
- **内存安全**：所有配置经过验证、缓冲区大小有界
|
||||
|
||||
### 文件变更
|
||||
|
||||
```
|
||||
rustfs/src/config/mod.rs | 10 +
|
||||
rustfs/src/config/workload_profiles.rs | 650 +++++++++++++++++
|
||||
rustfs/src/storage/ecfs.rs | 200 ++++++
|
||||
rustfs/src/main.rs | 40 ++
|
||||
docs/adaptive-buffer-sizing.md | 550 ++++++++++++++
|
||||
docs/IMPLEMENTATION_SUMMARY.md | 380 ++++++++++
|
||||
docs/MIGRATION_PHASE3.md | 380 ++++++++++
|
||||
docs/PHASE4_GUIDE.md | 425 +++++++++++
|
||||
docs/README.md | 3 +
|
||||
```
|
||||
|
||||
### 向后兼容性
|
||||
|
||||
- ✅ 零破坏性更改
|
||||
- ✅ 默认行为与原始实现匹配
|
||||
- ✅ 提供选择性退出机制
|
||||
- ✅ 所有现有测试通过
|
||||
- ✅ 迁移无需配置
|
||||
|
||||
### 使用示例
|
||||
|
||||
**默认（推荐）：**
|
||||
```bash
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**自定义配置文件：**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**选择性退出：**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**启用指标：**
|
||||
```bash
|
||||
cargo build --features metrics --release
|
||||
./target/release/rustfs /data
|
||||
```
|
||||
|
||||
### 总结
|
||||
|
||||
本实现为 RustFS 提供了企业级的自适应缓冲区优化能力，通过完整的四阶段迁移路径实现了从基础设施到完全集成的平滑过渡。系统默认启用，完全向后兼容，并提供了强大的工作负载优化功能，使不同场景下的性能得到显著提升。
|
||||
|
||||
完整的文档、全面的测试覆盖和生产就绪的实现确保了系统的可靠性和可维护性。通过可选的性能指标收集，运维团队可以持续监控和优化缓冲区配置，实现数据驱动的性能调优。
|
||||
@@ -1,412 +0,0 @@
|
||||
# Adaptive Buffer Sizing Implementation Summary
|
||||
|
||||
## Overview
|
||||
|
||||
This implementation extends PR #869 with a comprehensive adaptive buffer sizing optimization system that provides intelligent buffer size selection based on file size and workload type.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### 1. Workload Profile System
|
||||
|
||||
**File:** `rustfs/src/config/workload_profiles.rs` (501 lines)
|
||||
|
||||
A complete workload profiling system with:
|
||||
|
||||
- **6 Predefined Profiles:**
|
||||
- `GeneralPurpose`: Balanced performance (default)
|
||||
- `AiTraining`: Optimized for large sequential reads
|
||||
- `DataAnalytics`: Mixed read-write patterns
|
||||
- `WebWorkload`: Small file intensive
|
||||
- `IndustrialIoT`: Real-time streaming
|
||||
- `SecureStorage`: Security-first, memory-constrained
|
||||
|
||||
- **Custom Configuration Support:**
|
||||
```rust
|
||||
WorkloadProfile::Custom(BufferConfig {
|
||||
min_size: 16 * 1024,
|
||||
max_size: 512 * 1024,
|
||||
default_unknown: 128 * 1024,
|
||||
thresholds: vec![...],
|
||||
})
|
||||
```
|
||||
|
||||
- **Configuration Validation:**
|
||||
- Ensures min_size > 0
|
||||
- Validates max_size >= min_size
|
||||
- Checks threshold ordering
|
||||
- Validates buffer sizes within bounds
|
||||
|
||||
### 2. Enhanced Buffer Sizing Algorithm
|
||||
|
||||
**File:** `rustfs/src/storage/ecfs.rs` (+156 lines)
|
||||
|
||||
- **Backward Compatible:**
|
||||
- Preserved original `get_adaptive_buffer_size()` function
|
||||
- Existing code continues to work without changes
|
||||
|
||||
- **New Enhanced Function:**
|
||||
```rust
|
||||
fn get_adaptive_buffer_size_with_profile(
|
||||
file_size: i64,
|
||||
profile: Option<WorkloadProfile>
|
||||
) -> usize
|
||||
```
|
||||
|
||||
- **Auto-Detection:**
|
||||
- Automatically detects Chinese secure OS (Kylin, NeoKylin, UOS, OpenKylin)
|
||||
- Falls back to GeneralPurpose if no special environment detected
|
||||
|
||||
### 3. Comprehensive Testing
|
||||
|
||||
**Location:** `rustfs/src/storage/ecfs.rs` and `rustfs/src/config/workload_profiles.rs`
|
||||
|
||||
- Unit tests for all 6 workload profiles
|
||||
- Boundary condition testing
|
||||
- Configuration validation tests
|
||||
- Custom configuration tests
|
||||
- Unknown file size handling tests
|
||||
- Total: 15+ comprehensive test cases
|
||||
|
||||
### 4. Complete Documentation
|
||||
|
||||
**Files:**
|
||||
- `docs/adaptive-buffer-sizing.md` (460 lines)
|
||||
- `docs/README.md` (updated with navigation)
|
||||
|
||||
Documentation includes:
|
||||
- Overview and architecture
|
||||
- Detailed profile descriptions
|
||||
- Usage examples
|
||||
- Performance considerations
|
||||
- Best practices
|
||||
- Troubleshooting guide
|
||||
- Migration guide from PR #869
|
||||
|
||||
## Design Decisions
|
||||
|
||||
### 1. Backward Compatibility
|
||||
|
||||
**Decision:** Keep original `get_adaptive_buffer_size()` function unchanged.
|
||||
|
||||
**Rationale:**
|
||||
- Ensures no breaking changes
|
||||
- Existing code continues to work
|
||||
- Gradual migration path available
|
||||
|
||||
### 2. Profile-Based Configuration
|
||||
|
||||
**Decision:** Use enum-based profiles instead of global configuration.
|
||||
|
||||
**Rationale:**
|
||||
- Type-safe profile selection
|
||||
- Compile-time validation
|
||||
- Easy to extend with new profiles
|
||||
- Clear documentation of available options
|
||||
|
||||
### 3. Separate Module for Profiles
|
||||
|
||||
**Decision:** Create dedicated `workload_profiles` module.
|
||||
|
||||
**Rationale:**
|
||||
- Clear separation of concerns
|
||||
- Easy to locate and maintain
|
||||
- Can be used across the codebase
|
||||
- Facilitates testing
|
||||
|
||||
### 4. Conservative Default Values
|
||||
|
||||
**Decision:** Use moderate buffer sizes by default.
|
||||
|
||||
**Rationale:**
|
||||
- Prevents excessive memory usage
|
||||
- Suitable for most workloads
|
||||
- Users can opt-in to larger buffers
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
### Memory Usage by Profile
|
||||
|
||||
| Profile | Min Buffer | Max Buffer | Memory Footprint |
|
||||
|---------|-----------|-----------|------------------|
|
||||
| GeneralPurpose | 64KB | 1MB | Low-Medium |
|
||||
| AiTraining | 512KB | 4MB | High |
|
||||
| DataAnalytics | 128KB | 2MB | Medium |
|
||||
| WebWorkload | 32KB | 256KB | Low |
|
||||
| IndustrialIoT | 64KB | 512KB | Low |
|
||||
| SecureStorage | 32KB | 256KB | Low |
|
||||
|
||||
### Throughput Impact
|
||||
|
||||
- **Small buffers (32-64KB):** Better for high concurrency, many small files
|
||||
- **Medium buffers (128-512KB):** Balanced for mixed workloads
|
||||
- **Large buffers (1-4MB):** Maximum throughput for large sequential I/O
|
||||
|
||||
## Usage Patterns
|
||||
|
||||
### Simple Usage (Backward Compatible)
|
||||
|
||||
```rust
|
||||
// Existing code works unchanged
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
```
|
||||
|
||||
### Profile-Aware Usage
|
||||
|
||||
```rust
|
||||
// For AI/ML workloads
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
file_size,
|
||||
Some(WorkloadProfile::AiTraining)
|
||||
);
|
||||
|
||||
// Auto-detect environment
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(file_size, None);
|
||||
```
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```rust
|
||||
let custom = BufferConfig {
|
||||
min_size: 16 * 1024,
|
||||
max_size: 512 * 1024,
|
||||
default_unknown: 128 * 1024,
|
||||
thresholds: vec![
|
||||
(1024 * 1024, 64 * 1024),
|
||||
(i64::MAX, 256 * 1024),
|
||||
],
|
||||
};
|
||||
|
||||
let profile = WorkloadProfile::Custom(custom);
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(file_size, Some(profile));
|
||||
```
|
||||
|
||||
## Integration Points
|
||||
|
||||
The new functionality can be integrated into:
|
||||
|
||||
1. **`put_object`**: Choose profile based on object metadata or headers
|
||||
2. **`put_object_extract`**: Use appropriate profile for archive extraction
|
||||
3. **`upload_part`**: Apply profile for multipart uploads
|
||||
|
||||
Example integration (future enhancement):
|
||||
|
||||
```rust
|
||||
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
// Detect workload from headers or configuration
|
||||
let profile = detect_workload_from_request(&req);
|
||||
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
size,
|
||||
Some(profile)
|
||||
);
|
||||
|
||||
let body = tokio::io::BufReader::with_capacity(buffer_size, reader);
|
||||
// ... rest of implementation
|
||||
}
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Memory Safety
|
||||
|
||||
1. **Bounded Buffer Sizes:**
|
||||
- All configurations enforce min and max limits
|
||||
- Prevents out-of-memory conditions
|
||||
- Validation at configuration creation time
|
||||
|
||||
2. **Immutable Configurations:**
|
||||
- All config structures are immutable after creation
|
||||
- Thread-safe by design
|
||||
- No risk of race conditions
|
||||
|
||||
3. **Secure OS Detection:**
|
||||
- Read-only access to `/etc/os-release`
|
||||
- No privilege escalation required
|
||||
- Graceful fallback on error
|
||||
|
||||
### No New Vulnerabilities
|
||||
|
||||
- Only adds new functionality
|
||||
- Does not modify existing security-critical paths
|
||||
- Preserves all existing security measures
|
||||
- All new code is defensive and validated
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- Located in both modules with `#[cfg(test)]`
|
||||
- Test all workload profiles
|
||||
- Validate configuration logic
|
||||
- Test boundary conditions
|
||||
|
||||
### Integration Testing
|
||||
|
||||
Future integration tests should cover:
|
||||
- Actual file upload/download with different profiles
|
||||
- Performance benchmarks for each profile
|
||||
- Memory usage monitoring
|
||||
- Concurrent operations
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### 1. Runtime Configuration
|
||||
|
||||
Add environment variables or config file support:
|
||||
|
||||
```bash
|
||||
RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
RUSTFS_BUFFER_MIN_SIZE=32768
|
||||
RUSTFS_BUFFER_MAX_SIZE=1048576
|
||||
```
|
||||
|
||||
### 2. Dynamic Profiling
|
||||
|
||||
Collect metrics and automatically adjust profile:
|
||||
|
||||
```rust
|
||||
// Monitor actual I/O patterns and adjust buffer sizes
|
||||
let optimal_profile = analyze_io_patterns();
|
||||
```
|
||||
|
||||
### 3. Per-Bucket Configuration
|
||||
|
||||
Allow different profiles per bucket:
|
||||
|
||||
```rust
|
||||
// Configure profiles via bucket metadata
|
||||
bucket.set_buffer_profile(WorkloadProfile::WebWorkload);
|
||||
```
|
||||
|
||||
### 4. Performance Metrics
|
||||
|
||||
Add metrics to track buffer effectiveness:
|
||||
|
||||
```rust
|
||||
metrics::histogram!("buffer_utilization", utilization);
|
||||
metrics::counter!("buffer_resizes", 1);
|
||||
```
|
||||
|
||||
## Migration Path
|
||||
|
||||
### Phase 1: Current State ✅
|
||||
|
||||
- Infrastructure in place
|
||||
- Backward compatible
|
||||
- Fully documented
|
||||
- Tested
|
||||
|
||||
### Phase 2: Opt-In Usage ✅ **IMPLEMENTED**
|
||||
|
||||
- ✅ Configuration option to enable profiles (`RUSTFS_BUFFER_PROFILE_ENABLE`)
|
||||
- ✅ Workload profile selection (`RUSTFS_BUFFER_PROFILE`)
|
||||
- ✅ Default to existing behavior when disabled
|
||||
- ✅ Global configuration management
|
||||
- ✅ Integration in `put_object`, `put_object_extract`, and `upload_part`
|
||||
- ✅ Command-line and environment variable support
|
||||
- ✅ Performance monitoring ready
|
||||
|
||||
**How to Use:**
|
||||
```bash
|
||||
# Enable with environment variables
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Or use command-line flags
|
||||
./rustfs --buffer-profile-enable --buffer-profile WebWorkload /data
|
||||
```
|
||||
|
||||
### Phase 3: Default Enablement ✅ **IMPLEMENTED**
|
||||
|
||||
- ✅ Profile-aware buffer sizing enabled by default
|
||||
- ✅ Default profile: `GeneralPurpose` (same behavior as PR #869 for most files)
|
||||
- ✅ Backward compatibility via `--buffer-profile-disable` flag
|
||||
- ✅ Easy profile switching via `--buffer-profile` or `RUSTFS_BUFFER_PROFILE`
|
||||
- ✅ Updated documentation with Phase 3 examples
|
||||
|
||||
**Default Behavior:**
|
||||
```bash
|
||||
# Phase 3: Enabled by default with GeneralPurpose profile
|
||||
./rustfs /data
|
||||
|
||||
# Change to a different profile
|
||||
./rustfs --buffer-profile AiTraining /data
|
||||
|
||||
# Opt-out to legacy behavior if needed
|
||||
./rustfs --buffer-profile-disable /data
|
||||
```
|
||||
|
||||
**Key Changes from Phase 2:**
|
||||
- Phase 2: Required `--buffer-profile-enable` to opt-in
|
||||
- Phase 3: Enabled by default, use `--buffer-profile-disable` to opt-out
|
||||
- Maintains full backward compatibility
|
||||
- No breaking changes for existing deployments
|
||||
|
||||
### Phase 4: Full Integration ✅ **IMPLEMENTED**
|
||||
|
||||
- ✅ Deprecated legacy `get_adaptive_buffer_size()` function
|
||||
- ✅ Profile-only implementation via `get_buffer_size_opt_in()`
|
||||
- ✅ Performance metrics collection capability (with `metrics` feature)
|
||||
- ✅ Consolidated buffer sizing logic
|
||||
- ✅ All buffer sizes come from workload profiles
|
||||
|
||||
**Implementation Details:**
|
||||
```rust
|
||||
// Phase 4: Single entry point for buffer sizing
|
||||
fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
// Uses workload profiles exclusively
|
||||
// Legacy function deprecated but maintained for compatibility
|
||||
// Metrics collection integrated for performance monitoring
|
||||
}
|
||||
```
|
||||
|
||||
**Key Changes from Phase 3:**
|
||||
- Legacy function marked as `#[deprecated]` but still functional
|
||||
- Single, unified buffer sizing implementation
|
||||
- Performance metrics tracking (optional, via feature flag)
|
||||
- Even disabled mode uses GeneralPurpose profile (profile-only)
|
||||
|
||||
## Maintenance Guidelines
|
||||
|
||||
### Adding New Profiles
|
||||
|
||||
1. Add enum variant to `WorkloadProfile`
|
||||
2. Implement config method
|
||||
3. Add tests
|
||||
4. Update documentation
|
||||
5. Add usage examples
|
||||
|
||||
### Modifying Existing Profiles
|
||||
|
||||
1. Update threshold values in config method
|
||||
2. Update tests to match new values
|
||||
3. Update documentation
|
||||
4. Consider migration impact
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
1. Collect metrics from production
|
||||
2. Analyze buffer hit rates
|
||||
3. Adjust thresholds based on data
|
||||
4. A/B test changes
|
||||
5. Update documentation with findings
|
||||
|
||||
## Conclusion
|
||||
|
||||
This implementation provides a solid foundation for adaptive buffer sizing in RustFS:
|
||||
|
||||
- ✅ Comprehensive workload profiling system
|
||||
- ✅ Backward compatible design
|
||||
- ✅ Extensive testing
|
||||
- ✅ Complete documentation
|
||||
- ✅ Secure and memory-safe
|
||||
- ✅ Ready for production use
|
||||
|
||||
The modular design allows for gradual adoption and future enhancements without breaking existing functionality.
|
||||
|
||||
## References
|
||||
|
||||
- [PR #869: Fix large file upload freeze with adaptive buffer sizing](https://github.com/rustfs/rustfs/pull/869)
|
||||
- [Adaptive Buffer Sizing Documentation](./adaptive-buffer-sizing.md)
|
||||
- [Performance Testing Guide](./PERFORMANCE_TESTING.md)
|
||||
@@ -1,284 +0,0 @@
|
||||
# Migration Guide: Phase 2 to Phase 3
|
||||
|
||||
## Overview
|
||||
|
||||
Phase 3 of the adaptive buffer sizing feature makes workload profiles **enabled by default**. This document helps you understand the changes and how to migrate smoothly.
|
||||
|
||||
## What Changed
|
||||
|
||||
### Phase 2 (Opt-In)
|
||||
- Buffer profiling was **disabled by default**
|
||||
- Required explicit enabling via `--buffer-profile-enable` or `RUSTFS_BUFFER_PROFILE_ENABLE=true`
|
||||
- Used legacy PR #869 behavior unless explicitly enabled
|
||||
|
||||
### Phase 3 (Default Enablement)
|
||||
- Buffer profiling is **enabled by default** with `GeneralPurpose` profile
|
||||
- No configuration needed for default behavior
|
||||
- Can opt-out via `--buffer-profile-disable` or `RUSTFS_BUFFER_PROFILE_DISABLE=true`
|
||||
- Maintains full backward compatibility
|
||||
|
||||
## Impact Analysis
|
||||
|
||||
### For Most Users (No Action Required)
|
||||
|
||||
The `GeneralPurpose` profile (default in Phase 3) provides the **same buffer sizes** as PR #869 for most file sizes:
|
||||
- Small files (< 1MB): 64KB buffer
|
||||
- Medium files (1MB-100MB): 256KB buffer
|
||||
- Large files (≥ 100MB): 1MB buffer
|
||||
|
||||
**Result:** Your existing deployments will work exactly as before, with no performance changes.
|
||||
|
||||
### For Users Who Explicitly Enabled Profiles in Phase 2
|
||||
|
||||
If you were using:
|
||||
```bash
|
||||
# Phase 2
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
You can simplify to:
|
||||
```bash
|
||||
# Phase 3
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
The `RUSTFS_BUFFER_PROFILE_ENABLE` variable is no longer needed (but still respected for compatibility).
|
||||
|
||||
### For Users Who Want Exact Legacy Behavior
|
||||
|
||||
If you need the guaranteed exact behavior from PR #869 (before any profiling):
|
||||
|
||||
```bash
|
||||
# Phase 3 - Opt out to legacy behavior
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
|
||||
# Or via command-line
|
||||
./rustfs --buffer-profile-disable /data
|
||||
```
|
||||
|
||||
## Migration Scenarios
|
||||
|
||||
### Scenario 1: Default Deployment (No Changes Needed)
|
||||
|
||||
**Phase 2:**
|
||||
```bash
|
||||
./rustfs /data
|
||||
# Used PR #869 fixed algorithm
|
||||
```
|
||||
|
||||
**Phase 3:**
|
||||
```bash
|
||||
./rustfs /data
|
||||
# Uses GeneralPurpose profile (same buffer sizes as PR #869 for most cases)
|
||||
```
|
||||
|
||||
**Action:** None required. Behavior is essentially identical.
|
||||
|
||||
### Scenario 2: Using Custom Profile in Phase 2
|
||||
|
||||
**Phase 2:**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
export RUSTFS_BUFFER_PROFILE=WebWorkload
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Phase 3 (Simplified):**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE=WebWorkload
|
||||
./rustfs /data
|
||||
# RUSTFS_BUFFER_PROFILE_ENABLE no longer needed
|
||||
```
|
||||
|
||||
**Action:** Remove `RUSTFS_BUFFER_PROFILE_ENABLE=true` from your configuration.
|
||||
|
||||
### Scenario 3: Explicitly Disabled in Phase 2
|
||||
|
||||
**Phase 2:**
|
||||
```bash
|
||||
# Or just not setting RUSTFS_BUFFER_PROFILE_ENABLE
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Phase 3 (If you want to keep legacy behavior):**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Action:** Set `RUSTFS_BUFFER_PROFILE_DISABLE=true` if you want to guarantee exact PR #869 behavior.
|
||||
|
||||
### Scenario 4: AI/ML Workloads
|
||||
|
||||
**Phase 2:**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Phase 3 (Simplified):**
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Action:** Remove `RUSTFS_BUFFER_PROFILE_ENABLE=true`.
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Phase 3 Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `RUSTFS_BUFFER_PROFILE` | `GeneralPurpose` | The workload profile to use |
|
||||
| `RUSTFS_BUFFER_PROFILE_DISABLE` | `false` | Disable profiling and use legacy behavior |
|
||||
|
||||
### Phase 3 Command-Line Flags
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--buffer-profile <PROFILE>` | `GeneralPurpose` | Set the workload profile |
|
||||
| `--buffer-profile-disable` | disabled | Disable profiling (opt-out) |
|
||||
|
||||
### Deprecated (Still Supported for Compatibility)
|
||||
|
||||
| Variable | Status | Replacement |
|
||||
|----------|--------|-------------|
|
||||
| `RUSTFS_BUFFER_PROFILE_ENABLE` | Deprecated | Profiling is enabled by default; use `RUSTFS_BUFFER_PROFILE_DISABLE` to opt-out |
|
||||
|
||||
## Performance Expectations
|
||||
|
||||
### GeneralPurpose Profile (Default)
|
||||
|
||||
Same performance as PR #869 for most workloads:
|
||||
- Small files: Same 64KB buffer
|
||||
- Medium files: Same 256KB buffer
|
||||
- Large files: Same 1MB buffer
|
||||
|
||||
### Specialized Profiles
|
||||
|
||||
When you switch to a specialized profile, you get optimized buffer sizes:
|
||||
|
||||
| Profile | Performance Benefit | Use Case |
|
||||
|---------|-------------------|----------|
|
||||
| `AiTraining` | Up to 4x throughput on large files | ML model files, training datasets |
|
||||
| `WebWorkload` | Lower memory, higher concurrency | Static assets, CDN |
|
||||
| `DataAnalytics` | Balanced for mixed patterns | Data warehouses, BI |
|
||||
| `IndustrialIoT` | Low latency, memory-efficient | Sensor data, telemetry |
|
||||
| `SecureStorage` | Compliance-focused, minimal memory | Government, healthcare |
|
||||
|
||||
## Testing Your Migration
|
||||
|
||||
### Step 1: Test Default Behavior
|
||||
|
||||
```bash
|
||||
# Start with default configuration
|
||||
./rustfs /data
|
||||
|
||||
# Verify it works as expected
|
||||
# Check logs for: "Using buffer profile: GeneralPurpose"
|
||||
```
|
||||
|
||||
### Step 2: Test Your Workload Profile (If Using)
|
||||
|
||||
```bash
|
||||
# Set your specific profile
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Verify in logs: "Using buffer profile: AiTraining"
|
||||
```
|
||||
|
||||
### Step 3: Test Opt-Out (If Needed)
|
||||
|
||||
```bash
|
||||
# Disable profiling
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
|
||||
# Verify in logs: "using legacy adaptive buffer sizing"
|
||||
```
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
If you encounter any issues with Phase 3, you can easily roll back:
|
||||
|
||||
### Option 1: Disable Profiling
|
||||
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
This gives you the exact PR #869 behavior.
|
||||
|
||||
### Option 2: Use GeneralPurpose Profile Explicitly
|
||||
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE=GeneralPurpose
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
This uses profiling but with conservative buffer sizes.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Q: Will Phase 3 break my existing deployment?
|
||||
|
||||
**A:** No. The default `GeneralPurpose` profile uses the same buffer sizes as PR #869 for most scenarios. Your deployment will work exactly as before.
|
||||
|
||||
### Q: Do I need to change my configuration?
|
||||
|
||||
**A:** Only if you were explicitly using profiles in Phase 2. You can simplify by removing `RUSTFS_BUFFER_PROFILE_ENABLE=true`.
|
||||
|
||||
### Q: What if I want the exact legacy behavior?
|
||||
|
||||
**A:** Set `RUSTFS_BUFFER_PROFILE_DISABLE=true` to use the exact PR #869 algorithm.
|
||||
|
||||
### Q: Can I still use RUSTFS_BUFFER_PROFILE_ENABLE?
|
||||
|
||||
**A:** Yes, it's still supported for backward compatibility, but it's no longer necessary.
|
||||
|
||||
### Q: How do I know which profile is active?
|
||||
|
||||
**A:** Check the startup logs for messages like:
|
||||
- "Using buffer profile: GeneralPurpose"
|
||||
- "Buffer profiling is disabled, using legacy adaptive buffer sizing"
|
||||
|
||||
### Q: Should I switch to a specialized profile?
|
||||
|
||||
**A:** Only if you have specific workload characteristics:
|
||||
- AI/ML with large files → `AiTraining`
|
||||
- Web applications → `WebWorkload`
|
||||
- Secure/compliance environments → `SecureStorage`
|
||||
- Default is fine for most general-purpose workloads
|
||||
|
||||
## Support
|
||||
|
||||
If you encounter issues during migration:
|
||||
|
||||
1. Check logs for buffer profile information
|
||||
2. Try disabling profiling with `--buffer-profile-disable`
|
||||
3. Report issues with:
|
||||
- Your workload type
|
||||
- File sizes you're working with
|
||||
- Performance observations
|
||||
- Log excerpts showing buffer profile initialization
|
||||
|
||||
## Timeline
|
||||
|
||||
- **Phase 1:** Infrastructure (✅ Complete)
|
||||
- **Phase 2:** Opt-In Usage (✅ Complete)
|
||||
- **Phase 3:** Default Enablement (✅ Current - You are here)
|
||||
- **Phase 4:** Full Integration (Future)
|
||||
|
||||
## Conclusion
|
||||
|
||||
Phase 3 represents a smooth evolution of the adaptive buffer sizing feature. The default behavior remains compatible with PR #869, while providing an easy path to optimize for specific workloads when needed.
|
||||
|
||||
Most users can migrate without any changes, and those who need the exact legacy behavior can easily opt-out.
|
||||
@@ -1,383 +0,0 @@
|
||||
# Phase 4: Full Integration Guide
|
||||
|
||||
## Overview
|
||||
|
||||
Phase 4 represents the final stage of the adaptive buffer sizing migration path. It provides a unified, profile-based implementation with deprecated legacy functions and optional performance metrics.
|
||||
|
||||
## What's New in Phase 4
|
||||
|
||||
### 1. Deprecated Legacy Function
|
||||
|
||||
The `get_adaptive_buffer_size()` function is now deprecated:
|
||||
|
||||
```rust
|
||||
#[deprecated(
|
||||
since = "Phase 4",
|
||||
note = "Use workload profile configuration instead."
|
||||
)]
|
||||
fn get_adaptive_buffer_size(file_size: i64) -> usize
|
||||
```
|
||||
|
||||
**Why Deprecated?**
|
||||
- Profile-based approach is more flexible and powerful
|
||||
- Encourages use of the unified configuration system
|
||||
- Simplifies maintenance and future enhancements
|
||||
|
||||
**Still Works:**
|
||||
- Function is maintained for backward compatibility
|
||||
- Internally delegates to GeneralPurpose profile
|
||||
- No breaking changes for existing code
|
||||
|
||||
### 2. Profile-Only Implementation
|
||||
|
||||
All buffer sizing now goes through workload profiles:
|
||||
|
||||
**Before (Phase 3):**
|
||||
```rust
|
||||
fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
if is_buffer_profile_enabled() {
|
||||
// Use profiles
|
||||
} else {
|
||||
// Fall back to hardcoded get_adaptive_buffer_size()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**After (Phase 4):**
|
||||
```rust
|
||||
fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
if is_buffer_profile_enabled() {
|
||||
// Use configured profile
|
||||
} else {
|
||||
// Use GeneralPurpose profile (no hardcoded values)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Consistent behavior across all modes
|
||||
- Single source of truth for buffer sizes
|
||||
- Easier to test and maintain
|
||||
|
||||
### 3. Performance Metrics
|
||||
|
||||
Optional metrics collection for monitoring and optimization:
|
||||
|
||||
```rust
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
metrics::histogram!("buffer_size_bytes", buffer_size as f64);
|
||||
metrics::counter!("buffer_size_selections", 1);
|
||||
|
||||
if file_size >= 0 {
|
||||
let ratio = buffer_size as f64 / file_size as f64;
|
||||
metrics::histogram!("buffer_to_file_ratio", ratio);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### From Phase 3 to Phase 4
|
||||
|
||||
**Good News:** No action required for most users!
|
||||
|
||||
Phase 4 is fully backward compatible with Phase 3. Your existing configurations and deployments continue to work without changes.
|
||||
|
||||
### If You Have Custom Code
|
||||
|
||||
If your code directly calls `get_adaptive_buffer_size()`:
|
||||
|
||||
**Option 1: Update to use the profile system (Recommended)**
|
||||
```rust
|
||||
// Old code
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
|
||||
// New code - let the system handle it
|
||||
// (buffer sizing happens automatically in put_object, upload_part, etc.)
|
||||
```
|
||||
|
||||
**Option 2: Suppress deprecation warnings**
|
||||
```rust
|
||||
// If you must keep calling it directly
|
||||
#[allow(deprecated)]
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
```
|
||||
|
||||
**Option 3: Use the new API explicitly**
|
||||
```rust
|
||||
// Use the profile system directly
|
||||
use rustfs::config::workload_profiles::{WorkloadProfile, RustFSBufferConfig};
|
||||
|
||||
let config = RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose);
|
||||
let buffer_size = config.get_buffer_size(file_size);
|
||||
```
|
||||
|
||||
## Performance Metrics
|
||||
|
||||
### Enabling Metrics
|
||||
|
||||
**At Build Time:**
|
||||
```bash
|
||||
cargo build --features metrics --release
|
||||
```
|
||||
|
||||
**In Cargo.toml:**
|
||||
```toml
|
||||
[dependencies]
|
||||
rustfs = { version = "*", features = ["metrics"] }
|
||||
```
|
||||
|
||||
### Available Metrics
|
||||
|
||||
| Metric Name | Type | Description |
|
||||
|------------|------|-------------|
|
||||
| `buffer_size_bytes` | Histogram | Distribution of selected buffer sizes |
|
||||
| `buffer_size_selections` | Counter | Total number of buffer size calculations |
|
||||
| `buffer_to_file_ratio` | Histogram | Ratio of buffer size to file size |
|
||||
|
||||
### Using Metrics
|
||||
|
||||
**With Prometheus:**
|
||||
```rust
|
||||
// Metrics are automatically exported to Prometheus format
|
||||
// Access at http://localhost:9090/metrics
|
||||
```
|
||||
|
||||
**With Custom Backend:**
|
||||
```rust
|
||||
// Use the metrics crate's recorder interface
|
||||
use metrics_exporter_prometheus::PrometheusBuilder;
|
||||
|
||||
PrometheusBuilder::new()
|
||||
.install()
|
||||
.expect("failed to install Prometheus recorder");
|
||||
```
|
||||
|
||||
### Analyzing Metrics
|
||||
|
||||
**Buffer Size Distribution:**
|
||||
```promql
|
||||
# Most common buffer sizes
|
||||
histogram_quantile(0.5, buffer_size_bytes) # Median
|
||||
histogram_quantile(0.95, buffer_size_bytes) # 95th percentile
|
||||
histogram_quantile(0.99, buffer_size_bytes) # 99th percentile
|
||||
```
|
||||
|
||||
**Buffer Efficiency:**
|
||||
```promql
|
||||
# Average ratio of buffer to file size
|
||||
avg(buffer_to_file_ratio)
|
||||
|
||||
# Files where buffer is > 10% of file size
|
||||
buffer_to_file_ratio > 0.1
|
||||
```
|
||||
|
||||
**Usage Patterns:**
|
||||
```promql
|
||||
# Rate of buffer size selections
|
||||
rate(buffer_size_selections[5m])
|
||||
|
||||
# Total selections over time
|
||||
increase(buffer_size_selections[1h])
|
||||
```
|
||||
|
||||
## Optimizing Based on Metrics
|
||||
|
||||
### Scenario 1: High Memory Usage
|
||||
|
||||
**Symptom:** Most buffers are at maximum size
|
||||
```promql
|
||||
histogram_quantile(0.9, buffer_size_bytes) > 1048576 # 1MB
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Switch to a more conservative profile
|
||||
- Use SecureStorage or WebWorkload profile
|
||||
- Or create custom profile with lower max_size
|
||||
|
||||
### Scenario 2: Poor Throughput
|
||||
|
||||
**Symptom:** Buffer-to-file ratio is very small
|
||||
```promql
|
||||
avg(buffer_to_file_ratio) < 0.01 # Less than 1%
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Switch to a more aggressive profile
|
||||
- Use AiTraining or DataAnalytics profile
|
||||
- Increase buffer sizes for your workload
|
||||
|
||||
### Scenario 3: Mismatched Profile
|
||||
|
||||
**Symptom:** Wide distribution of file sizes with single profile
|
||||
```promql
|
||||
# High variance in buffer sizes
|
||||
stddev(buffer_size_bytes) > 500000
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Consider per-bucket profiles (future feature)
|
||||
- Use GeneralPurpose for mixed workloads
|
||||
- Or implement custom thresholds
|
||||
|
||||
## Testing Phase 4
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Run the Phase 4 specific tests:
|
||||
```bash
|
||||
cd /home/runner/work/rustfs/rustfs
|
||||
cargo test test_phase4_full_integration
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
Test with different configurations:
|
||||
```bash
|
||||
# Test default behavior
|
||||
./rustfs /data
|
||||
|
||||
# Test with different profiles
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Test opt-out mode
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
### Metrics Verification
|
||||
|
||||
With metrics enabled:
|
||||
```bash
|
||||
# Build with metrics
|
||||
cargo build --features metrics --release
|
||||
|
||||
# Run and check metrics endpoint
|
||||
./target/release/rustfs /data &
|
||||
curl http://localhost:9090/metrics | grep buffer_size
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Q: I'm getting deprecation warnings
|
||||
|
||||
**A:** You're calling `get_adaptive_buffer_size()` directly. Options:
|
||||
1. Remove the direct call (let the system handle it)
|
||||
2. Use `#[allow(deprecated)]` to suppress warnings
|
||||
3. Migrate to the profile system API
|
||||
|
||||
### Q: How do I know which profile is being used?
|
||||
|
||||
**A:** Check the startup logs:
|
||||
```
|
||||
Buffer profiling is enabled by default (Phase 3), profile: GeneralPurpose
|
||||
Using buffer profile: GeneralPurpose
|
||||
```
|
||||
|
||||
### Q: Can I still opt-out in Phase 4?
|
||||
|
||||
**A:** Yes! Use `--buffer-profile-disable`:
|
||||
```bash
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
This uses GeneralPurpose profile (same buffer sizes as PR #869).
|
||||
|
||||
### Q: What's the difference between opt-out in Phase 3 vs Phase 4?
|
||||
|
||||
**A:**
|
||||
- **Phase 3**: Opt-out uses hardcoded legacy function
|
||||
- **Phase 4**: Opt-out uses GeneralPurpose profile
|
||||
- **Result**: Identical buffer sizes, but Phase 4 is profile-based
|
||||
|
||||
### Q: Do I need to enable metrics?
|
||||
|
||||
**A:** No, metrics are completely optional. They're useful for:
|
||||
- Production monitoring
|
||||
- Performance analysis
|
||||
- Profile optimization
|
||||
- Capacity planning
|
||||
|
||||
If you don't need these, skip the metrics feature.
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Let the System Handle Buffer Sizing
|
||||
|
||||
**Don't:**
|
||||
```rust
|
||||
// Avoid direct calls
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
let reader = BufReader::with_capacity(buffer_size, file);
|
||||
```
|
||||
|
||||
**Do:**
|
||||
```rust
|
||||
// Let put_object/upload_part handle it automatically
|
||||
// Buffer sizing happens transparently
|
||||
```
|
||||
|
||||
### 2. Use Appropriate Profiles
|
||||
|
||||
Match your profile to your workload:
|
||||
- AI/ML models: `AiTraining`
|
||||
- Static assets: `WebWorkload`
|
||||
- Mixed files: `GeneralPurpose`
|
||||
- Compliance: `SecureStorage`
|
||||
|
||||
### 3. Monitor in Production
|
||||
|
||||
Enable metrics in production:
|
||||
```bash
|
||||
cargo build --features metrics --release
|
||||
```
|
||||
|
||||
Use the data to:
|
||||
- Validate profile choice
|
||||
- Identify optimization opportunities
|
||||
- Plan capacity
|
||||
|
||||
### 4. Test Profile Changes
|
||||
|
||||
Before changing profiles in production:
|
||||
```bash
|
||||
# Test in staging
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /staging-data
|
||||
|
||||
# Monitor metrics for a period
|
||||
# Compare with baseline
|
||||
|
||||
# Roll out to production when validated
|
||||
```
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
Based on collected metrics, future versions may include:
|
||||
|
||||
1. **Auto-tuning**: Automatically adjust profiles based on observed patterns
|
||||
2. **Per-bucket profiles**: Different profiles for different buckets
|
||||
3. **Dynamic thresholds**: Adjust thresholds based on system load
|
||||
4. **ML-based optimization**: Use machine learning to optimize buffer sizes
|
||||
5. **Adaptive limits**: Automatically adjust max_size based on available memory
|
||||
|
||||
## Conclusion
|
||||
|
||||
Phase 4 represents the mature state of the adaptive buffer sizing system:
|
||||
- ✅ Unified, profile-based implementation
|
||||
- ✅ Deprecated legacy code (but backward compatible)
|
||||
- ✅ Optional performance metrics
|
||||
- ✅ Production-ready and battle-tested
|
||||
- ✅ Future-proof and extensible
|
||||
|
||||
Most users can continue using the system without any changes, while advanced users gain powerful new capabilities for monitoring and optimization.
|
||||
|
||||
## References
|
||||
|
||||
- [Adaptive Buffer Sizing Guide](./adaptive-buffer-sizing.md)
|
||||
- [Implementation Summary](./IMPLEMENTATION_SUMMARY.md)
|
||||
- [Phase 3 Migration Guide](./MIGRATION_PHASE3.md)
|
||||
- [Performance Testing Guide](./PERFORMANCE_TESTING.md)
|
||||
@@ -4,17 +4,6 @@ Welcome to the RustFS distributed file system documentation center!
|
||||
|
||||
## 📚 Documentation Navigation
|
||||
|
||||
### ⚡ Performance Optimization
|
||||
|
||||
RustFS provides intelligent performance optimization features for different workloads.
|
||||
|
||||
| Document | Description | Audience |
|
||||
|------|------|----------|
|
||||
| [Adaptive Buffer Sizing](./adaptive-buffer-sizing.md) | Intelligent buffer sizing optimization for optimal performance across workload types | Developers and system administrators |
|
||||
| [Phase 3 Migration Guide](./MIGRATION_PHASE3.md) | Migration guide from Phase 2 to Phase 3 (Default Enablement) | Operations and DevOps teams |
|
||||
| [Phase 4 Full Integration Guide](./PHASE4_GUIDE.md) | Complete guide to Phase 4 features: deprecated legacy functions, performance metrics | Advanced users and performance engineers |
|
||||
| [Performance Testing Guide](./PERFORMANCE_TESTING.md) | Performance benchmarking and optimization guide | Performance engineers |
|
||||
|
||||
### 🔐 KMS (Key Management Service)
|
||||
|
||||
RustFS KMS delivers enterprise-grade key management and data encryption.
|
||||
|
||||
@@ -1,765 +0,0 @@
|
||||
# Adaptive Buffer Sizing Optimization
|
||||
|
||||
RustFS implements intelligent adaptive buffer sizing optimization that automatically adjusts buffer sizes based on file size and workload type to achieve optimal balance between performance, memory usage, and security.
|
||||
|
||||
## Overview
|
||||
|
||||
The adaptive buffer sizing system provides:
|
||||
|
||||
- **Automatic buffer size selection** based on file size
|
||||
- **Workload-specific optimizations** for different use cases
|
||||
- **Special environment support** (Kylin, NeoKylin, Unity OS, etc.)
|
||||
- **Memory pressure awareness** with configurable limits
|
||||
- **Unknown file size handling** for streaming scenarios
|
||||
|
||||
## Workload Profiles
|
||||
|
||||
### GeneralPurpose (Default)
|
||||
|
||||
Balanced performance and memory usage for general-purpose workloads.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 1MB): 64KB buffer
|
||||
- Medium files (1MB-100MB): 256KB buffer
|
||||
- Large files (≥ 100MB): 1MB buffer
|
||||
|
||||
**Best for:**
|
||||
- General file storage
|
||||
- Mixed workloads
|
||||
- Default configuration when workload type is unknown
|
||||
|
||||
### AiTraining
|
||||
|
||||
Optimized for AI/ML training workloads with large sequential reads.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 10MB): 512KB buffer
|
||||
- Medium files (10MB-500MB): 2MB buffer
|
||||
- Large files (≥ 500MB): 4MB buffer
|
||||
|
||||
**Best for:**
|
||||
- Machine learning model files
|
||||
- Training datasets
|
||||
- Large sequential data processing
|
||||
- Maximum throughput requirements
|
||||
|
||||
### DataAnalytics
|
||||
|
||||
Optimized for data analytics with mixed read-write patterns.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 5MB): 128KB buffer
|
||||
- Medium files (5MB-200MB): 512KB buffer
|
||||
- Large files (≥ 200MB): 2MB buffer
|
||||
|
||||
**Best for:**
|
||||
- Data warehouse operations
|
||||
- Analytics workloads
|
||||
- Business intelligence
|
||||
- Mixed access patterns
|
||||
|
||||
### WebWorkload
|
||||
|
||||
Optimized for web applications with small file intensive operations.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 512KB): 32KB buffer
|
||||
- Medium files (512KB-10MB): 128KB buffer
|
||||
- Large files (≥ 10MB): 256KB buffer
|
||||
|
||||
**Best for:**
|
||||
- Web assets (images, CSS, JavaScript)
|
||||
- Static content delivery
|
||||
- CDN origin storage
|
||||
- High concurrency scenarios
|
||||
|
||||
### IndustrialIoT
|
||||
|
||||
Optimized for industrial IoT with real-time streaming requirements.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 1MB): 64KB buffer
|
||||
- Medium files (1MB-50MB): 256KB buffer
|
||||
- Large files (≥ 50MB): 512KB buffer (capped for memory constraints)
|
||||
|
||||
**Best for:**
|
||||
- Sensor data streams
|
||||
- Real-time telemetry
|
||||
- Edge computing scenarios
|
||||
- Low latency requirements
|
||||
- Memory-constrained devices
|
||||
|
||||
### SecureStorage
|
||||
|
||||
Security-first configuration with strict memory limits for compliance.
|
||||
|
||||
**Buffer Sizing:**
|
||||
- Small files (< 1MB): 32KB buffer
|
||||
- Medium files (1MB-50MB): 128KB buffer
|
||||
- Large files (≥ 50MB): 256KB buffer (strict limit)
|
||||
|
||||
**Best for:**
|
||||
- Compliance-heavy environments
|
||||
- Secure government systems (Kylin, NeoKylin, UOS)
|
||||
- Financial services
|
||||
- Healthcare data storage
|
||||
- Memory-constrained secure environments
|
||||
|
||||
**Auto-Detection:**
|
||||
This profile is automatically selected when running on Chinese secure operating systems:
|
||||
- Kylin
|
||||
- NeoKylin
|
||||
- UOS (Unity OS)
|
||||
- OpenKylin
|
||||
|
||||
## Usage
|
||||
|
||||
### Using Default Configuration
|
||||
|
||||
The system automatically uses the `GeneralPurpose` profile by default:
|
||||
|
||||
```rust
|
||||
// The buffer size is automatically calculated based on file size
|
||||
// Uses GeneralPurpose profile by default
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
```
|
||||
|
||||
### Using Specific Workload Profile
|
||||
|
||||
```rust
|
||||
use rustfs::config::workload_profiles::WorkloadProfile;
|
||||
|
||||
// For AI/ML workloads
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
file_size,
|
||||
Some(WorkloadProfile::AiTraining)
|
||||
);
|
||||
|
||||
// For web workloads
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
file_size,
|
||||
Some(WorkloadProfile::WebWorkload)
|
||||
);
|
||||
|
||||
// For secure storage
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
file_size,
|
||||
Some(WorkloadProfile::SecureStorage)
|
||||
);
|
||||
```
|
||||
|
||||
### Auto-Detection Mode
|
||||
|
||||
The system can automatically detect the runtime environment:
|
||||
|
||||
```rust
|
||||
// Auto-detects OS environment or falls back to GeneralPurpose
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(file_size, None);
|
||||
```
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
For specialized requirements, create a custom configuration:
|
||||
|
||||
```rust
|
||||
use rustfs::config::workload_profiles::{BufferConfig, WorkloadProfile};
|
||||
|
||||
let custom_config = BufferConfig {
|
||||
min_size: 16 * 1024, // 16KB minimum
|
||||
max_size: 512 * 1024, // 512KB maximum
|
||||
default_unknown: 128 * 1024, // 128KB for unknown sizes
|
||||
thresholds: vec![
|
||||
(1024 * 1024, 64 * 1024), // < 1MB: 64KB
|
||||
(50 * 1024 * 1024, 256 * 1024), // 1MB-50MB: 256KB
|
||||
(i64::MAX, 512 * 1024), // >= 50MB: 512KB
|
||||
],
|
||||
};
|
||||
|
||||
let profile = WorkloadProfile::Custom(custom_config);
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(file_size, Some(profile));
|
||||
```
|
||||
|
||||
## Phase 3: Default Enablement (Current Implementation)
|
||||
|
||||
**⚡ NEW: Workload profiles are now enabled by default!**
|
||||
|
||||
Starting from Phase 3, adaptive buffer sizing with workload profiles is **enabled by default** using the `GeneralPurpose` profile. This provides improved performance out-of-the-box while maintaining full backward compatibility.
|
||||
|
||||
### Default Behavior
|
||||
|
||||
```bash
|
||||
# Phase 3: Profile-aware buffer sizing enabled by default with GeneralPurpose profile
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
This now automatically uses intelligent buffer sizing based on file size and workload characteristics.
|
||||
|
||||
### Changing the Workload Profile
|
||||
|
||||
```bash
|
||||
# Use a different profile (AI/ML workloads)
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Or via command-line
|
||||
./rustfs --buffer-profile AiTraining /data
|
||||
|
||||
# Use web workload profile
|
||||
./rustfs --buffer-profile WebWorkload /data
|
||||
```
|
||||
|
||||
### Opt-Out (Legacy Behavior)
|
||||
|
||||
If you need the exact behavior from PR #869 (fixed algorithm), you can disable profiling:
|
||||
|
||||
```bash
|
||||
# Disable buffer profiling (revert to PR #869 behavior)
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
|
||||
# Or via command-line
|
||||
./rustfs --buffer-profile-disable /data
|
||||
```
|
||||
|
||||
### Available Profile Names
|
||||
|
||||
The following profile names are supported (case-insensitive):
|
||||
|
||||
| Profile Name | Aliases | Description |
|
||||
|-------------|---------|-------------|
|
||||
| `GeneralPurpose` | `general` | Default balanced configuration (same as PR #869 for most files) |
|
||||
| `AiTraining` | `ai` | Optimized for AI/ML workloads |
|
||||
| `DataAnalytics` | `analytics` | Mixed read-write patterns |
|
||||
| `WebWorkload` | `web` | Small file intensive operations |
|
||||
| `IndustrialIoT` | `iot` | Real-time streaming |
|
||||
| `SecureStorage` | `secure` | Security-first, memory constrained |
|
||||
|
||||
### Behavior Summary
|
||||
|
||||
**Phase 3 Default (Enabled):**
|
||||
- Uses workload-aware buffer sizing with `GeneralPurpose` profile
|
||||
- Provides same buffer sizes as PR #869 for most scenarios
|
||||
- Allows easy switching to specialized profiles
|
||||
- Buffer sizes: 64KB, 256KB, 1MB based on file size (GeneralPurpose)
|
||||
|
||||
**With `RUSTFS_BUFFER_PROFILE_DISABLE=true`:**
|
||||
- Uses the exact original adaptive buffer sizing from PR #869
|
||||
- For users who want guaranteed legacy behavior
|
||||
- Buffer sizes: 64KB, 256KB, 1MB based on file size
|
||||
|
||||
**With Different Profiles:**
|
||||
- `AiTraining`: 512KB, 2MB, 4MB - maximize throughput
|
||||
- `WebWorkload`: 32KB, 128KB, 256KB - optimize concurrency
|
||||
- `SecureStorage`: 32KB, 128KB, 256KB - compliance-focused
|
||||
- And more...
|
||||
|
||||
### Migration Examples
|
||||
|
||||
**Phase 2 → Phase 3 Migration:**
|
||||
|
||||
```bash
|
||||
# Phase 2 (Opt-In): Had to explicitly enable
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
export RUSTFS_BUFFER_PROFILE=GeneralPurpose
|
||||
./rustfs /data
|
||||
|
||||
# Phase 3 (Default): Enabled automatically
|
||||
./rustfs /data # ← Same behavior, no configuration needed!
|
||||
```
|
||||
|
||||
**Using Different Profiles:**
|
||||
|
||||
```bash
|
||||
# AI/ML workloads - larger buffers for maximum throughput
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Web workloads - smaller buffers for high concurrency
|
||||
export RUSTFS_BUFFER_PROFILE=WebWorkload
|
||||
./rustfs /data
|
||||
|
||||
# Secure environments - compliance-focused
|
||||
export RUSTFS_BUFFER_PROFILE=SecureStorage
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
**Reverting to Legacy Behavior:**
|
||||
|
||||
```bash
|
||||
# If you encounter issues or need exact PR #869 behavior
|
||||
export RUSTFS_BUFFER_PROFILE_DISABLE=true
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
## Phase 4: Full Integration (Current Implementation)
|
||||
|
||||
**🚀 NEW: Profile-only implementation with performance metrics!**
|
||||
|
||||
Phase 4 represents the final stage of the adaptive buffer sizing system, providing a unified, profile-based approach with optional performance monitoring.
|
||||
|
||||
### Key Features
|
||||
|
||||
1. **Deprecated Legacy Function**
|
||||
- `get_adaptive_buffer_size()` is now deprecated
|
||||
- Maintained for backward compatibility only
|
||||
- All new code uses the workload profile system
|
||||
|
||||
2. **Profile-Only Implementation**
|
||||
- Single entry point: `get_buffer_size_opt_in()`
|
||||
- All buffer sizes come from workload profiles
|
||||
- Even "disabled" mode uses GeneralPurpose profile (no hardcoded values)
|
||||
|
||||
3. **Performance Metrics** (Optional)
|
||||
- Built-in metrics collection with `metrics` feature flag
|
||||
- Tracks buffer size selections
|
||||
- Monitors buffer-to-file size ratios
|
||||
- Helps optimize profile configurations
|
||||
|
||||
### Unified Buffer Sizing
|
||||
|
||||
```rust
|
||||
// Phase 4: Single, unified implementation
|
||||
fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
// Enabled by default (Phase 3)
|
||||
// Uses workload profiles exclusively
|
||||
// Optional metrics collection
|
||||
}
|
||||
```
|
||||
|
||||
### Performance Monitoring
|
||||
|
||||
When compiled with the `metrics` feature flag:
|
||||
|
||||
```bash
|
||||
# Build with metrics support
|
||||
cargo build --features metrics
|
||||
|
||||
# Run and collect metrics
|
||||
./rustfs /data
|
||||
|
||||
# Metrics collected:
|
||||
# - buffer_size_bytes: Histogram of selected buffer sizes
|
||||
# - buffer_size_selections: Counter of buffer size calculations
|
||||
# - buffer_to_file_ratio: Ratio of buffer size to file size
|
||||
```
|
||||
|
||||
### Migration from Phase 3
|
||||
|
||||
No action required! Phase 4 is fully backward compatible with Phase 3:
|
||||
|
||||
```bash
|
||||
# Phase 3 usage continues to work
|
||||
./rustfs /data
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
./rustfs /data
|
||||
|
||||
# Phase 4 adds deprecation warnings for direct legacy function calls
|
||||
# (if you have custom code calling get_adaptive_buffer_size)
|
||||
```
|
||||
|
||||
### What Changed
|
||||
|
||||
| Aspect | Phase 3 | Phase 4 |
|
||||
|--------|---------|---------|
|
||||
| Legacy Function | Active | Deprecated (still works) |
|
||||
| Implementation | Hybrid (legacy fallback) | Profile-only |
|
||||
| Metrics | None | Optional via feature flag |
|
||||
| Buffer Source | Profiles or hardcoded | Profiles only |
|
||||
|
||||
### Benefits
|
||||
|
||||
1. **Simplified Codebase**
|
||||
- Single implementation path
|
||||
- Easier to maintain and optimize
|
||||
- Consistent behavior across all scenarios
|
||||
|
||||
2. **Better Observability**
|
||||
- Optional metrics for performance monitoring
|
||||
- Data-driven profile optimization
|
||||
- Production usage insights
|
||||
|
||||
3. **Future-Proof**
|
||||
- No legacy code dependencies
|
||||
- Easy to add new profiles
|
||||
- Extensible for future enhancements
|
||||
|
||||
### Code Example
|
||||
|
||||
**Phase 3 (Still Works):**
|
||||
```rust
|
||||
// Enabled by default
|
||||
let buffer_size = get_buffer_size_opt_in(file_size);
|
||||
```
|
||||
|
||||
**Phase 4 (Recommended):**
|
||||
```rust
|
||||
// Same call, but now with optional metrics and profile-only implementation
|
||||
let buffer_size = get_buffer_size_opt_in(file_size);
|
||||
// Metrics automatically collected if feature enabled
|
||||
```
|
||||
|
||||
**Deprecated (Backward Compatible):**
|
||||
```rust
|
||||
// This still works but generates deprecation warnings
|
||||
#[allow(deprecated)]
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
```
|
||||
|
||||
### Enabling Metrics
|
||||
|
||||
Add to `Cargo.toml`:
|
||||
```toml
|
||||
[dependencies]
|
||||
rustfs = { version = "*", features = ["metrics"] }
|
||||
```
|
||||
|
||||
Or build with feature flag:
|
||||
```bash
|
||||
cargo build --features metrics --release
|
||||
```
|
||||
|
||||
### Metrics Dashboard
|
||||
|
||||
When metrics are enabled, you can visualize:
|
||||
|
||||
- **Buffer Size Distribution**: Most common buffer sizes used
|
||||
- **Profile Effectiveness**: How well profiles match actual workloads
|
||||
- **Memory Efficiency**: Buffer-to-file size ratios
|
||||
- **Usage Patterns**: File size distribution and buffer selection trends
|
||||
|
||||
Use your preferred metrics backend (Prometheus, InfluxDB, etc.) to collect and visualize these metrics.
|
||||
|
||||
## Phase 2: Opt-In Usage (Previous Implementation)
|
||||
|
||||
**Note:** Phase 2 documentation is kept for historical reference. The current version uses Phase 4 (Full Integration).
|
||||
|
||||
<details>
|
||||
<summary>Click to expand Phase 2 documentation</summary>
|
||||
|
||||
Starting from Phase 2 of the migration path, workload profiles can be enabled via environment variables or command-line arguments.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Enable workload profiling using these environment variables:
|
||||
|
||||
```bash
|
||||
# Enable buffer profiling (opt-in)
|
||||
export RUSTFS_BUFFER_PROFILE_ENABLE=true
|
||||
|
||||
# Set the workload profile
|
||||
export RUSTFS_BUFFER_PROFILE=AiTraining
|
||||
|
||||
# Start RustFS
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
### Command-Line Arguments
|
||||
|
||||
Alternatively, use command-line flags:
|
||||
|
||||
```bash
|
||||
# Enable buffer profiling with AI training profile
|
||||
./rustfs --buffer-profile-enable --buffer-profile AiTraining /data
|
||||
|
||||
# Enable buffer profiling with web workload profile
|
||||
./rustfs --buffer-profile-enable --buffer-profile WebWorkload /data
|
||||
|
||||
# Disable buffer profiling (use legacy behavior)
|
||||
./rustfs /data
|
||||
```
|
||||
|
||||
### Behavior
|
||||
|
||||
When `RUSTFS_BUFFER_PROFILE_ENABLE=false` (default in Phase 2):
|
||||
- Uses the original adaptive buffer sizing from PR #869
|
||||
- No breaking changes to existing deployments
|
||||
- Buffer sizes: 64KB, 256KB, 1MB based on file size
|
||||
|
||||
When `RUSTFS_BUFFER_PROFILE_ENABLE=true`:
|
||||
- Uses the configured workload profile
|
||||
- Allows for workload-specific optimizations
|
||||
- Buffer sizes vary based on the selected profile
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
|
||||
## Configuration Validation
|
||||
|
||||
All buffer configurations are validated to ensure correctness:
|
||||
|
||||
```rust
|
||||
let config = BufferConfig { /* ... */ };
|
||||
config.validate()?; // Returns Err if invalid
|
||||
```
|
||||
|
||||
**Validation Rules:**
|
||||
- `min_size` must be > 0
|
||||
- `max_size` must be >= `min_size`
|
||||
- `default_unknown` must be between `min_size` and `max_size`
|
||||
- Thresholds must be in ascending order
|
||||
- Buffer sizes in thresholds must be within `[min_size, max_size]`
|
||||
|
||||
## Environment Detection
|
||||
|
||||
The system automatically detects special operating system environments by reading `/etc/os-release` on Linux systems:
|
||||
|
||||
```rust
|
||||
if let Some(profile) = WorkloadProfile::detect_os_environment() {
|
||||
// Returns SecureStorage profile for Kylin, NeoKylin, UOS, etc.
|
||||
let buffer_size = profile.config().calculate_buffer_size(file_size);
|
||||
}
|
||||
```
|
||||
|
||||
**Detected Environments:**
|
||||
- Kylin (麒麟)
|
||||
- NeoKylin (中标麒麟)
|
||||
- UOS / Unity OS (统信)
|
||||
- OpenKylin (开放麒麟)
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### Memory Usage
|
||||
|
||||
Different profiles have different memory footprints:
|
||||
|
||||
| Profile | Min Buffer | Max Buffer | Typical Memory |
|
||||
|---------|-----------|-----------|----------------|
|
||||
| GeneralPurpose | 64KB | 1MB | Low-Medium |
|
||||
| AiTraining | 512KB | 4MB | High |
|
||||
| DataAnalytics | 128KB | 2MB | Medium |
|
||||
| WebWorkload | 32KB | 256KB | Low |
|
||||
| IndustrialIoT | 64KB | 512KB | Low |
|
||||
| SecureStorage | 32KB | 256KB | Low |
|
||||
|
||||
### Throughput Impact
|
||||
|
||||
Larger buffers generally provide better throughput for large files by reducing system call overhead:
|
||||
|
||||
- **Small buffers (32-64KB)**: Lower memory, more syscalls, suitable for many small files
|
||||
- **Medium buffers (128-512KB)**: Balanced approach for mixed workloads
|
||||
- **Large buffers (1-4MB)**: Maximum throughput, best for large sequential reads
|
||||
|
||||
### Concurrency Considerations
|
||||
|
||||
For high-concurrency scenarios (e.g., WebWorkload):
|
||||
- Smaller buffers reduce per-connection memory
|
||||
- Allows more concurrent connections
|
||||
- Better overall system resource utilization
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Choose the Right Profile
|
||||
|
||||
Select the profile that matches your primary workload:
|
||||
|
||||
```rust
|
||||
// AI/ML training
|
||||
WorkloadProfile::AiTraining
|
||||
|
||||
// Web application
|
||||
WorkloadProfile::WebWorkload
|
||||
|
||||
// General purpose storage
|
||||
WorkloadProfile::GeneralPurpose
|
||||
```
|
||||
|
||||
### 2. Monitor Memory Usage
|
||||
|
||||
In production, monitor memory consumption:
|
||||
|
||||
```rust
|
||||
// For memory-constrained environments, use smaller buffers
|
||||
WorkloadProfile::SecureStorage // or IndustrialIoT
|
||||
```
|
||||
|
||||
### 3. Test Performance
|
||||
|
||||
Benchmark your specific workload to verify the profile choice:
|
||||
|
||||
```bash
|
||||
# Run performance tests with different profiles
|
||||
cargo test --release -- --ignored performance_tests
|
||||
```
|
||||
|
||||
### 4. Consider File Size Distribution
|
||||
|
||||
If you know your typical file sizes:
|
||||
|
||||
- Mostly small files (< 1MB): Use `WebWorkload` or `SecureStorage`
|
||||
- Mostly large files (> 100MB): Use `AiTraining` or `DataAnalytics`
|
||||
- Mixed sizes: Use `GeneralPurpose`
|
||||
|
||||
### 5. Compliance Requirements
|
||||
|
||||
For regulated environments:
|
||||
|
||||
```rust
|
||||
// Automatically uses SecureStorage on detected secure OS
|
||||
let config = RustFSBufferConfig::with_auto_detect();
|
||||
|
||||
// Or explicitly set SecureStorage
|
||||
let config = RustFSBufferConfig::new(WorkloadProfile::SecureStorage);
|
||||
```
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### S3 Put Object
|
||||
|
||||
```rust
|
||||
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let size = req.input.content_length.unwrap_or(-1);
|
||||
|
||||
// Use workload-aware buffer sizing
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
size,
|
||||
Some(WorkloadProfile::GeneralPurpose)
|
||||
);
|
||||
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body)
|
||||
);
|
||||
|
||||
// Process upload...
|
||||
}
|
||||
```
|
||||
|
||||
### Multipart Upload
|
||||
|
||||
```rust
|
||||
async fn upload_part(&self, req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
let size = req.input.content_length.unwrap_or(-1);
|
||||
|
||||
// For large multipart uploads, consider using AiTraining profile
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
size,
|
||||
Some(WorkloadProfile::AiTraining)
|
||||
);
|
||||
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body_stream)
|
||||
);
|
||||
|
||||
// Process part upload...
|
||||
}
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### High Memory Usage
|
||||
|
||||
If experiencing high memory usage:
|
||||
|
||||
1. Switch to a more conservative profile:
|
||||
```rust
|
||||
WorkloadProfile::WebWorkload // or SecureStorage
|
||||
```
|
||||
|
||||
2. Set explicit memory limits in custom configuration:
|
||||
```rust
|
||||
let config = BufferConfig {
|
||||
min_size: 16 * 1024,
|
||||
max_size: 128 * 1024, // Cap at 128KB
|
||||
// ...
|
||||
};
|
||||
```
|
||||
|
||||
### Low Throughput
|
||||
|
||||
If experiencing low throughput for large files:
|
||||
|
||||
1. Use a more aggressive profile:
|
||||
```rust
|
||||
WorkloadProfile::AiTraining // or DataAnalytics
|
||||
```
|
||||
|
||||
2. Increase buffer sizes in custom configuration:
|
||||
```rust
|
||||
let config = BufferConfig {
|
||||
max_size: 4 * 1024 * 1024, // 4MB max buffer
|
||||
// ...
|
||||
};
|
||||
```
|
||||
|
||||
### Streaming/Unknown Size Handling
|
||||
|
||||
For chunked transfers or streaming:
|
||||
|
||||
```rust
|
||||
// Pass -1 for unknown size
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(-1, None);
|
||||
// Returns the profile's default_unknown size
|
||||
```
|
||||
|
||||
## Technical Implementation
|
||||
|
||||
### Algorithm
|
||||
|
||||
The buffer size is selected based on file size thresholds:
|
||||
|
||||
```rust
|
||||
pub fn calculate_buffer_size(&self, file_size: i64) -> usize {
|
||||
if file_size < 0 {
|
||||
return self.default_unknown;
|
||||
}
|
||||
|
||||
for (threshold, buffer_size) in &self.thresholds {
|
||||
if file_size < *threshold {
|
||||
return (*buffer_size).clamp(self.min_size, self.max_size);
|
||||
}
|
||||
}
|
||||
|
||||
self.max_size
|
||||
}
|
||||
```
|
||||
|
||||
### Thread Safety
|
||||
|
||||
All configuration structures are:
|
||||
- Immutable after creation
|
||||
- Safe to share across threads
|
||||
- Cloneable for per-thread customization
|
||||
|
||||
### Performance Overhead
|
||||
|
||||
- Configuration lookup: O(n) where n = number of thresholds (typically 2-4)
|
||||
- Negligible overhead compared to I/O operations
|
||||
- Configuration can be cached per-connection
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### From PR #869
|
||||
|
||||
The original `get_adaptive_buffer_size` function is preserved for backward compatibility:
|
||||
|
||||
```rust
|
||||
// Old code (still works)
|
||||
let buffer_size = get_adaptive_buffer_size(file_size);
|
||||
|
||||
// New code (recommended)
|
||||
let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
file_size,
|
||||
Some(WorkloadProfile::GeneralPurpose)
|
||||
);
|
||||
```
|
||||
|
||||
### Upgrading Existing Code
|
||||
|
||||
1. **Identify workload type** for each use case
|
||||
2. **Replace** `get_adaptive_buffer_size` with `get_adaptive_buffer_size_with_profile`
|
||||
3. **Choose** appropriate profile
|
||||
4. **Test** performance impact
|
||||
|
||||
## References
|
||||
|
||||
- [PR #869: Fix large file upload freeze with adaptive buffer sizing](https://github.com/rustfs/rustfs/pull/869)
|
||||
- [Performance Testing Guide](./PERFORMANCE_TESTING.md)
|
||||
- [Configuration Documentation](./ENVIRONMENT_VARIABLES.md)
|
||||
|
||||
## License
|
||||
|
||||
Copyright 2024 RustFS Team
|
||||
|
||||
Licensed under the Apache License, Version 2.0.
|
||||
@@ -1,192 +0,0 @@
|
||||
# Fix for Large File Upload Freeze Issue
|
||||
|
||||
## Problem Description
|
||||
|
||||
When uploading large files (10GB-20GB) consecutively, uploads may freeze with the following error:
|
||||
|
||||
```
|
||||
[2025-11-10 14:29:22.110443 +00:00] ERROR [s3s::service]
|
||||
AwsChunkedStreamError: Underlying: error reading a body from connection
|
||||
```
|
||||
|
||||
## Root Cause Analysis
|
||||
|
||||
### 1. Small Default Buffer Size
|
||||
The issue was caused by using `tokio_util::io::StreamReader::new()` which has a default buffer size of only **8KB**. This is far too small for large file uploads and causes:
|
||||
|
||||
- **Excessive system calls**: For a 10GB file with 8KB buffer, approximately **1.3 million read operations** are required
|
||||
- **High CPU overhead**: Each read involves AWS chunked encoding/decoding overhead
|
||||
- **Memory allocation pressure**: Frequent small allocations and deallocations
|
||||
- **Increased timeout risk**: Slow read pace can trigger connection timeouts
|
||||
|
||||
### 2. AWS Chunked Encoding Overhead
|
||||
AWS S3 uses chunked transfer encoding which adds metadata to each chunk. With a small buffer:
|
||||
- More chunks need to be processed
|
||||
- More metadata parsing operations
|
||||
- Higher probability of parsing errors or timeouts
|
||||
|
||||
### 3. Connection Timeout Under Load
|
||||
When multiple large files are uploaded consecutively:
|
||||
- Small buffers lead to slow data transfer rates
|
||||
- Network connections may timeout waiting for data
|
||||
- The s3s library reports "error reading a body from connection"
|
||||
|
||||
## Solution
|
||||
|
||||
Wrap `StreamReader::new()` with `tokio::io::BufReader::with_capacity()` using a 1MB buffer size (`DEFAULT_READ_BUFFER_SIZE = 1024 * 1024`).
|
||||
|
||||
### Changes Made
|
||||
|
||||
Modified three critical locations in `rustfs/src/storage/ecfs.rs`:
|
||||
|
||||
1. **put_object** (line ~2338): Standard object upload
|
||||
2. **put_object_extract** (line ~376): Archive file extraction and upload
|
||||
3. **upload_part** (line ~2864): Multipart upload
|
||||
|
||||
### Before
|
||||
```rust
|
||||
let body = StreamReader::new(
|
||||
body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))
|
||||
);
|
||||
```
|
||||
|
||||
### After
|
||||
```rust
|
||||
// Use a larger buffer size (1MB) for StreamReader to prevent chunked stream read timeouts
|
||||
// when uploading large files (10GB+). The default 8KB buffer is too small and causes
|
||||
// excessive syscalls and potential connection timeouts.
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
DEFAULT_READ_BUFFER_SIZE,
|
||||
StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
```
|
||||
|
||||
## Performance Impact
|
||||
|
||||
### For a 10GB File Upload:
|
||||
|
||||
| Metric | Before (8KB buffer) | After (1MB buffer) | Improvement |
|
||||
|--------|--------------------|--------------------|-------------|
|
||||
| Read operations | ~1,310,720 | ~10,240 | **99.2% reduction** |
|
||||
| System call overhead | High | Low | Significantly reduced |
|
||||
| Memory allocations | Frequent small | Less frequent large | More efficient |
|
||||
| Timeout risk | High | Low | Much more stable |
|
||||
|
||||
### Benefits
|
||||
|
||||
1. **Reduced System Calls**: ~99% reduction in read operations for large files
|
||||
2. **Lower CPU Usage**: Less AWS chunked encoding/decoding overhead
|
||||
3. **Better Memory Efficiency**: Fewer allocations and better cache locality
|
||||
4. **Improved Reliability**: Significantly reduced timeout probability
|
||||
5. **Higher Throughput**: Better network utilization
|
||||
|
||||
## Testing Recommendations
|
||||
|
||||
To verify the fix works correctly, test the following scenarios:
|
||||
|
||||
1. **Single Large File Upload**
|
||||
- Upload a 10GB file
|
||||
- Upload a 20GB file
|
||||
- Monitor for timeout errors
|
||||
|
||||
2. **Consecutive Large File Uploads**
|
||||
- Upload 5 files of 10GB each consecutively
|
||||
- Upload 3 files of 20GB each consecutively
|
||||
- Ensure no freezing or timeout errors
|
||||
|
||||
3. **Multipart Upload**
|
||||
- Upload large files using multipart upload
|
||||
- Test with various part sizes
|
||||
- Verify all parts complete successfully
|
||||
|
||||
4. **Archive Extraction**
|
||||
- Upload large tar/gzip files with X-Amz-Meta-Snowball-Auto-Extract
|
||||
- Verify extraction completes without errors
|
||||
|
||||
## Monitoring
|
||||
|
||||
After deployment, monitor these metrics:
|
||||
|
||||
- Upload completion rate for files > 1GB
|
||||
- Average upload time for large files
|
||||
- Frequency of chunked stream errors
|
||||
- CPU usage during uploads
|
||||
- Memory usage during uploads
|
||||
|
||||
## Related Configuration
|
||||
|
||||
The buffer size is defined in `crates/ecstore/src/set_disk.rs`:
|
||||
|
||||
```rust
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
```
|
||||
|
||||
This value is used consistently across the codebase for stream reading operations.
|
||||
|
||||
## Additional Considerations
|
||||
|
||||
### Implementation Details
|
||||
|
||||
The solution uses `tokio::io::BufReader` to wrap the `StreamReader`, as `tokio-util 0.7.17` does not provide a `StreamReader::with_capacity()` method. The `BufReader` provides the same buffering benefits while being compatible with the current tokio-util version.
|
||||
|
||||
### Adaptive Buffer Sizing (Implemented)
|
||||
|
||||
The fix now includes **dynamic adaptive buffer sizing** based on file size for optimal performance and memory usage:
|
||||
|
||||
```rust
|
||||
/// Calculate adaptive buffer size based on file size for optimal streaming performance.
|
||||
fn get_adaptive_buffer_size(file_size: i64) -> usize {
|
||||
match file_size {
|
||||
// Unknown size or negative (chunked/streaming): use 1MB buffer for safety
|
||||
size if size < 0 => 1024 * 1024,
|
||||
// Small files (< 1MB): use 64KB to minimize memory overhead
|
||||
size if size < 1_048_576 => 65_536,
|
||||
// Medium files (1MB - 100MB): use 256KB for balanced performance
|
||||
size if size < 104_857_600 => 262_144,
|
||||
// Large files (>= 100MB): use 1MB buffer for maximum throughput
|
||||
_ => 1024 * 1024,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Benefits**:
|
||||
- **Memory Efficiency**: Small files use smaller buffers (64KB), reducing memory overhead
|
||||
- **Balanced Performance**: Medium files use 256KB buffers for optimal balance
|
||||
- **Maximum Throughput**: Large files (100MB+) use 1MB buffers to minimize syscalls
|
||||
- **Automatic Selection**: Buffer size is chosen automatically based on content-length
|
||||
|
||||
**Performance Impact by File Size**:
|
||||
|
||||
| File Size | Buffer Size | Memory Saved vs Fixed 1MB | Syscalls (approx) |
|
||||
|-----------|-------------|--------------------------|-------------------|
|
||||
| 100 KB | 64 KB | 960 KB (94% reduction) | ~2 |
|
||||
| 10 MB | 256 KB | 768 KB (75% reduction) | ~40 |
|
||||
| 100 MB | 1 MB | 0 KB (same) | ~100 |
|
||||
| 10 GB | 1 MB | 0 KB (same) | ~10,240 |
|
||||
|
||||
### Future Improvements
|
||||
|
||||
1. **Connection Keep-Alive**: Ensure HTTP keep-alive is properly configured for consecutive uploads
|
||||
|
||||
2. **Rate Limiting**: Consider implementing upload rate limiting to prevent resource exhaustion
|
||||
|
||||
3. **Configurable Thresholds**: Make buffer size thresholds configurable via environment variables or config file
|
||||
|
||||
### Alternative Approaches Considered
|
||||
|
||||
1. **Increase s3s timeout**: Would only mask the problem, not fix the root cause
|
||||
2. **Retry logic**: Would increase complexity and potentially make things worse
|
||||
3. **Connection pooling**: Already handled by underlying HTTP stack
|
||||
4. **Upgrade tokio-util**: Would provide `StreamReader::with_capacity()` but requires testing entire dependency tree
|
||||
|
||||
## References
|
||||
|
||||
- Issue: "Uploading files of 10GB or 20GB consecutively may cause the upload to freeze"
|
||||
- Error: `AwsChunkedStreamError: Underlying: error reading a body from connection`
|
||||
- Library: `tokio_util::io::StreamReader`
|
||||
- Default buffer: 8KB (tokio_util default)
|
||||
- New buffer: 1MB (`DEFAULT_READ_BUFFER_SIZE`)
|
||||
|
||||
## Conclusion
|
||||
|
||||
This fix addresses the root cause of large file upload freezes by using an appropriately sized buffer for stream reading. The 1MB buffer significantly reduces system call overhead, improves throughput, and eliminates timeout issues during consecutive large file uploads.
|
||||
@@ -1,14 +1,12 @@
|
||||
# rustfs-helm
|
||||
|
||||
You can use this helm chart to deploy rustfs on k8s cluster. The chart supports standalone and distributed mode. For standalone mode, there is only one pod and one pvc; for distributed mode, there are two styles, 4 pods and 16 pvcs(each pod has 4 pvcs), 16 pods and 16 pvcs(each pod has 1 pvc). You should decide which mode and style suits for your situation. You can specify the parameters `mode` and `replicaCount` to install different mode and style.
|
||||
You can use this helm chart to deploy rustfs on k8s cluster.
|
||||
|
||||
## Parameters Overview
|
||||
|
||||
| parameter | description | default value |
|
||||
| -- | -- | -- |
|
||||
| replicaCount | Number of cluster nodes. | Default is `4`. |
|
||||
| mode.standalone.enabled | RustFS standalone mode support, namely one pod one pvc. | Default is `false` |
|
||||
| mode.distributed.enabled | RustFS distributed mode support, namely multiple pod multiple pvc. | Default is `true`. |
|
||||
| image.repository | docker image repository. | rustfs/rustfs. |
|
||||
| image.tag | the tag for rustfs docker image | "latest" |
|
||||
| secret.rustfs.access_key | RustFS Access Key ID | `rustfsadmin` |
|
||||
@@ -17,6 +15,7 @@ You can use this helm chart to deploy rustfs on k8s cluster. The chart supports
|
||||
| ingress.className | Specify the ingress class, traefik or nginx. | `nginx` |
|
||||
|
||||
|
||||
|
||||
**NOTE**: [`local-path`](https://github.com/rancher/local-path-provisioner) is used by k3s. If you want to use `local-path`, running the command,
|
||||
|
||||
```
|
||||
@@ -26,7 +25,7 @@ kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisione
|
||||
## Requirement
|
||||
|
||||
* Helm V3
|
||||
* RustFS >= 1.0.0-alpha.68
|
||||
* RustFS >= 1.0.0-alpha.66
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -44,8 +43,6 @@ helm install rustfs -n rustfs --create-namespace ./ --set ingress.className="ngi
|
||||
|
||||
> `traefik` or `nginx`, the different is the session sticky/affinity annotations.
|
||||
|
||||
**NOTE**: If you want to install standalone mode, specify the installation parameter `--set mode.standalone.enabled="true",mode.distributed.enabled="false"`; If you want to install distributed mode with 16 pods, specify the installation parameter `--set replicaCount="16"`.
|
||||
|
||||
Check the pod status
|
||||
|
||||
```
|
||||
@@ -62,12 +59,12 @@ Check the ingress status
|
||||
```
|
||||
kubectl -n rustfs get ing
|
||||
NAME CLASS HOSTS ADDRESS PORTS AGE
|
||||
rustfs nginx your.rustfs.com 10.43.237.152 80, 443 29m
|
||||
rustfs nginx xmg.rustfs.com 10.43.237.152 80, 443 29m
|
||||
```
|
||||
|
||||
Access the rustfs cluster via `https://your.rustfs.com` with the default username and password `rustfsadmin`.
|
||||
Access the rustfs cluster via `https://xmg.rustfs.com` with the default username and password `rustfsadmin`.
|
||||
|
||||
> Replace the `your.rustfs.com` with your own domain as well as the certificates.
|
||||
> Replace the `xmg.rustfs.com` with your own domain as well as the certificates.
|
||||
|
||||
## Uninstall
|
||||
|
||||
@@ -76,4 +73,3 @@ Uninstalling the rustfs installation with command,
|
||||
```
|
||||
helm uninstall rustfs -n rustfs
|
||||
```
|
||||
|
||||
|
||||
@@ -6,15 +6,12 @@ data:
|
||||
RUSTFS_ADDRESS: {{ .Values.config.rustfs.address | quote }}
|
||||
RUSTFS_CONSOLE_ADDRESS: {{ .Values.config.rustfs.console_address | quote }}
|
||||
RUSTFS_OBS_LOG_DIRECTORY: {{ .Values.config.rustfs.obs_log_directory | quote }}
|
||||
RUSTFS_SINKS_FILE_PATH: {{ .Values.config.rustfs.sinks_file_path | quote }}
|
||||
RUSTFS_CONSOLE_ENABLE: {{ .Values.config.rustfs.console_enable | quote }}
|
||||
RUSTFS_LOG_LEVEL: {{ .Values.config.rustfs.log_level | quote }}
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
{{- if eq (int .Values.replicaCount) 4 }}
|
||||
RUSTFS_VOLUMES: "http://rustfs-{0...3}.rustfs-headless.rustfs.svc.cluster.local:9000/data/rustfs{0...3}"
|
||||
{{- else if eq (int .Values.replicaCount) 16 }}
|
||||
RUSTFS_VOLUMES: "http://rustfs-{0...15}.rustfs-headless.rustfs.svc.cluster.local:9000/data"
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
RUSTFS_VOLUMES: "/data"
|
||||
{{- end }}
|
||||
RUSTFS_OBS_ENVIRONMENT: "develop"
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
{{- if .Values.mode.standalone.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "rustfs.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "rustfs.name" . }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "rustfs.name" . }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "rustfs.name" . }}
|
||||
spec:
|
||||
{{- if .Values.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
- name: init-step
|
||||
image: busybox
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
mkdir -p /data /logs
|
||||
chown -R 10001:10001 /data /logs
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: logs
|
||||
mountPath: /logs
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
command: ["/usr/bin/rustfs"]
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
{{- if .Values.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.containerSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.service.ep_port }}
|
||||
name: endpoint
|
||||
- containerPort: {{ .Values.service.console_port }}
|
||||
name: console
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "rustfs.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "rustfs.fullname" . }}-secret
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.requests.memory }}
|
||||
cpu: {{ .Values.resources.requests.cpu }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.limits.memory }}
|
||||
cpu: {{ .Values.resources.limits.cpu }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 9000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 9000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: /logs
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "rustfs.fullname" . }}-logs
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "rustfs.fullname" . }}-data
|
||||
{{- end }}
|
||||
@@ -1,24 +0,0 @@
|
||||
{{- if .Values.mode.standalone.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "rustfs.fullname" . }}-data
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: {{ .Values.storageclass.name }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.storageclass.size }}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "rustfs.fullname" . }}-logs
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: {{ .Values.storageclass.name }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.storageclass.size }}
|
||||
{{- end }}
|
||||
@@ -1,4 +1,3 @@
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
@@ -23,21 +22,18 @@ spec:
|
||||
name: console
|
||||
selector:
|
||||
app: {{ include "rustfs.name" . }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "rustfs.fullname" . }}-svc
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
{{- if eq .Values.ingress.className "traefik" }}
|
||||
{{- with .Values.ingress.traefikAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "rustfs.labels" . | nindent 4 }}
|
||||
spec:
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
@@ -15,17 +14,10 @@ spec:
|
||||
labels:
|
||||
app: {{ include "rustfs.name" . }}
|
||||
spec:
|
||||
{{- if .Values.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
- name: init-step
|
||||
image: busybox
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
env:
|
||||
- name: REPLICA_COUNT
|
||||
value: "{{ .Values.replicaCount }}"
|
||||
@@ -41,8 +33,8 @@ spec:
|
||||
mkdir -p /data
|
||||
fi
|
||||
|
||||
chown -R 10001:10001 /data
|
||||
chown -R 10001:10001 /logs
|
||||
chown -R 1000:1000 /data
|
||||
chown -R 1000:1000 /logs
|
||||
volumeMounts:
|
||||
{{- if eq (int .Values.replicaCount) 4 }}
|
||||
{{- range $i := until (int .Values.replicaCount) }}
|
||||
@@ -60,9 +52,9 @@ spec:
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
command: ["/usr/bin/rustfs"]
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
{{- if .Values.containerSecurityContext }}
|
||||
{{- if .Values.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.containerSecurityContext | nindent 12 }}
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.service.ep_port }}
|
||||
@@ -97,6 +89,7 @@ spec:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 9000
|
||||
exec:
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
@@ -114,17 +107,12 @@ spec:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: {{ $.Values.storageclass.name }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ $.Values.storageclass.size}}
|
||||
{{- if eq (int .Values.replicaCount) 4 }}
|
||||
{{- range $i := until (int .Values.replicaCount) }}
|
||||
{{- if eq (int .Values.replicaCount) 4 }}
|
||||
{{- range $i := until (int .Values.replicaCount) }}
|
||||
- metadata:
|
||||
name: data-rustfs-{{ $i }}
|
||||
spec:
|
||||
@@ -133,8 +121,8 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ $.Values.storageclass.size}}
|
||||
{{- end }}
|
||||
{{- else if eq (int .Values.replicaCount) 16 }}
|
||||
{{- end }}
|
||||
{{- else if eq (int .Values.replicaCount) 16 }}
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
@@ -143,5 +131,4 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ $.Values.storageclass.size}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
Input your crt content.
|
||||
Please input your cert file content.
|
||||
-----END CERTIFICATE-----
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
Input your private key.
|
||||
Please input your key file content
|
||||
-----END PRIVATE KEY-----
|
||||
|
||||
@@ -9,9 +9,9 @@ replicaCount: 4
|
||||
image:
|
||||
repository: rustfs/rustfs
|
||||
# This sets the pull policy for images.
|
||||
pullPolicy: IfNotPresent
|
||||
pullPolicy: Always
|
||||
# Overrides the image tag whose default is the chart appVersion.
|
||||
tag: "latest"
|
||||
tag: "1.0.0-alpha.66"
|
||||
|
||||
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||
imagePullSecrets: []
|
||||
@@ -19,13 +19,6 @@ imagePullSecrets: []
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
|
||||
mode:
|
||||
standalone:
|
||||
enabled: false
|
||||
distributed:
|
||||
enabled: true
|
||||
|
||||
secret:
|
||||
rustfs:
|
||||
access_key: rustfsadmin
|
||||
@@ -39,6 +32,7 @@ config:
|
||||
log_level: "debug"
|
||||
rust_log: "debug"
|
||||
console_enable: "true"
|
||||
sinks_file_path: "/logs"
|
||||
obs_log_directory: "/logs"
|
||||
|
||||
# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
|
||||
@@ -61,16 +55,16 @@ podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
podSecurityContext:
|
||||
fsGroup: 10001
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
{}
|
||||
# fsGroup: 2000
|
||||
|
||||
containerSecurityContext:
|
||||
securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
|
||||
service:
|
||||
type: NodePort
|
||||
@@ -94,7 +88,7 @@ ingress:
|
||||
nginx.ingress.kubernetes.io/session-cookie-max-age: "3600"
|
||||
nginx.ingress.kubernetes.io/session-cookie-name: rustfs
|
||||
hosts:
|
||||
- host: your.rustfs.com
|
||||
- host: xmg.rustfs.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: ImplementationSpecific
|
||||
|
||||
@@ -31,8 +31,7 @@ name = "rustfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = ["metrics"]
|
||||
metrics = []
|
||||
default = []
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -74,7 +73,7 @@ http.workspace = true
|
||||
http-body.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
socket2 = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process"] }
|
||||
tokio-rustls = { workspace = true }
|
||||
tokio-stream.workspace = true
|
||||
tokio-util.workspace = true
|
||||
|
||||
@@ -19,65 +19,14 @@ use crate::admin::auth::validate_admin_request;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use hyper::StatusCode;
|
||||
use matchit::Params;
|
||||
use rustfs_ecstore::config::com::{read_config, save_config};
|
||||
use rustfs_ecstore::new_object_layer_fn;
|
||||
use rustfs_kms::{
|
||||
ConfigureKmsRequest, ConfigureKmsResponse, KmsConfig, KmsConfigSummary, KmsServiceStatus, KmsStatusResponse, StartKmsRequest,
|
||||
ConfigureKmsRequest, ConfigureKmsResponse, KmsConfigSummary, KmsServiceStatus, KmsStatusResponse, StartKmsRequest,
|
||||
StartKmsResponse, StopKmsResponse, get_global_kms_service_manager,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// Path to store KMS configuration in the cluster metadata
|
||||
const KMS_CONFIG_PATH: &str = "config/kms_config.json";
|
||||
|
||||
/// Save KMS configuration to cluster storage
|
||||
async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err("Storage layer not initialized".to_string());
|
||||
};
|
||||
|
||||
let data = serde_json::to_vec(config).map_err(|e| format!("Failed to serialize KMS config: {e}"))?;
|
||||
|
||||
save_config(store, KMS_CONFIG_PATH, data)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to save KMS config to storage: {e}"))?;
|
||||
|
||||
info!("KMS configuration persisted to cluster storage at {}", KMS_CONFIG_PATH);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load KMS configuration from cluster storage
|
||||
pub async fn load_kms_config() -> Option<KmsConfig> {
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
warn!("Storage layer not initialized, cannot load KMS config");
|
||||
return None;
|
||||
};
|
||||
|
||||
match read_config(store, KMS_CONFIG_PATH).await {
|
||||
Ok(data) => match serde_json::from_slice::<KmsConfig>(&data) {
|
||||
Ok(config) => {
|
||||
info!("Loaded KMS configuration from cluster storage");
|
||||
Some(config)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to deserialize KMS config: {}", e);
|
||||
None
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
// Config not found is normal on first run
|
||||
if e.to_string().contains("ConfigNotFound") || e.to_string().contains("not found") {
|
||||
info!("No persisted KMS configuration found (first run or not configured yet)");
|
||||
} else {
|
||||
warn!("Failed to load KMS config from storage: {}", e);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Configure KMS service handler
|
||||
pub struct ConfigureKmsHandler;
|
||||
|
||||
@@ -133,19 +82,11 @@ impl Operation for ConfigureKmsHandler {
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Configure the service
|
||||
let (success, message, status) = match service_manager.configure(kms_config.clone()).await {
|
||||
let (success, message, status) = match service_manager.configure(kms_config).await {
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS configured in memory but failed to persist: {e}");
|
||||
error!("{}", error_msg);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!("KMS configured successfully and persisted with status: {:?}", status);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!("KMS configured successfully with status: {:?}", status);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to configure KMS: {e}");
|
||||
@@ -500,19 +441,11 @@ impl Operation for ReconfigureKmsHandler {
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Reconfigure the service (stops, reconfigures, and starts)
|
||||
let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await {
|
||||
let (success, message, status) = match service_manager.reconfigure(kms_config).await {
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}");
|
||||
error!("{}", error_msg);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!("KMS reconfigured successfully and persisted with status: {:?}", status);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!("KMS reconfigured successfully with status: {:?}", status);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
|
||||
@@ -17,8 +17,6 @@ use const_str::concat;
|
||||
use std::string::ToString;
|
||||
shadow_rs::shadow!(build);
|
||||
|
||||
pub mod workload_profiles;
|
||||
|
||||
#[cfg(test)]
|
||||
mod config_test;
|
||||
|
||||
@@ -114,16 +112,6 @@ pub struct Opt {
|
||||
/// Default KMS key ID for encryption
|
||||
#[arg(long, env = "RUSTFS_KMS_DEFAULT_KEY_ID")]
|
||||
pub kms_default_key_id: Option<String>,
|
||||
|
||||
/// Disable adaptive buffer sizing with workload profiles
|
||||
/// Set this flag to use legacy fixed-size buffer behavior from PR #869
|
||||
#[arg(long, default_value_t = false, env = "RUSTFS_BUFFER_PROFILE_DISABLE")]
|
||||
pub buffer_profile_disable: bool,
|
||||
|
||||
/// Workload profile for adaptive buffer sizing
|
||||
/// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage
|
||||
#[arg(long, default_value_t = String::from("GeneralPurpose"), env = "RUSTFS_BUFFER_PROFILE")]
|
||||
pub buffer_profile: String,
|
||||
}
|
||||
|
||||
// lazy_static::lazy_static! {
|
||||
|
||||
@@ -1,632 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
//! Adaptive buffer sizing optimization for different workload types.
|
||||
//!
|
||||
//! This module provides intelligent buffer size selection based on file size and workload profile
|
||||
//! to achieve optimal balance between performance, memory usage, and security.
|
||||
|
||||
use rustfs_config::{KI_B, MI_B};
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Global buffer configuration that can be set at application startup
|
||||
static GLOBAL_BUFFER_CONFIG: OnceLock<RustFSBufferConfig> = OnceLock::new();
|
||||
|
||||
/// Global flag indicating whether buffer profiles are enabled
|
||||
static BUFFER_PROFILE_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Enable or disable buffer profiling globally
|
||||
///
|
||||
/// This controls whether the opt-in buffer profiling feature is active.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `enabled` - Whether to enable buffer profiling
|
||||
pub fn set_buffer_profile_enabled(enabled: bool) {
|
||||
BUFFER_PROFILE_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Check if buffer profiling is enabled globally
|
||||
pub fn is_buffer_profile_enabled() -> bool {
|
||||
BUFFER_PROFILE_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Initialize the global buffer configuration
|
||||
///
|
||||
/// This should be called once at application startup with the desired profile.
|
||||
/// If not called, the default GeneralPurpose profile will be used.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - The buffer configuration to use globally
|
||||
///
|
||||
/// # Examples
|
||||
/// ```ignore
|
||||
/// use rustfs::config::workload_profiles::{RustFSBufferConfig, WorkloadProfile};
|
||||
///
|
||||
/// // Initialize with AiTraining profile
|
||||
/// init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::AiTraining));
|
||||
/// ```
|
||||
pub fn init_global_buffer_config(config: RustFSBufferConfig) {
|
||||
let _ = GLOBAL_BUFFER_CONFIG.set(config);
|
||||
}
|
||||
|
||||
/// Get the global buffer configuration
|
||||
///
|
||||
/// Returns the configured profile, or GeneralPurpose if not initialized.
|
||||
pub fn get_global_buffer_config() -> &'static RustFSBufferConfig {
|
||||
GLOBAL_BUFFER_CONFIG.get_or_init(RustFSBufferConfig::default)
|
||||
}
|
||||
|
||||
/// Workload profile types that define buffer sizing strategies
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum WorkloadProfile {
|
||||
/// General purpose - default configuration with balanced performance and memory
|
||||
GeneralPurpose,
|
||||
/// AI/ML training: optimized for large sequential reads with maximum throughput
|
||||
AiTraining,
|
||||
/// Data analytics: mixed read-write patterns with moderate buffer sizes
|
||||
DataAnalytics,
|
||||
/// Web workloads: small file intensive with minimal memory overhead
|
||||
WebWorkload,
|
||||
/// Industrial IoT: real-time streaming with low latency priority
|
||||
IndustrialIoT,
|
||||
/// Secure storage: security first, memory constrained for compliance
|
||||
SecureStorage,
|
||||
/// Custom configuration for specialized requirements
|
||||
Custom(BufferConfig),
|
||||
}
|
||||
|
||||
/// Buffer size configuration for adaptive buffering
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct BufferConfig {
|
||||
/// Minimum buffer size in bytes (for very small files or memory-constrained environments)
|
||||
pub min_size: usize,
|
||||
/// Maximum buffer size in bytes (cap for large files to prevent excessive memory usage)
|
||||
pub max_size: usize,
|
||||
/// Default size for unknown file size scenarios (streaming/chunked uploads)
|
||||
pub default_unknown: usize,
|
||||
/// File size thresholds and corresponding buffer sizes: (file_size_threshold, buffer_size)
|
||||
/// Thresholds should be in ascending order
|
||||
pub thresholds: Vec<(i64, usize)>,
|
||||
}
|
||||
|
||||
/// Complete buffer configuration for RustFS
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RustFSBufferConfig {
|
||||
/// Selected workload profile
|
||||
pub workload: WorkloadProfile,
|
||||
/// Computed buffer configuration (either from profile or custom)
|
||||
pub base_config: BufferConfig,
|
||||
}
|
||||
|
||||
impl WorkloadProfile {
|
||||
/// Parse a workload profile from a string name
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `name` - The name of the profile (case-insensitive)
|
||||
///
|
||||
/// # Returns
|
||||
/// The corresponding WorkloadProfile, or GeneralPurpose if name is not recognized
|
||||
///
|
||||
/// # Examples
|
||||
/// ```
|
||||
/// use rustfs::config::workload_profiles::WorkloadProfile;
|
||||
///
|
||||
/// let profile = WorkloadProfile::from_name("AiTraining");
|
||||
/// let profile2 = WorkloadProfile::from_name("aitraining"); // case-insensitive
|
||||
/// let profile3 = WorkloadProfile::from_name("unknown"); // defaults to GeneralPurpose
|
||||
/// ```
|
||||
pub fn from_name(name: &str) -> Self {
|
||||
match name.to_lowercase().as_str() {
|
||||
"generalpurpose" | "general" => WorkloadProfile::GeneralPurpose,
|
||||
"aitraining" | "ai" => WorkloadProfile::AiTraining,
|
||||
"dataanalytics" | "analytics" => WorkloadProfile::DataAnalytics,
|
||||
"webworkload" | "web" => WorkloadProfile::WebWorkload,
|
||||
"industrialiot" | "iot" => WorkloadProfile::IndustrialIoT,
|
||||
"securestorage" | "secure" => WorkloadProfile::SecureStorage,
|
||||
_ => {
|
||||
// Default to GeneralPurpose for unknown profiles
|
||||
WorkloadProfile::GeneralPurpose
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the buffer configuration for this workload profile
|
||||
pub fn config(&self) -> BufferConfig {
|
||||
match self {
|
||||
WorkloadProfile::GeneralPurpose => Self::general_purpose_config(),
|
||||
WorkloadProfile::AiTraining => Self::ai_training_config(),
|
||||
WorkloadProfile::DataAnalytics => Self::data_analytics_config(),
|
||||
WorkloadProfile::WebWorkload => Self::web_workload_config(),
|
||||
WorkloadProfile::IndustrialIoT => Self::industrial_iot_config(),
|
||||
WorkloadProfile::SecureStorage => Self::secure_storage_config(),
|
||||
WorkloadProfile::Custom(config) => config.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// General purpose configuration: balanced performance and memory usage
|
||||
/// - Small files (< 1MB): 64KB buffer
|
||||
/// - Medium files (1MB-100MB): 256KB buffer
|
||||
/// - Large files (>= 100MB): 1MB buffer
|
||||
fn general_purpose_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 64 * KI_B,
|
||||
max_size: MI_B,
|
||||
default_unknown: MI_B,
|
||||
thresholds: vec![
|
||||
(MI_B as i64, 64 * KI_B), // < 1MB: 64KB
|
||||
(100 * MI_B as i64, 256 * KI_B), // 1MB-100MB: 256KB
|
||||
(i64::MAX, MI_B), // >= 100MB: 1MB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// AI/ML training configuration: optimized for large sequential reads
|
||||
/// - Small files (< 10MB): 512KB buffer
|
||||
/// - Medium files (10MB-500MB): 2MB buffer
|
||||
/// - Large files (>= 500MB): 4MB buffer for maximum throughput
|
||||
fn ai_training_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 512 * KI_B,
|
||||
max_size: 4 * MI_B,
|
||||
default_unknown: 2 * MI_B,
|
||||
thresholds: vec![
|
||||
(10 * MI_B as i64, 512 * KI_B), // < 10MB: 512KB
|
||||
(500 * MI_B as i64, 2 * MI_B), // 10MB-500MB: 2MB
|
||||
(i64::MAX, 4 * MI_B), // >= 500MB: 4MB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Data analytics configuration: mixed read-write patterns
|
||||
/// - Small files (< 5MB): 128KB buffer
|
||||
/// - Medium files (5MB-200MB): 512KB buffer
|
||||
/// - Large files (>= 200MB): 2MB buffer
|
||||
fn data_analytics_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 128 * KI_B,
|
||||
max_size: 2 * MI_B,
|
||||
default_unknown: 512 * KI_B,
|
||||
thresholds: vec![
|
||||
(5 * MI_B as i64, 128 * KI_B), // < 5MB: 128KB
|
||||
(200 * MI_B as i64, 512 * KI_B), // 5MB-200MB: 512KB
|
||||
(i64::MAX, 2 * MI_B), // >= 200MB: 2MB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Web workload configuration: small file intensive
|
||||
/// - Small files (< 512KB): 32KB buffer to minimize memory
|
||||
/// - Medium files (512KB-10MB): 128KB buffer
|
||||
/// - Large files (>= 10MB): 256KB buffer (rare for web assets)
|
||||
fn web_workload_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: 256 * KI_B,
|
||||
default_unknown: 128 * KI_B,
|
||||
thresholds: vec![
|
||||
(512 * KI_B as i64, 32 * KI_B), // < 512KB: 32KB
|
||||
(10 * MI_B as i64, 128 * KI_B), // 512KB-10MB: 128KB
|
||||
(i64::MAX, 256 * KI_B), // >= 10MB: 256KB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Industrial IoT configuration: real-time streaming with low latency
|
||||
/// - Small files (< 1MB): 64KB buffer for quick processing
|
||||
/// - Medium files (1MB-50MB): 256KB buffer
|
||||
/// - Large files (>= 50MB): 512KB buffer (cap for memory constraints)
|
||||
fn industrial_iot_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 64 * KI_B,
|
||||
max_size: 512 * KI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![
|
||||
(MI_B as i64, 64 * KI_B), // < 1MB: 64KB
|
||||
(50 * MI_B as i64, 256 * KI_B), // 1MB-50MB: 256KB
|
||||
(i64::MAX, 512 * KI_B), // >= 50MB: 512KB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Secure storage configuration: security first, memory constrained
|
||||
/// - Small files (< 1MB): 32KB buffer (minimal memory footprint)
|
||||
/// - Medium files (1MB-50MB): 128KB buffer
|
||||
/// - Large files (>= 50MB): 256KB buffer (strict memory limit for compliance)
|
||||
fn secure_storage_config() -> BufferConfig {
|
||||
BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: 256 * KI_B,
|
||||
default_unknown: 128 * KI_B,
|
||||
thresholds: vec![
|
||||
(MI_B as i64, 32 * KI_B), // < 1MB: 32KB
|
||||
(50 * MI_B as i64, 128 * KI_B), // 1MB-50MB: 128KB
|
||||
(i64::MAX, 256 * KI_B), // >= 50MB: 256KB
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect special OS environment and return appropriate workload profile
|
||||
/// Supports Chinese secure operating systems (Kylin, NeoKylin, Unity OS, etc.)
|
||||
pub fn detect_os_environment() -> Option<WorkloadProfile> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// Read /etc/os-release to detect Chinese secure OS distributions
|
||||
if let Ok(content) = std::fs::read_to_string("/etc/os-release") {
|
||||
let content_lower = content.to_lowercase();
|
||||
// Check for Chinese secure OS distributions
|
||||
if content_lower.contains("kylin")
|
||||
|| content_lower.contains("neokylin")
|
||||
|| content_lower.contains("uos")
|
||||
|| content_lower.contains("unity")
|
||||
|| content_lower.contains("openkylin")
|
||||
{
|
||||
// Use SecureStorage profile for Chinese secure OS environments
|
||||
return Some(WorkloadProfile::SecureStorage);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl BufferConfig {
|
||||
/// Calculate the optimal buffer size for a given file size
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `file_size` - The size of the file in bytes, or -1 if unknown
|
||||
///
|
||||
/// # Returns
|
||||
/// Optimal buffer size in bytes based on the configuration
|
||||
pub fn calculate_buffer_size(&self, file_size: i64) -> usize {
|
||||
// Handle unknown or negative file sizes
|
||||
if file_size < 0 {
|
||||
return self.default_unknown.clamp(self.min_size, self.max_size);
|
||||
}
|
||||
|
||||
// Find the appropriate buffer size from thresholds
|
||||
for (threshold, buffer_size) in &self.thresholds {
|
||||
if file_size < *threshold {
|
||||
return (*buffer_size).clamp(self.min_size, self.max_size);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to max_size if no threshold matched (shouldn't happen with i64::MAX threshold)
|
||||
self.max_size
|
||||
}
|
||||
|
||||
/// Validate the buffer configuration
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.min_size == 0 {
|
||||
return Err("min_size must be greater than 0".to_string());
|
||||
}
|
||||
if self.max_size < self.min_size {
|
||||
return Err("max_size must be >= min_size".to_string());
|
||||
}
|
||||
if self.default_unknown < self.min_size || self.default_unknown > self.max_size {
|
||||
return Err("default_unknown must be between min_size and max_size".to_string());
|
||||
}
|
||||
if self.thresholds.is_empty() {
|
||||
return Err("thresholds cannot be empty".to_string());
|
||||
}
|
||||
|
||||
// Validate thresholds are in ascending order
|
||||
let mut prev_threshold = -1i64;
|
||||
for (threshold, buffer_size) in &self.thresholds {
|
||||
if *threshold <= prev_threshold {
|
||||
return Err("thresholds must be in ascending order".to_string());
|
||||
}
|
||||
if *buffer_size < self.min_size || *buffer_size > self.max_size {
|
||||
return Err(format!(
|
||||
"buffer_size {} must be between min_size {} and max_size {}",
|
||||
buffer_size, self.min_size, self.max_size
|
||||
));
|
||||
}
|
||||
prev_threshold = *threshold;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RustFSBufferConfig {
|
||||
/// Create a new buffer configuration with the given workload profile
|
||||
pub fn new(workload: WorkloadProfile) -> Self {
|
||||
let base_config = workload.config();
|
||||
Self { workload, base_config }
|
||||
}
|
||||
|
||||
/// Create a configuration with auto-detected OS environment
|
||||
/// Falls back to GeneralPurpose if no special environment detected
|
||||
pub fn with_auto_detect() -> Self {
|
||||
let workload = WorkloadProfile::detect_os_environment().unwrap_or(WorkloadProfile::GeneralPurpose);
|
||||
Self::new(workload)
|
||||
}
|
||||
|
||||
/// Get the buffer size for a given file size
|
||||
pub fn get_buffer_size(&self, file_size: i64) -> usize {
|
||||
self.base_config.calculate_buffer_size(file_size)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RustFSBufferConfig {
|
||||
fn default() -> Self {
|
||||
Self::new(WorkloadProfile::GeneralPurpose)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_general_purpose_config() {
|
||||
let config = WorkloadProfile::GeneralPurpose.config();
|
||||
|
||||
// Test small files (< 1MB) - should use 64KB
|
||||
assert_eq!(config.calculate_buffer_size(0), 64 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size(512 * KI_B as i64), 64 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((MI_B - 1) as i64), 64 * KI_B);
|
||||
|
||||
// Test medium files (1MB - 100MB) - should use 256KB
|
||||
assert_eq!(config.calculate_buffer_size(MI_B as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((50 * MI_B) as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B - 1) as i64), 256 * KI_B);
|
||||
|
||||
// Test large files (>= 100MB) - should use 1MB
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B) as i64), MI_B);
|
||||
assert_eq!(config.calculate_buffer_size((500 * MI_B) as i64), MI_B);
|
||||
assert_eq!(config.calculate_buffer_size((10 * 1024 * MI_B) as i64), MI_B);
|
||||
|
||||
// Test unknown size
|
||||
assert_eq!(config.calculate_buffer_size(-1), MI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ai_training_config() {
|
||||
let config = WorkloadProfile::AiTraining.config();
|
||||
|
||||
// Test small files
|
||||
assert_eq!(config.calculate_buffer_size((5 * MI_B) as i64), 512 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((10 * MI_B - 1) as i64), 512 * KI_B);
|
||||
|
||||
// Test medium files
|
||||
assert_eq!(config.calculate_buffer_size((10 * MI_B) as i64), 2 * MI_B);
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B) as i64), 2 * MI_B);
|
||||
assert_eq!(config.calculate_buffer_size((500 * MI_B - 1) as i64), 2 * MI_B);
|
||||
|
||||
// Test large files
|
||||
assert_eq!(config.calculate_buffer_size((500 * MI_B) as i64), 4 * MI_B);
|
||||
assert_eq!(config.calculate_buffer_size((1024 * MI_B) as i64), 4 * MI_B);
|
||||
|
||||
// Test unknown size
|
||||
assert_eq!(config.calculate_buffer_size(-1), 2 * MI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_web_workload_config() {
|
||||
let config = WorkloadProfile::WebWorkload.config();
|
||||
|
||||
// Test small files
|
||||
assert_eq!(config.calculate_buffer_size((100 * KI_B) as i64), 32 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((512 * KI_B - 1) as i64), 32 * KI_B);
|
||||
|
||||
// Test medium files
|
||||
assert_eq!(config.calculate_buffer_size((512 * KI_B) as i64), 128 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((5 * MI_B) as i64), 128 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((10 * MI_B - 1) as i64), 128 * KI_B);
|
||||
|
||||
// Test large files
|
||||
assert_eq!(config.calculate_buffer_size((10 * MI_B) as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((50 * MI_B) as i64), 256 * KI_B);
|
||||
|
||||
// Test unknown size
|
||||
assert_eq!(config.calculate_buffer_size(-1), 128 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_secure_storage_config() {
|
||||
let config = WorkloadProfile::SecureStorage.config();
|
||||
|
||||
// Test small files
|
||||
assert_eq!(config.calculate_buffer_size((500 * KI_B) as i64), 32 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((MI_B - 1) as i64), 32 * KI_B);
|
||||
|
||||
// Test medium files
|
||||
assert_eq!(config.calculate_buffer_size(MI_B as i64), 128 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((25 * MI_B) as i64), 128 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((50 * MI_B - 1) as i64), 128 * KI_B);
|
||||
|
||||
// Test large files
|
||||
assert_eq!(config.calculate_buffer_size((50 * MI_B) as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B) as i64), 256 * KI_B);
|
||||
|
||||
// Test unknown size
|
||||
assert_eq!(config.calculate_buffer_size(-1), 128 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_industrial_iot_config() {
|
||||
let config = WorkloadProfile::IndustrialIoT.config();
|
||||
|
||||
// Test configuration
|
||||
assert_eq!(config.calculate_buffer_size((500 * KI_B) as i64), 64 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((25 * MI_B) as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B) as i64), 512 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size(-1), 256 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_analytics_config() {
|
||||
let config = WorkloadProfile::DataAnalytics.config();
|
||||
|
||||
// Test configuration
|
||||
assert_eq!(config.calculate_buffer_size((2 * MI_B) as i64), 128 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((100 * MI_B) as i64), 512 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size((500 * MI_B) as i64), 2 * MI_B);
|
||||
assert_eq!(config.calculate_buffer_size(-1), 512 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_config() {
|
||||
let custom_config = BufferConfig {
|
||||
min_size: 16 * KI_B,
|
||||
max_size: 512 * KI_B,
|
||||
default_unknown: 128 * KI_B,
|
||||
thresholds: vec![(MI_B as i64, 64 * KI_B), (i64::MAX, 256 * KI_B)],
|
||||
};
|
||||
|
||||
let profile = WorkloadProfile::Custom(custom_config.clone());
|
||||
let config = profile.config();
|
||||
|
||||
assert_eq!(config.calculate_buffer_size(512 * KI_B as i64), 64 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size(2 * MI_B as i64), 256 * KI_B);
|
||||
assert_eq!(config.calculate_buffer_size(-1), 128 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_buffer_config_validation() {
|
||||
// Valid configuration
|
||||
let valid_config = BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: MI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![(MI_B as i64, 128 * KI_B), (i64::MAX, 512 * KI_B)],
|
||||
};
|
||||
assert!(valid_config.validate().is_ok());
|
||||
|
||||
// Invalid: min_size is 0
|
||||
let invalid_config = BufferConfig {
|
||||
min_size: 0,
|
||||
max_size: MI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![(MI_B as i64, 128 * KI_B)],
|
||||
};
|
||||
assert!(invalid_config.validate().is_err());
|
||||
|
||||
// Invalid: max_size < min_size
|
||||
let invalid_config = BufferConfig {
|
||||
min_size: MI_B,
|
||||
max_size: 32 * KI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![(MI_B as i64, 128 * KI_B)],
|
||||
};
|
||||
assert!(invalid_config.validate().is_err());
|
||||
|
||||
// Invalid: default_unknown out of range
|
||||
let invalid_config = BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: 256 * KI_B,
|
||||
default_unknown: MI_B,
|
||||
thresholds: vec![(MI_B as i64, 128 * KI_B)],
|
||||
};
|
||||
assert!(invalid_config.validate().is_err());
|
||||
|
||||
// Invalid: empty thresholds
|
||||
let invalid_config = BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: MI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![],
|
||||
};
|
||||
assert!(invalid_config.validate().is_err());
|
||||
|
||||
// Invalid: thresholds not in ascending order
|
||||
let invalid_config = BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: MI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![(100 * MI_B as i64, 512 * KI_B), (MI_B as i64, 128 * KI_B)],
|
||||
};
|
||||
assert!(invalid_config.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rustfs_buffer_config() {
|
||||
let config = RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(config.get_buffer_size(500 * KI_B as i64), 64 * KI_B);
|
||||
assert_eq!(config.get_buffer_size(50 * MI_B as i64), 256 * KI_B);
|
||||
assert_eq!(config.get_buffer_size(200 * MI_B as i64), MI_B);
|
||||
|
||||
let default_config = RustFSBufferConfig::default();
|
||||
assert_eq!(default_config.get_buffer_size(500 * KI_B as i64), 64 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_workload_profile_equality() {
|
||||
assert_eq!(WorkloadProfile::GeneralPurpose, WorkloadProfile::GeneralPurpose);
|
||||
assert_ne!(WorkloadProfile::GeneralPurpose, WorkloadProfile::AiTraining);
|
||||
|
||||
let custom1 = BufferConfig {
|
||||
min_size: 32 * KI_B,
|
||||
max_size: MI_B,
|
||||
default_unknown: 256 * KI_B,
|
||||
thresholds: vec![(MI_B as i64, 128 * KI_B)],
|
||||
};
|
||||
let custom2 = custom1.clone();
|
||||
|
||||
assert_eq!(WorkloadProfile::Custom(custom1.clone()), WorkloadProfile::Custom(custom2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_workload_profile_from_name() {
|
||||
// Test exact matches (case-insensitive)
|
||||
assert_eq!(WorkloadProfile::from_name("GeneralPurpose"), WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(WorkloadProfile::from_name("generalpurpose"), WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(WorkloadProfile::from_name("GENERALPURPOSE"), WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(WorkloadProfile::from_name("general"), WorkloadProfile::GeneralPurpose);
|
||||
|
||||
assert_eq!(WorkloadProfile::from_name("AiTraining"), WorkloadProfile::AiTraining);
|
||||
assert_eq!(WorkloadProfile::from_name("aitraining"), WorkloadProfile::AiTraining);
|
||||
assert_eq!(WorkloadProfile::from_name("ai"), WorkloadProfile::AiTraining);
|
||||
|
||||
assert_eq!(WorkloadProfile::from_name("DataAnalytics"), WorkloadProfile::DataAnalytics);
|
||||
assert_eq!(WorkloadProfile::from_name("dataanalytics"), WorkloadProfile::DataAnalytics);
|
||||
assert_eq!(WorkloadProfile::from_name("analytics"), WorkloadProfile::DataAnalytics);
|
||||
|
||||
assert_eq!(WorkloadProfile::from_name("WebWorkload"), WorkloadProfile::WebWorkload);
|
||||
assert_eq!(WorkloadProfile::from_name("webworkload"), WorkloadProfile::WebWorkload);
|
||||
assert_eq!(WorkloadProfile::from_name("web"), WorkloadProfile::WebWorkload);
|
||||
|
||||
assert_eq!(WorkloadProfile::from_name("IndustrialIoT"), WorkloadProfile::IndustrialIoT);
|
||||
assert_eq!(WorkloadProfile::from_name("industrialiot"), WorkloadProfile::IndustrialIoT);
|
||||
assert_eq!(WorkloadProfile::from_name("iot"), WorkloadProfile::IndustrialIoT);
|
||||
|
||||
assert_eq!(WorkloadProfile::from_name("SecureStorage"), WorkloadProfile::SecureStorage);
|
||||
assert_eq!(WorkloadProfile::from_name("securestorage"), WorkloadProfile::SecureStorage);
|
||||
assert_eq!(WorkloadProfile::from_name("secure"), WorkloadProfile::SecureStorage);
|
||||
|
||||
// Test unknown name defaults to GeneralPurpose
|
||||
assert_eq!(WorkloadProfile::from_name("unknown"), WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(WorkloadProfile::from_name("invalid"), WorkloadProfile::GeneralPurpose);
|
||||
assert_eq!(WorkloadProfile::from_name(""), WorkloadProfile::GeneralPurpose);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_buffer_config() {
|
||||
use super::{is_buffer_profile_enabled, set_buffer_profile_enabled};
|
||||
|
||||
// Test enable/disable
|
||||
set_buffer_profile_enabled(true);
|
||||
assert!(is_buffer_profile_enabled());
|
||||
|
||||
set_buffer_profile_enabled(false);
|
||||
assert!(!is_buffer_profile_enabled());
|
||||
|
||||
// Reset for other tests
|
||||
set_buffer_profile_enabled(false);
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ async fn async_main() -> Result<()> {
|
||||
let guard = match init_obs(Some(opt.clone().obs_endpoint)).await {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
println!("Failed to initialize observability: {e}");
|
||||
println!("Failed to initialize observability: {}", e);
|
||||
return Err(Error::other(e));
|
||||
}
|
||||
};
|
||||
@@ -256,9 +256,6 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
// Initialize KMS system if enabled
|
||||
init_kms_system(&opt).await?;
|
||||
|
||||
// Initialize buffer profiling system
|
||||
init_buffer_profile_system(&opt);
|
||||
|
||||
// Initialize event notifier
|
||||
init_event_notifier().await;
|
||||
// Start the audit system
|
||||
@@ -553,7 +550,7 @@ async fn init_kms_system(opt: &config::Opt) -> Result<()> {
|
||||
|
||||
// If KMS is enabled in configuration, configure and start the service
|
||||
if opt.kms_enable {
|
||||
info!("KMS is enabled via command line, configuring and starting service...");
|
||||
info!("KMS is enabled, configuring and starting service...");
|
||||
|
||||
// Create KMS configuration from command line options
|
||||
let kms_config = match opt.kms_backend.as_str() {
|
||||
@@ -622,77 +619,10 @@ async fn init_kms_system(opt: &config::Opt) -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| Error::other(format!("Failed to start KMS: {e}")))?;
|
||||
|
||||
info!("KMS service configured and started successfully from command line options");
|
||||
info!("KMS service configured and started successfully");
|
||||
} else {
|
||||
// Try to load persisted KMS configuration from cluster storage
|
||||
info!("Attempting to load persisted KMS configuration from cluster storage...");
|
||||
|
||||
if let Some(persisted_config) = admin::handlers::kms_dynamic::load_kms_config().await {
|
||||
info!("Found persisted KMS configuration, attempting to configure and start service...");
|
||||
|
||||
// Configure the KMS service with persisted config
|
||||
match service_manager.configure(persisted_config).await {
|
||||
Ok(()) => {
|
||||
// Start the KMS service
|
||||
match service_manager.start().await {
|
||||
Ok(()) => {
|
||||
info!("KMS service configured and started successfully from persisted configuration");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to start KMS with persisted configuration: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to configure KMS with persisted configuration: {}", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!("No persisted KMS configuration found. KMS is ready for dynamic configuration via API.");
|
||||
}
|
||||
info!("KMS service manager initialized. KMS is ready for dynamic configuration via API.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Initialize the adaptive buffer sizing system with workload profile configuration.
|
||||
///
|
||||
/// This system provides intelligent buffer size selection based on file size and workload type.
|
||||
/// Workload-aware buffer sizing is enabled by default with the GeneralPurpose profile,
|
||||
/// which provides the same buffer sizes as the original implementation for compatibility.
|
||||
///
|
||||
/// # Configuration
|
||||
/// - Default: Enabled with GeneralPurpose profile
|
||||
/// - Opt-out: Use `--buffer-profile-disable` flag
|
||||
/// - Custom profile: Set via `--buffer-profile` or `RUSTFS_BUFFER_PROFILE` environment variable
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `opt` - The application configuration options
|
||||
fn init_buffer_profile_system(opt: &config::Opt) {
|
||||
use crate::config::workload_profiles::{
|
||||
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
|
||||
};
|
||||
|
||||
if opt.buffer_profile_disable {
|
||||
// User explicitly disabled buffer profiling - use GeneralPurpose profile in disabled mode
|
||||
info!("Buffer profiling disabled via --buffer-profile-disable, using GeneralPurpose profile");
|
||||
set_buffer_profile_enabled(false);
|
||||
} else {
|
||||
// Enabled by default: use configured workload profile
|
||||
info!("Buffer profiling enabled with profile: {}", opt.buffer_profile);
|
||||
|
||||
// Parse the workload profile from configuration string
|
||||
let profile = WorkloadProfile::from_name(&opt.buffer_profile);
|
||||
|
||||
// Log the selected profile for operational visibility
|
||||
info!("Active buffer profile: {:?}", profile);
|
||||
|
||||
// Initialize the global buffer configuration
|
||||
init_global_buffer_config(RustFSBufferConfig::new(profile));
|
||||
|
||||
// Enable buffer profiling globally
|
||||
set_buffer_profile_enabled(true);
|
||||
|
||||
info!("Buffer profiling system initialized successfully");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use rustfs_audit::system::AuditSystemState;
|
||||
use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system};
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
use rustfs_ecstore::config::GLOBAL_SERVER_CONFIG;
|
||||
use tracing::{info, warn};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// Start the audit system.
|
||||
/// This function checks if the audit subsystem is configured in the global server configuration.
|
||||
@@ -89,7 +89,7 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
error!(
|
||||
target: "rustfs::main::start_audit_system",
|
||||
"Audit system startup failed: {:?}",
|
||||
e
|
||||
|
||||
@@ -13,9 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::auth::get_condition_values;
|
||||
use crate::config::workload_profiles::{
|
||||
RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled,
|
||||
};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::entity;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
@@ -121,7 +118,6 @@ use rustfs_utils::{
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
|
||||
use std::ops::Add;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fmt::Debug,
|
||||
@@ -153,103 +149,6 @@ static RUSTFS_OWNER: LazyLock<Owner> = LazyLock::new(|| Owner {
|
||||
id: Some("c19050dbcee97fda828689dda99097a6321af2248fa760517237346e5d9c8a66".to_owned()),
|
||||
});
|
||||
|
||||
/// Calculate adaptive buffer size with workload profile support.
|
||||
///
|
||||
/// This enhanced version supports different workload profiles for optimal performance
|
||||
/// across various use cases (AI/ML, web workloads, secure storage, etc.).
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `file_size` - The size of the file in bytes, or -1 if unknown
|
||||
/// * `profile` - Optional workload profile. If None, uses auto-detection or GeneralPurpose
|
||||
///
|
||||
/// # Returns
|
||||
/// Optimal buffer size in bytes based on the workload profile and file size
|
||||
///
|
||||
/// # Examples
|
||||
/// ```ignore
|
||||
/// // Use general purpose profile (default)
|
||||
/// let buffer_size = get_adaptive_buffer_size_with_profile(1024 * 1024, None);
|
||||
///
|
||||
/// // Use AI training profile for large model files
|
||||
/// let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
/// 500 * 1024 * 1024,
|
||||
/// Some(WorkloadProfile::AiTraining)
|
||||
/// );
|
||||
///
|
||||
/// // Use secure storage profile for compliance scenarios
|
||||
/// let buffer_size = get_adaptive_buffer_size_with_profile(
|
||||
/// 10 * 1024 * 1024,
|
||||
/// Some(WorkloadProfile::SecureStorage)
|
||||
/// );
|
||||
/// ```
|
||||
///
|
||||
#[allow(dead_code)]
|
||||
fn get_adaptive_buffer_size_with_profile(file_size: i64, profile: Option<WorkloadProfile>) -> usize {
|
||||
let config = match profile {
|
||||
Some(p) => RustFSBufferConfig::new(p),
|
||||
None => {
|
||||
// Auto-detect OS environment or use general purpose
|
||||
RustFSBufferConfig::with_auto_detect()
|
||||
}
|
||||
};
|
||||
|
||||
config.get_buffer_size(file_size)
|
||||
}
|
||||
|
||||
/// Get adaptive buffer size using global workload profile configuration.
|
||||
///
|
||||
/// This is the primary buffer sizing function that uses the workload profile
|
||||
/// system configured at startup to provide optimal buffer sizes for different scenarios.
|
||||
///
|
||||
/// The function automatically selects buffer sizes based on:
|
||||
/// - Configured workload profile (default: GeneralPurpose)
|
||||
/// - File size characteristics
|
||||
/// - Optional performance metrics collection
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `file_size` - The size of the file in bytes, or -1 if unknown
|
||||
///
|
||||
/// # Returns
|
||||
/// Optimal buffer size in bytes based on the configured workload profile
|
||||
///
|
||||
/// # Performance Metrics
|
||||
/// When compiled with the `metrics` feature flag, this function tracks:
|
||||
/// - Buffer size distribution
|
||||
/// - Selection frequency
|
||||
/// - Buffer-to-file size ratios
|
||||
///
|
||||
/// # Examples
|
||||
/// ```ignore
|
||||
/// // Uses configured profile (default: GeneralPurpose)
|
||||
/// let buffer_size = get_buffer_size_opt_in(file_size);
|
||||
/// ```
|
||||
fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
let buffer_size = if is_buffer_profile_enabled() {
|
||||
// Use globally configured workload profile (enabled by default in Phase 3)
|
||||
let config = get_global_buffer_config();
|
||||
config.get_buffer_size(file_size)
|
||||
} else {
|
||||
// Opt-out mode: Use GeneralPurpose profile for consistent behavior
|
||||
let config = RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose);
|
||||
config.get_buffer_size(file_size)
|
||||
};
|
||||
|
||||
// Optional performance metrics collection for monitoring and optimization
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
use metrics::histogram;
|
||||
histogram!("rustfs_buffer_size_bytes").record(buffer_size as f64);
|
||||
counter!("rustfs_buffer_size_selections").increment(1);
|
||||
|
||||
if file_size >= 0 {
|
||||
let ratio = buffer_size as f64 / file_size as f64;
|
||||
histogram!("rustfs_buffer_to_file_ratio").record(ratio);
|
||||
}
|
||||
}
|
||||
|
||||
buffer_size
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
// pub store: ECStore,
|
||||
@@ -471,6 +370,8 @@ impl FS {
|
||||
let event_version_id = version_id;
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
|
||||
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
|
||||
|
||||
let size = match content_length {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
@@ -485,15 +386,6 @@ impl FS {
|
||||
}
|
||||
};
|
||||
|
||||
// Apply adaptive buffer sizing based on file size for optimal streaming performance.
|
||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||
// Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile.
|
||||
let buffer_size = get_buffer_size_opt_in(size);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
|
||||
let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else {
|
||||
return Err(s3_error!(InvalidArgument, "key extension not found"));
|
||||
};
|
||||
@@ -1150,7 +1042,7 @@ impl S3 for FS {
|
||||
warn!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err.to_string());
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::Custom("ErrRestoreTransitionedObject".into()),
|
||||
format!("unable to restore transitioned bucket/object {bucket}/{object}: {err}"),
|
||||
format!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1402,7 +1294,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
if is_dir_object(&object.object_name) && object.version_id.is_none() {
|
||||
object.version_id = Some(Uuid::nil());
|
||||
object.version_id = Some(Uuid::max());
|
||||
}
|
||||
|
||||
if replicate_deletes {
|
||||
@@ -1619,10 +1511,6 @@ impl S3 for FS {
|
||||
version_id,
|
||||
part_number,
|
||||
range,
|
||||
if_none_match,
|
||||
if_match,
|
||||
if_modified_since,
|
||||
if_unmodified_since,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
@@ -1669,36 +1557,6 @@ impl S3 for FS {
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let info = reader.object_info;
|
||||
|
||||
if let Some(match_etag) = if_none_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag == match_etag.as_str()) {
|
||||
return Err(S3Error::new(S3ErrorCode::NotModified));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(modified_since) = if_modified_since {
|
||||
// obj_time < givenTime + 1s
|
||||
if info.mod_time.is_some_and(|mod_time| {
|
||||
let give_time: OffsetDateTime = modified_since.into();
|
||||
mod_time < give_time.add(time::Duration::seconds(1))
|
||||
}) {
|
||||
return Err(S3Error::new(S3ErrorCode::NotModified));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(match_etag) = if_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag != match_etag.as_str()) {
|
||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||
}
|
||||
} else if let Some(unmodified_since) = if_unmodified_since {
|
||||
if info.mod_time.is_some_and(|mod_time| {
|
||||
let give_time: OffsetDateTime = unmodified_since.into();
|
||||
mod_time > give_time.add(time::Duration::seconds(1))
|
||||
}) {
|
||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||
}
|
||||
}
|
||||
|
||||
debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot");
|
||||
for part in &info.parts {
|
||||
debug!(
|
||||
@@ -2011,10 +1869,6 @@ impl S3 for FS {
|
||||
version_id,
|
||||
part_number,
|
||||
range,
|
||||
if_none_match,
|
||||
if_match,
|
||||
if_modified_since,
|
||||
if_unmodified_since,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
@@ -2053,35 +1907,6 @@ impl S3 for FS {
|
||||
|
||||
let info = store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
if let Some(match_etag) = if_none_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag == match_etag.as_str()) {
|
||||
return Err(S3Error::new(S3ErrorCode::NotModified));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(modified_since) = if_modified_since {
|
||||
// obj_time < givenTime + 1s
|
||||
if info.mod_time.is_some_and(|mod_time| {
|
||||
let give_time: OffsetDateTime = modified_since.into();
|
||||
mod_time < give_time.add(time::Duration::seconds(1))
|
||||
}) {
|
||||
return Err(S3Error::new(S3ErrorCode::NotModified));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(match_etag) = if_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag != match_etag.as_str()) {
|
||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||
}
|
||||
} else if let Some(unmodified_since) = if_unmodified_since {
|
||||
if info.mod_time.is_some_and(|mod_time| {
|
||||
let give_time: OffsetDateTime = unmodified_since.into();
|
||||
mod_time > give_time.add(time::Duration::seconds(1))
|
||||
}) {
|
||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||
}
|
||||
}
|
||||
|
||||
let event_info = info.clone();
|
||||
let content_type = {
|
||||
if let Some(content_type) = &info.content_type {
|
||||
@@ -2188,20 +2013,15 @@ impl S3 for FS {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let mut bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
|
||||
let mut req = req;
|
||||
|
||||
if req.credentials.as_ref().is_none_or(|cred| cred.access_key.is_empty()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let bucket_infos = if let Err(e) = authorize_request(&mut req, Action::S3Action(S3Action::ListAllMyBucketsAction)).await {
|
||||
if e.code() != &S3ErrorCode::AccessDenied {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
let mut list_bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
|
||||
list_bucket_infos = futures::stream::iter(list_bucket_infos)
|
||||
if authorize_request(&mut req, Action::S3Action(S3Action::ListAllMyBucketsAction))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
bucket_infos = futures::stream::iter(bucket_infos)
|
||||
.filter_map(|info| async {
|
||||
let mut req_clone = req.clone();
|
||||
let req_info = req_clone.extensions.get_mut::<ReqInfo>().expect("ReqInfo not found");
|
||||
@@ -2221,14 +2041,7 @@ impl S3 for FS {
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
if list_bucket_infos.is_empty() {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
list_bucket_infos
|
||||
} else {
|
||||
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
|
||||
};
|
||||
}
|
||||
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
.iter()
|
||||
@@ -2305,11 +2118,6 @@ impl S3 for FS {
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
|
||||
let incl_deleted = req
|
||||
.headers
|
||||
.get(rustfs_utils::http::headers::RUSTFS_INCLUDE_DELETED)
|
||||
.is_some_and(|v| v.to_str().unwrap_or_default() == "true");
|
||||
|
||||
let object_infos = store
|
||||
.list_objects_v2(
|
||||
&bucket,
|
||||
@@ -2319,7 +2127,6 @@ impl S3 for FS {
|
||||
max_keys,
|
||||
fetch_owner.unwrap_or_default(),
|
||||
start_after,
|
||||
incl_deleted,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
@@ -2496,43 +2303,9 @@ impl S3 for FS {
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id,
|
||||
content_md5,
|
||||
if_match,
|
||||
if_none_match,
|
||||
..
|
||||
} = input;
|
||||
|
||||
if if_match.is_some() || if_none_match.is_some() {
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
||||
Ok(info) => {
|
||||
if !info.delete_marker {
|
||||
if let Some(ifmatch) = if_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag != ifmatch.as_str()) {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
}
|
||||
if let Some(ifnonematch) = if_none_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag == ifnonematch.as_str()) {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) || !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
if if_match.is_some() && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
|
||||
let mut size = match content_length {
|
||||
@@ -2553,14 +2326,7 @@ impl S3 for FS {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
}
|
||||
|
||||
// Apply adaptive buffer sizing based on file size for optimal streaming performance.
|
||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||
// Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile.
|
||||
let buffer_size = get_buffer_size_opt_in(size);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
|
||||
|
||||
// let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))));
|
||||
|
||||
@@ -3080,14 +2846,7 @@ impl S3 for FS {
|
||||
|
||||
let mut size = size.ok_or_else(|| s3_error!(UnexpectedContent))?;
|
||||
|
||||
// Apply adaptive buffer sizing based on part size for optimal streaming performance.
|
||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||
// Buffer sizes range from 32KB to 4MB depending on part size and configured workload profile.
|
||||
let buffer_size = get_buffer_size_opt_in(size);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
let body = StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
|
||||
|
||||
// mc cp step 4
|
||||
|
||||
@@ -3488,6 +3247,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn list_multipart_uploads(
|
||||
&self,
|
||||
req: S3Request<ListMultipartUploadsInput>,
|
||||
@@ -3516,6 +3276,11 @@ impl S3 for FS {
|
||||
}
|
||||
}
|
||||
|
||||
warn!(
|
||||
"List multipart uploads with bucket={}, prefix={}, delimiter={:?}, key_marker={:?}, upload_id_marker={:?}, max_uploads={}",
|
||||
bucket, prefix, delimiter, key_marker, upload_id_marker, max_uploads
|
||||
);
|
||||
|
||||
let result = store
|
||||
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
|
||||
.await
|
||||
@@ -3567,43 +3332,9 @@ impl S3 for FS {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
if_match,
|
||||
if_none_match,
|
||||
..
|
||||
} = input;
|
||||
|
||||
if if_match.is_some() || if_none_match.is_some() {
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
||||
Ok(info) => {
|
||||
if !info.delete_marker {
|
||||
if let Some(ifmatch) = if_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag != ifmatch.as_str()) {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
}
|
||||
if let Some(ifnonematch) = if_none_match {
|
||||
if info.etag.as_ref().is_some_and(|etag| etag == ifnonematch.as_str()) {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) || !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
if if_match.is_some() && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) };
|
||||
|
||||
let opts = &get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?;
|
||||
@@ -5147,7 +4878,6 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_config::MI_B;
|
||||
|
||||
#[test]
|
||||
fn test_fs_creation() {
|
||||
@@ -5220,204 +4950,6 @@ mod tests {
|
||||
assert_eq!(gz_format.extension(), "gz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adaptive_buffer_size_with_profile() {
|
||||
const KB: i64 = 1024;
|
||||
const MB: i64 = 1024 * 1024;
|
||||
|
||||
// Test GeneralPurpose profile (default behavior, should match get_adaptive_buffer_size)
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::GeneralPurpose)),
|
||||
64 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::GeneralPurpose)),
|
||||
256 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(200 * MB, Some(WorkloadProfile::GeneralPurpose)),
|
||||
DEFAULT_READ_BUFFER_SIZE
|
||||
);
|
||||
|
||||
// Test AiTraining profile - larger buffers for large files
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::AiTraining)),
|
||||
512 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::AiTraining)),
|
||||
2 * MB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(600 * MB, Some(WorkloadProfile::AiTraining)),
|
||||
4 * MB as usize
|
||||
);
|
||||
|
||||
// Test WebWorkload profile - smaller buffers for web assets
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(100 * KB, Some(WorkloadProfile::WebWorkload)),
|
||||
32 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::WebWorkload)),
|
||||
128 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::WebWorkload)),
|
||||
256 * KB as usize
|
||||
);
|
||||
|
||||
// Test SecureStorage profile - memory-constrained buffers
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::SecureStorage)),
|
||||
32 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::SecureStorage)),
|
||||
128 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::SecureStorage)),
|
||||
256 * KB as usize
|
||||
);
|
||||
|
||||
// Test IndustrialIoT profile - low latency, moderate buffers
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(512 * KB, Some(WorkloadProfile::IndustrialIoT)),
|
||||
64 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::IndustrialIoT)),
|
||||
256 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::IndustrialIoT)),
|
||||
512 * KB as usize
|
||||
);
|
||||
|
||||
// Test DataAnalytics profile
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(2 * MB, Some(WorkloadProfile::DataAnalytics)),
|
||||
128 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::DataAnalytics)),
|
||||
512 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(500 * MB, Some(WorkloadProfile::DataAnalytics)),
|
||||
2 * MB as usize
|
||||
);
|
||||
|
||||
// Test with None (should auto-detect or use GeneralPurpose)
|
||||
let result = get_adaptive_buffer_size_with_profile(50 * MB, None);
|
||||
// Should be either SecureStorage (if on special OS) or GeneralPurpose
|
||||
assert!(result == 128 * KB as usize || result == 256 * KB as usize);
|
||||
|
||||
// Test unknown file size with different profiles
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::AiTraining)),
|
||||
2 * MB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::WebWorkload)),
|
||||
128 * KB as usize
|
||||
);
|
||||
assert_eq!(
|
||||
get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::SecureStorage)),
|
||||
128 * KB as usize
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_phase3_default_behavior() {
|
||||
use crate::config::workload_profiles::{
|
||||
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
|
||||
};
|
||||
|
||||
const KB: i64 = 1024;
|
||||
const MB: i64 = 1024 * 1024;
|
||||
|
||||
// Test Phase 3: Enabled by default with GeneralPurpose profile
|
||||
set_buffer_profile_enabled(true);
|
||||
init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose));
|
||||
|
||||
// Verify GeneralPurpose profile provides consistent buffer sizes
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B);
|
||||
assert_eq!(get_buffer_size_opt_in(-1), MI_B); // Unknown size
|
||||
|
||||
// Reset for other tests
|
||||
set_buffer_profile_enabled(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_buffer_size_opt_in() {
|
||||
use crate::config::workload_profiles::{is_buffer_profile_enabled, set_buffer_profile_enabled};
|
||||
|
||||
const KB: i64 = 1024;
|
||||
const MB: i64 = 1024 * 1024;
|
||||
|
||||
// \[1\] Default state: profile is not enabled, global configuration is not explicitly initialized
|
||||
// get_buffer_size_opt_in should be equivalent to the GeneralPurpose configuration
|
||||
set_buffer_profile_enabled(false);
|
||||
assert!(!is_buffer_profile_enabled());
|
||||
|
||||
// GeneralPurpose rules:
|
||||
// \< 1MB -> 64KB，1MB-100MB -> 256KB，\>=100MB -> 1MB
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B);
|
||||
|
||||
// \[2\] Enable the profile switch, but the global configuration is still the default GeneralPurpose
|
||||
set_buffer_profile_enabled(true);
|
||||
assert!(is_buffer_profile_enabled());
|
||||
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B);
|
||||
|
||||
// \[3\] Close again to ensure unchanged behavior
|
||||
set_buffer_profile_enabled(false);
|
||||
assert!(!is_buffer_profile_enabled());
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_phase4_full_integration() {
|
||||
use crate::config::workload_profiles::{
|
||||
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
|
||||
};
|
||||
|
||||
const KB: i64 = 1024;
|
||||
const MB: i64 = 1024 * 1024;
|
||||
|
||||
// \[1\] During the entire test process, the global configuration is initialized only once.
|
||||
// In order not to interfere with other tests, use GeneralPurpose (consistent with the default).
|
||||
// If it has been initialized elsewhere, this call will be ignored by OnceLock and the behavior will still be GeneralPurpose.
|
||||
init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose));
|
||||
|
||||
// Make sure to turn off profile initially
|
||||
set_buffer_profile_enabled(false);
|
||||
|
||||
// \[2\] Verify behavior of get_buffer_size_opt_in in disabled profile (GeneralPurpose)
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B);
|
||||
|
||||
// \[3\] When profile is enabled, the behavior remains consistent with the global GeneralPurpose configuration
|
||||
set_buffer_profile_enabled(true);
|
||||
assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize);
|
||||
assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B);
|
||||
|
||||
// \[4\] Complex scenes, boundary values: such as unknown size
|
||||
assert_eq!(get_buffer_size_opt_in(-1), MI_B);
|
||||
|
||||
set_buffer_profile_enabled(false);
|
||||
}
|
||||
|
||||
// Note: S3Request structure is complex and requires many fields.
|
||||
// For real testing, we would need proper integration test setup.
|
||||
// Removing this test as it requires too much S3 infrastructure setup.
|
||||
|
||||
@@ -157,7 +157,7 @@ impl OperationHelper {
|
||||
.clone()
|
||||
.status(status)
|
||||
.status_code(status_code)
|
||||
.time_to_response(format!("{ttr:.2?}"))
|
||||
.time_to_response(format!("{:.2?}", ttr))
|
||||
.time_to_response_in_ns(ttr.as_nanos().to_string())
|
||||
.build();
|
||||
|
||||
|
||||
@@ -65,12 +65,15 @@ pub async fn del_opts(
|
||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||
|
||||
if let Some(ref id) = vid {
|
||||
if *id != Uuid::nil().to_string()
|
||||
&& let Err(err) = Uuid::parse_str(id.as_str())
|
||||
{
|
||||
if let Err(err) = Uuid::parse_str(id.as_str()) {
|
||||
error!("del_opts: invalid version id: {} error: {}", id, err);
|
||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
|
||||
if !versioned {
|
||||
error!("del_opts: object not versioned: {}", object);
|
||||
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let mut opts = put_opts_from_headers(headers, metadata.clone()).map_err(|err| {
|
||||
@@ -80,7 +83,7 @@ pub async fn del_opts(
|
||||
|
||||
opts.version_id = {
|
||||
if is_dir_object(object) && vid.is_none() {
|
||||
Some(Uuid::nil().to_string())
|
||||
Some(Uuid::max().to_string())
|
||||
} else {
|
||||
vid
|
||||
}
|
||||
@@ -110,11 +113,13 @@ pub async fn get_opts(
|
||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||
|
||||
if let Some(ref id) = vid {
|
||||
if *id != Uuid::nil().to_string()
|
||||
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
||||
{
|
||||
if let Err(_err) = Uuid::parse_str(id.as_str()) {
|
||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
|
||||
if !versioned {
|
||||
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let mut opts = get_default_opts(headers, HashMap::new(), false)
|
||||
@@ -122,7 +127,7 @@ pub async fn get_opts(
|
||||
|
||||
opts.version_id = {
|
||||
if is_dir_object(object) && vid.is_none() {
|
||||
Some(Uuid::nil().to_string())
|
||||
Some(Uuid::max().to_string())
|
||||
} else {
|
||||
vid
|
||||
}
|
||||
@@ -184,11 +189,13 @@ pub async fn put_opts(
|
||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||
|
||||
if let Some(ref id) = vid {
|
||||
if *id != Uuid::nil().to_string()
|
||||
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
||||
{
|
||||
if let Err(_err) = Uuid::parse_str(id.as_str()) {
|
||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
|
||||
if !versioned {
|
||||
return Err(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let mut opts = put_opts_from_headers(headers, metadata)
|
||||
@@ -196,7 +203,7 @@ pub async fn put_opts(
|
||||
|
||||
opts.version_id = {
|
||||
if is_dir_object(object) && vid.is_none() {
|
||||
Some(Uuid::nil().to_string())
|
||||
Some(Uuid::max().to_string())
|
||||
} else {
|
||||
vid
|
||||
}
|
||||
@@ -596,7 +603,7 @@ mod tests {
|
||||
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert_eq!(opts.version_id, Some(Uuid::nil().to_string()));
|
||||
assert_eq!(opts.version_id, Some(Uuid::max().to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -669,7 +676,7 @@ mod tests {
|
||||
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert_eq!(opts.version_id, Some(Uuid::nil().to_string()));
|
||||
assert_eq!(opts.version_id, Some(Uuid::max().to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -713,7 +720,7 @@ mod tests {
|
||||
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert_eq!(opts.version_id, Some(Uuid::nil().to_string()));
|
||||
assert_eq!(opts.version_id, Some(Uuid::max().to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -28,8 +28,8 @@ fi
|
||||
current_dir=$(pwd)
|
||||
echo "Current directory: $current_dir"
|
||||
|
||||
# mkdir -p ./target/volume/test
|
||||
mkdir -p ./target/volume/test{1..4}
|
||||
mkdir -p ./target/volume/test
|
||||
# mkdir -p ./target/volume/test{1..4}
|
||||
|
||||
|
||||
if [ -z "$RUST_LOG" ]; then
|
||||
@@ -41,8 +41,8 @@ fi
|
||||
|
||||
# export RUSTFS_STORAGE_CLASS_INLINE_BLOCK="512 KB"
|
||||
|
||||
export RUSTFS_VOLUMES="./target/volume/test{1...4}"
|
||||
# export RUSTFS_VOLUMES="./target/volume/test"
|
||||
# export RUSTFS_VOLUMES="./target/volume/test{1...4}"
|
||||
export RUSTFS_VOLUMES="./target/volume/test"
|
||||
export RUSTFS_ADDRESS=":9000"
|
||||
export RUSTFS_CONSOLE_ENABLE=true
|
||||
export RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
@@ -77,7 +77,7 @@ export RUSTFS_OBS_LOG_FLUSH_MS=300
|
||||
#tokio runtime
|
||||
export RUSTFS_RUNTIME_WORKER_THREADS=16
|
||||
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
|
||||
export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=true
|
||||
export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=false
|
||||
# shellcheck disable=SC2125
|
||||
export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024
|
||||
export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60
|
||||
|
||||
Reference in New Issue
Block a user