Add complete MNMD Docker deployment example with startup coordination and VolumeNotFound fix (#642)

* Initial plan * Add MNMD Docker deployment example with 4 nodes x 4 drives - Create docs/examples/mnmd/ directory structure - Add docker-compose.yml with proper disk indexing (1..4) - Add wait-and-start.sh for startup coordination - Add README.md with usage instructions and alternatives - Add CHECKLIST.md with step-by-step verification - Fixes VolumeNotFound issue by using correct volume paths - Implements health checks and startup ordering - Uses service names for stable inter-node addressing Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add docs/examples README as index for deployment examples Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add automated test script for MNMD deployment - Add test-deployment.sh with comprehensive validation - Test container status, health, endpoints, connectivity - Update README to reference test script - Make script executable Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * improve code * improve code * improve dep crates `cargo shear --fix` * upgrade aws-sdk-s3 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-01-16 17:20:33 +00:00 · 2025-10-12 13:15:14 +08:00
parent aac9b1edb7
commit ad99019749
10 changed files with 1050 additions and 107 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1372,9 +1372,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

 [[package]]
 name = "cc"
-version = "1.2.40"
+version = "1.2.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1d05d92f4b1fd76aad469d46cdd858ca761576082cd37df81416691e50199fb"
+checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7"
 dependencies = [
 "find-msvc-tools",
 "jobserver",
@@ -2924,7 +2924,7 @@ dependencies = [
 "flatbuffers",
 "futures",
 "http 1.3.1",
- "md5 0.8.0",
+ "md5",
 "rand 0.9.2",
 "reqwest",
 "rmp-serde",
@@ -3197,9 +3197,9 @@ dependencies = [

 [[package]]
 name = "find-msvc-tools"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0399f9d26e5191ce32c498bebd31e7a3ceabc2745f0ac54af3f335126c3f24b3"
+checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"

 [[package]]
 name = "findshlibs"
@@ -4561,12 +4561,6 @@ dependencies = [
 "digest 0.10.7",
 ]

-[[package]]
-name = "md5"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
-
 [[package]]
 name = "md5"
 version = "0.8.0"
@@ -4814,11 +4808,11 @@ dependencies = [

 [[package]]
 name = "nu-ansi-term"
-version = "0.50.1"
+version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
+checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]

 [[package]]
@@ -5364,12 +5358,12 @@ dependencies = [

 [[package]]
 name = "pem"
-version = "3.0.5"
+version = "3.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3"
+checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
 dependencies = [
 "base64 0.22.1",
- "serde",
+ "serde_core",
 ]

 [[package]]
@@ -5642,7 +5636,7 @@ version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
 dependencies = [
- "toml_edit 0.23.6",
+ "toml_edit 0.23.7",
 ]

 [[package]]
@@ -6084,9 +6078,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.11.3"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c"
+checksum = "4a52d8d02cacdb176ef4678de6c052efb4b3da14b78e4db683a4252762be5433"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -6096,9 +6090,9 @@ dependencies = [

 [[package]]
 name = "regex-automata"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad"
+checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -6107,15 +6101,15 @@ dependencies = [

 [[package]]
 name = "regex-lite"
-version = "0.1.7"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30"
+checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da"

 [[package]]
 name = "regex-syntax"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
+checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298"

 [[package]]
 name = "reqwest"
@@ -6384,18 +6378,16 @@ dependencies = [
 "flatbuffers",
 "futures",
 "futures-util",
- "hashbrown 0.16.0",
 "http 1.3.1",
 "http-body 1.0.1",
 "hyper 1.7.0",
 "hyper-util",
 "libsystemd",
 "matchit",
- "md5 0.8.0",
+ "md5",
 "mimalloc",
 "mime_guess",
 "opentelemetry",
- "percent-encoding",
 "pin-project-lite",
 "pprof",
 "reqwest",
@@ -6572,7 +6564,6 @@ dependencies = [
 "async-trait",
 "aws-credential-types",
 "aws-sdk-s3",
- "aws-smithy-runtime-api",
 "aws-smithy-types",
 "base64 0.22.1",
 "byteorder",
@@ -6583,7 +6574,6 @@ dependencies = [
 "enumset",
 "flatbuffers",
 "futures",
- "futures-util",
 "glob",
 "hex-simd",
 "hmac 0.12.1",
@@ -6596,7 +6586,6 @@ dependencies = [
 "moka",
 "nix 0.30.1",
 "num_cpus",
- "once_cell",
 "parking_lot",
 "path-absolutize",
 "pin-project-lite",
@@ -6616,7 +6605,6 @@ dependencies = [
 "rustfs-policy",
 "rustfs-protos",
 "rustfs-rio",
- "rustfs-rsc",
 "rustfs-signer",
 "rustfs-utils",
 "rustfs-workers",
@@ -6633,7 +6621,6 @@ dependencies = [
 "thiserror 2.0.17",
 "time",
 "tokio",
- "tokio-stream",
 "tokio-util",
 "tonic",
 "tower",
@@ -6700,7 +6687,7 @@ dependencies = [
 "base64 0.22.1",
 "chacha20poly1305",
 "chrono",
- "md5 0.8.0",
+ "md5",
 "moka",
 "once_cell",
 "rand 0.9.2",
@@ -6886,33 +6873,6 @@ dependencies = [
 "tracing",
 ]

-[[package]]
-name = "rustfs-rsc"
-version = "2025.506.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8229535cdd7a9d1f5757bd7b588f342e6ea984d66c6b1f6350f9de85d3ce9c25"
-dependencies = [
- "async-stream",
- "base64 0.22.1",
- "bytes",
- "chrono",
- "crc32fast",
- "futures",
- "futures-core",
- "futures-util",
- "hex",
- "hmac 0.12.1",
- "hyper 1.7.0",
- "md5 0.7.0",
- "once_cell",
- "regex",
- "reqwest",
- "serde",
- "serde-xml-rs",
- "sha2 0.10.9",
- "urlencoding",
-]
-
 [[package]]
 name = "rustfs-s3select-api"
 version = "0.0.5"
@@ -7478,18 +7438,6 @@ dependencies = [
 "serde",
 ]

-[[package]]
-name = "serde-xml-rs"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782"
-dependencies = [
- "log",
- "serde",
- "thiserror 1.0.69",
- "xml-rs",
-]
-
 [[package]]
 name = "serde_core"
 version = "1.0.228"
@@ -8536,9 +8484,9 @@ dependencies = [

 [[package]]
 name = "toml_datetime"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1"
+checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533"
 dependencies = [
 "serde_core",
 ]
@@ -8559,21 +8507,21 @@ dependencies = [

 [[package]]
 name = "toml_edit"
-version = "0.23.6"
+version = "0.23.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b"
+checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
 dependencies = [
 "indexmap",
- "toml_datetime 0.7.2",
+ "toml_datetime 0.7.3",
 "toml_parser",
 "winnow",
 ]

 [[package]]
 name = "toml_parser"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627"
+checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e"
 dependencies = [
 "winnow",
 ]
@@ -8848,9 +8796,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"

 [[package]]
 name = "tz-rs"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1450bf2b99397e72070e7935c89facaa80092ac812502200375f1f7d33c71a1"
+checksum = "14eff19b8dc1ace5bf7e4d920b2628ae3837f422ff42210cb1567cbf68b5accf"

 [[package]]
 name = "tzdb"
@@ -9711,12 +9659,6 @@ dependencies = [
 "rustix 1.1.2",
 ]

-[[package]]
-name = "xml-rs"
-version = "0.8.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7"
-
 [[package]]
 name = "xmlparser"
 version = "0.13.6"
@@ -9864,9 +9806,9 @@ dependencies = [

 [[package]]
 name = "zip"
-version = "5.1.1"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532"
+checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
 dependencies = [
 "aes",
 "arbitrary",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -102,8 +102,7 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.8.8" }
 aws-credential-types = { version = "1.2.8" }
 aws-smithy-types = { version = "1.3.3" }
-aws-smithy-runtime-api = { version = "1.9.1" }
-aws-sdk-s3 = { version = "1.106.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.108.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
 axum = "0.8.6"
 axum-extra = "0.10.3"
 axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false }
@@ -163,7 +162,7 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.11", features = ["future"] }
 netif = "0.1.6"
 nix = { version = "0.30.1", features = ["fs"] }
-nu-ansi-term = "0.50.1"
+nu-ansi-term = "0.50.3"
 num_cpus = { version = "1.17.0" }
 nvml-wrapper = "0.11.0"
 object_store = "0.12.4"
@@ -187,7 +186,6 @@ path-absolutize = "3.1.1"
 path-clean = "1.0.1"
 blake3 = { version = "1.8.2" }
 pbkdf2 = "0.12.2"
-percent-encoding = "2.3.2"
 pin-project-lite = "0.2.16"
 prost = "0.14.1"
 pretty_assertions = "1.4.1"
@@ -196,7 +194,7 @@ rand = "0.9.2"
 rayon = "1.11.0"
 rdkafka = { version = "0.38.0", features = ["tokio"] }
 reed-solomon-simd = { version = "3.0.1" }
-regex = { version = "1.11.3" }
+regex = { version = "1.12.1" }
 reqwest = { version = "0.12.23", default-features = false, features = [
    "rustls-tls-webpki-roots",
    "charset",
@@ -213,7 +211,6 @@ rsa = "0.9.8"
 rumqttc = { version = "0.25.0" }
 rust-embed = { version = "8.7.2" }
 rustc-hash = { version = "2.1.1" }
-rustfs-rsc = "2025.506.1"
 rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false }
 rustls-pki-types = "1.12.0"
 rustls-pemfile = "2.2.0"
@@ -277,7 +274,7 @@ wildmatch = { version = "2.5.0", features = ["serde"] }
 zeroize = { version = "1.8.2", features = ["derive"] }
 winapi = { version = "0.3.9" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "5.1.1"
+zip = "6.0.0"
 zstd = "0.13.3"


--- a/crates/ecstore/Cargo.toml
+++ b/crates/ecstore/Cargo.toml
@@ -75,7 +75,6 @@ hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
 tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
-tokio-stream = { workspace = true }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true
@@ -89,8 +88,6 @@ rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true }
-once_cell = { workspace = true }
-rustfs-rsc = { workspace = true }
 urlencoding = { workspace = true }
 smallvec = { workspace = true }
 shadow-rs.workspace = true
@@ -99,13 +96,11 @@ rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-rio.workspace = true
 rustfs-signer.workspace = true
 rustfs-checksums.workspace = true
-futures-util.workspace = true
 async-recursion.workspace = true
 aws-credential-types = { workspace = true }
 aws-smithy-types = { workspace = true }
 parking_lot = { workspace = true }
 moka = { workspace = true }
-aws-smithy-runtime-api = { workspace = true }

 [target.'cfg(not(windows))'.dependencies]
 nix = { workspace = true }
--- a/crates/utils/Cargo.toml
+++ b/crates/utils/Cargo.toml
@@ -29,7 +29,7 @@ base64-simd = { workspace = true, optional = true }
 blake3 = { workspace = true, optional = true }
 brotli = { workspace = true, optional = true }
 bytes = { workspace = true, optional = true }
-crc32fast = { workspace = true }
+crc32fast = { workspace = true, optional = true }
 flate2 = { workspace = true, optional = true }
 futures = { workspace = true, optional = true }
 hashbrown = { workspace = true, optional = true }
@@ -89,7 +89,7 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde",
 compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
 string = ["dep:regex", "dep:rand"]
 crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"]
-hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd"]
+hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd", "dep:crc32fast"]
 os = ["dep:nix", "dep:tempfile", "winapi"]  # operating system utilities
 integration = []  # integration test features
 sys = ["dep:sysinfo"]  # system information features
--- a/docs/examples/README.md
+++ b/docs/examples/README.md
@@ -0,0 +1,60 @@
+# RustFS Deployment Examples
+
+This directory contains practical deployment examples and configurations for RustFS.
+
+## Available Examples
+
+### [MNMD (Multi-Node Multi-Drive)](./mnmd/)
+
+Complete Docker Compose example for deploying RustFS in a 4-node, 4-drive-per-node configuration.
+
+**Features:**
+- Proper disk indexing (1..4) to avoid VolumeNotFound errors
+- Startup coordination via `wait-and-start.sh` script
+- Service discovery using Docker service names
+- Health checks with alternatives for different base images
+- Comprehensive documentation and verification checklist
+
+**Use Case:** Production-ready multi-node deployment for high availability and performance.
+
+**Quick Start:**
+```bash
+cd docs/examples/mnmd
+docker-compose up -d
+```
+
+**See also:**
+- [MNMD README](./mnmd/README.md) - Detailed usage guide
+- [MNMD CHECKLIST](./mnmd/CHECKLIST.md) - Step-by-step verification
+
+## Other Deployment Examples
+
+For additional deployment examples, see:
+- [`examples/`](/examples/) - Root-level examples directory with:
+  - `docker-quickstart.sh` - Quick start script for basic deployments
+  - `enhanced-docker-deployment.sh` - Advanced deployment scenarios
+  - `docker-comprehensive.yml` - Docker Compose with multiple profiles
+- [`.docker/compose/`](/.docker/compose/) - Docker Compose configurations:
+  - `docker-compose.cluster.yaml` - Basic cluster setup
+  - `docker-compose.observability.yaml` - Observability stack integration
+
+## Related Documentation
+
+- [Console & Endpoint Service Separation](../console-separation.md)
+- [Environment Variables](../ENVIRONMENT_VARIABLES.md)
+- [Performance Testing](../PERFORMANCE_TESTING.md)
+
+## Contributing
+
+When adding new examples:
+1. Create a dedicated subdirectory under `docs/examples/`
+2. Include a comprehensive README.md
+3. Provide working configuration files
+4. Add verification steps or checklists
+5. Document common issues and troubleshooting
+
+## Support
+
+For issues or questions:
+- GitHub Issues: https://github.com/rustfs/rustfs/issues
+- Documentation: https://rustfs.io
--- a/docs/examples/mnmd/CHECKLIST.md
+++ b/docs/examples/mnmd/CHECKLIST.md
@@ -0,0 +1,329 @@
+# MNMD Deployment Checklist
+
+This checklist provides step-by-step verification for deploying RustFS in MNMD (Multi-Node Multi-Drive) mode using
+Docker.
+
+## Pre-Deployment Checks
+
+### 1. System Requirements
+
+- [ ] Docker Engine 20.10+ installed
+- [ ] Docker Compose 2.0+ installed
+- [ ] At least 8GB RAM available
+- [ ] At least 40GB disk space available (for 4 nodes × 4 volumes)
+
+Verify with:
+
+```bash
+docker --version
+docker-compose --version
+free -h
+df -h
+```
+
+### 2. File System Checks
+
+- [ ] Using XFS, ext4, or another suitable filesystem (not NFS for production)
+- [ ] File system supports extended attributes
+
+Verify with:
+
+```bash
+df -T | grep -E '(xfs|ext4)'
+```
+
+### 3. Permissions and SELinux
+
+- [ ] Current user is in `docker` group or can run `sudo docker`
+- [ ] SELinux is properly configured (if enabled)
+
+Verify with:
+
+```bash
+groups | grep docker
+getenforce  # If enabled, should show "Permissive" or "Enforcing" with proper policies
+```
+
+### 4. Network Configuration
+
+- [ ] Ports 9000-9031 are available
+- [ ] No firewall blocking Docker bridge network
+
+Verify with:
+
+```bash
+# Check if ports are free
+netstat -tuln | grep -E ':(9000|9001|9010|9011|9020|9021|9030|9031)'
+# Should return nothing if ports are free
+```
+
+### 5. Files Present
+
+- [ ] `docker-compose.yml` exists in current directory
+
+Verify with:
+
+```bash
+cd docs/examples/mnmd
+ls -la
+chmod +x wait-and-start.sh  # If needed
+```
+
+## Deployment Steps
+
+### 1. Start the Cluster
+
+- [ ] Navigate to the example directory
+- [ ] Pull the latest RustFS image
+- [ ] Start the cluster
+
+```bash
+cd docs/examples/mnmd
+docker-compose pull
+docker-compose up -d
+```
+
+### 2. Monitor Startup
+
+- [ ] Watch container logs during startup
+- [ ] Verify no VolumeNotFound errors
+- [ ] Check that peer discovery completes
+
+```bash
+# Watch all logs
+docker-compose logs -f
+
+# Watch specific node
+docker-compose logs -f rustfs-node1
+
+# Look for successful startup messages
+docker-compose logs | grep -i "ready\|listening\|started"
+```
+
+### 3. Verify Container Status
+
+- [ ] All 4 containers are running
+- [ ] All 4 containers show as healthy
+
+```bash
+docker-compose ps
+
+# Expected output: 4 containers in "Up" state with "healthy" status
+```
+
+### 4. Check Health Endpoints
+
+- [ ] API health endpoints respond on all nodes
+- [ ] Console health endpoints respond on all nodes
+
+```bash
+# Test API endpoints
+curl http://localhost:9000/health
+curl http://localhost:9010/health
+curl http://localhost:9020/health
+curl http://localhost:9030/health
+
+# Test Console endpoints
+curl http://localhost:9001/health
+curl http://localhost:9011/health
+curl http://localhost:9021/health
+curl http://localhost:9031/health
+
+# All should return successful health status
+```
+
+## Post-Deployment Verification
+
+### 1. In-Container Checks
+
+- [ ] Data directories exist
+- [ ] Directories have correct permissions
+- [ ] RustFS process is running
+
+```bash
+# Check node1
+docker exec rustfs-node1 ls -la /data/
+docker exec rustfs-node1 ps aux | grep rustfs
+
+# Verify all 4 data directories exist
+docker exec rustfs-node1 ls -d /data/rustfs{1..4}
+```
+
+### 2. DNS and Network Validation
+
+- [ ] Service names resolve correctly
+- [ ] Inter-node connectivity works
+
+```bash
+# DNS resolution test
+docker exec rustfs-node1 nslookup rustfs-node2
+docker exec rustfs-node1 nslookup rustfs-node3
+docker exec rustfs-node1 nslookup rustfs-node4
+
+# Connectivity test (using nc if available)
+docker exec rustfs-node1 nc -zv rustfs-node2 9000
+docker exec rustfs-node1 nc -zv rustfs-node3 9000
+docker exec rustfs-node1 nc -zv rustfs-node4 9000
+
+# Or using telnet/curl
+docker exec rustfs-node1 curl -v http://rustfs-node2:9000/health
+```
+
+### 3. Volume Configuration Validation
+
+- [ ] RUSTFS_VOLUMES environment variable is correct
+- [ ] All 16 endpoints are configured (4 nodes × 4 drives)
+
+```bash
+# Check environment variable
+docker exec rustfs-node1 env | grep RUSTFS_VOLUMES
+
+# Expected output:
+# RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+```
+
+### 4. Cluster Functionality
+
+- [ ] Can list buckets via API
+- [ ] Can create a bucket
+- [ ] Can upload an object
+- [ ] Can download an object
+
+```bash
+# Configure AWS CLI or s3cmd
+export AWS_ACCESS_KEY_ID=rustfsadmin
+export AWS_SECRET_ACCESS_KEY=rustfsadmin
+
+# Using AWS CLI (if installed)
+aws --endpoint-url http://localhost:9000 s3 mb s3://test-bucket
+aws --endpoint-url http://localhost:9000 s3 ls
+echo "test content" > test.txt
+aws --endpoint-url http://localhost:9000 s3 cp test.txt s3://test-bucket/
+aws --endpoint-url http://localhost:9000 s3 ls s3://test-bucket/
+aws --endpoint-url http://localhost:9000 s3 cp s3://test-bucket/test.txt downloaded.txt
+cat downloaded.txt
+
+# Or using curl
+curl -X PUT http://localhost:9000/test-bucket \
+  -H "Host: localhost:9000" \
+  --user rustfsadmin:rustfsadmin
+```
+
+### 5. Healthcheck Verification
+
+- [ ] Docker reports all services as healthy
+- [ ] Healthcheck scripts work in containers
+
+```bash
+# Check Docker health status
+docker inspect rustfs-node1 --format='{{.State.Health.Status}}'
+docker inspect rustfs-node2 --format='{{.State.Health.Status}}'
+docker inspect rustfs-node3 --format='{{.State.Health.Status}}'
+docker inspect rustfs-node4 --format='{{.State.Health.Status}}'
+
+# All should return "healthy"
+
+# Test healthcheck command manually
+docker exec rustfs-node1 nc -z localhost 9000
+echo $?  # Should be 0
+```
+
+## Troubleshooting Checks
+
+### If VolumeNotFound Error Occurs
+
+- [ ] Verify volume indexing starts at 1, not 0
+- [ ] Check that RUSTFS_VOLUMES matches mounted paths
+- [ ] Ensure all /data/rustfs{1..4} directories exist
+
+```bash
+# Check mounted volumes
+docker inspect rustfs-node1 | jq '.[].Mounts'
+
+# Verify directories in container
+docker exec rustfs-node1 ls -la /data/
+```
+
+### If Healthcheck Fails
+
+- [ ] Check if `nc` is available in the image
+- [ ] Try alternative healthcheck (curl/wget)
+- [ ] Increase `start_period` in docker-compose.yml
+
+```bash
+# Check if nc is available
+docker exec rustfs-node1 which nc
+
+# Test healthcheck manually
+docker exec rustfs-node1 nc -z localhost 9000
+
+# Check logs for errors
+docker-compose logs rustfs-node1 | grep -i error
+```
+
+### If Startup Takes Too Long
+
+- [ ] Check peer discovery timeout in logs
+- [ ] Verify network connectivity between nodes
+- [ ] Consider increasing timeout in wait-and-start.sh
+
+```bash
+# Check startup logs
+docker-compose logs rustfs-node1 | grep -i "waiting\|peer\|timeout"
+
+# Check network
+docker network inspect mnmd_rustfs-mnmd
+```
+
+### If Containers Crash or Restart
+
+- [ ] Review container logs
+- [ ] Check resource usage (CPU/Memory)
+- [ ] Verify no port conflicts
+
+```bash
+# View last crash logs
+docker-compose logs --tail=100 rustfs-node1
+
+# Check resource usage
+docker stats --no-stream
+
+# Check restart count
+docker-compose ps
+```
+
+## Cleanup Checklist
+
+When done testing:
+
+- [ ] Stop the cluster: `docker-compose down`
+- [ ] Remove volumes (optional, destroys data): `docker-compose down -v`
+- [ ] Clean up dangling images: `docker image prune`
+- [ ] Verify ports are released: `netstat -tuln | grep -E ':(9000|9001|9010|9011|9020|9021|9030|9031)'`
+
+## Production Deployment Additional Checks
+
+Before deploying to production:
+
+- [ ] Change default credentials (RUSTFS_ACCESS_KEY, RUSTFS_SECRET_KEY)
+- [ ] Configure TLS certificates
+- [ ] Set up proper logging and monitoring
+- [ ] Configure backups for volumes
+- [ ] Review and adjust resource limits
+- [ ] Set up external load balancer (if needed)
+- [ ] Document disaster recovery procedures
+- [ ] Test failover scenarios
+- [ ] Verify data persistence after container restart
+
+## Summary
+
+This checklist ensures:
+
+- ✓ Correct disk indexing (1..4 instead of 0..3)
+- ✓ Proper startup coordination via wait-and-start.sh
+- ✓ Service discovery via Docker service names
+- ✓ Health checks function correctly
+- ✓ All 16 endpoints (4 nodes × 4 drives) are operational
+- ✓ No VolumeNotFound errors occur
+
+For more details, see [README.md](./README.md) in this directory.
--- a/docs/examples/mnmd/README.md
+++ b/docs/examples/mnmd/README.md
@@ -0,0 +1,268 @@
+# RustFS MNMD (Multi-Node Multi-Drive) Docker Example
+
+This directory contains a complete, ready-to-use MNMD deployment example for RustFS with 4 nodes and 4 drives per node (
+4x4 configuration).
+
+## Overview
+
+This example addresses common deployment issues including:
+
+- **VolumeNotFound errors** - Fixed by using correct disk indexing (`/data/rustfs{1...4}` instead of
+  `/data/rustfs{0...3}`)
+- **Startup race conditions** - Solved with a simple `sleep` command in each service.
+- **Service discovery** - Uses Docker service names (`rustfs-node{1..4}`) instead of hard-coded IPs
+- **Health checks** - Implements proper health monitoring with `nc` (with alternatives documented)
+
+## Quick Start
+
+From this directory (`docs/examples/mnmd`), run:
+
+```bash
+# Start the cluster
+docker-compose up -d
+
+# Check the status
+docker-compose ps
+
+# View logs
+docker-compose logs -f
+
+# Test the deployment
+curl http://localhost:9000/health
+curl http://localhost:9001/health
+
+# Run comprehensive tests
+./test-deployment.sh
+
+# Stop the cluster
+docker-compose down
+
+# Clean up volumes (WARNING: deletes all data)
+docker-compose down -v
+```
+
+## Configuration Details
+
+### Volume Configuration
+
+The example uses the following volume configuration:
+
+```bash
+RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+```
+
+This expands to 16 endpoints (4 nodes × 4 drives):
+
+- Node 1: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4`
+- Node 2: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4`
+- Node 3: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4`
+- Node 4: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4`
+
+**Important:** Disk indexing starts at 1 to match the mounted paths (`/data/rustfs1..4`).
+
+### Port Mappings
+
+| Node  | API Port | Console Port |
+|-------|----------|--------------|
+| node1 | 9000     | 9001         |
+| node2 | 9010     | 9011         |
+| node3 | 9020     | 9021         |
+| node4 | 9030     | 9031         |
+
+### Startup Coordination
+
+To prevent race conditions during startup where nodes might not find each other, a simple `sleep 3` command is added to
+each service's command. This provides a brief delay, allowing the network and other services to initialize before RustFS
+starts. For more complex scenarios, a more robust health-check dependency or an external entrypoint script might be
+required.
+
+### Health Checks
+
+Default health check using `nc` (netcat):
+
+```yaml
+healthcheck:
+  test: [ "CMD-SHELL", "nc -z localhost 9000 || exit 1" ]
+  interval: 10s
+  timeout: 5s
+  retries: 3
+  start_period: 30s
+```
+
+#### Alternative Health Checks
+
+If your base image lacks `nc`, use one of these alternatives:
+
+**Using curl:**
+
+```yaml
+healthcheck:
+  test: [ "CMD-SHELL", "curl -f http://localhost:9000/health || exit 1" ]
+  interval: 10s
+  timeout: 5s
+  retries: 3
+  start_period: 30s
+```
+
+**Using wget:**
+
+```yaml
+healthcheck:
+  test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9000/health || exit 1" ]
+  interval: 10s
+  timeout: 5s
+  retries: 3
+  start_period: 30s
+```
+
+### Brace Expansion Alternatives
+
+If your Docker Compose runtime doesn't support brace expansion (`{1...4}`), replace with explicit endpoints:
+
+```yaml
+environment:
+  - RUSTFS_VOLUMES=http://rustfs-node1:9000/data/rustfs1,http://rustfs-node1:9000/data/rustfs2,http://rustfs-node1:9000/data/rustfs3,http://rustfs-node1:9000/data/rustfs4,http://rustfs-node2:9000/data/rustfs1,http://rustfs-node2:9000/data/rustfs2,http://rustfs-node2:9000/data/rustfs3,http://rustfs-node2:9000/data/rustfs4,http://rustfs-node3:9000/data/rustfs1,http://rustfs-node3:9000/data/rustfs2,http://rustfs-node3:9000/data/rustfs3,http://rustfs-node3:9000/data/rustfs4,http://rustfs-node4:9000/data/rustfs1,http://rustfs-node4:9000/data/rustfs2,http://rustfs-node4:9000/data/rustfs3,http://rustfs-node4:9000/data/rustfs4
+```
+
+## Using RUSTFS_CMD
+
+The `RUSTFS_CMD` environment variable provides a fallback when no command is specified:
+
+```yaml
+environment:
+  - RUSTFS_CMD=rustfs  # Default fallback command
+```
+
+This allows the entrypoint to execute the correct command when Docker doesn't provide one.
+
+## Testing the Deployment
+
+After starting the cluster, verify it's working:
+
+### Automated Testing
+
+Use the provided test script for comprehensive validation:
+
+```bash
+./test-deployment.sh
+```
+
+This script tests:
+
+- Container status (4/4 running)
+- Health checks (4/4 healthy)
+- API endpoints (4 ports)
+- Console endpoints (4 ports)
+- Inter-node connectivity
+- Data directory existence
+
+### Manual Testing
+
+For manual verification:
+
+```bash
+# 1. Check all containers are healthy
+docker-compose ps
+
+# 2. Test API endpoints
+for port in 9000 9010 9020 9030; do
+  echo "Testing port $port..."
+  curl -s http://localhost:${port}/health | jq '.'
+done
+
+# 3. Test console endpoints
+for port in 9001 9011 9021 9031; do
+  echo "Testing console port $port..."
+  curl -s http://localhost:${port}/health | jq '.'
+done
+
+# 4. Check inter-node connectivity
+docker exec rustfs-node1 nc -zv rustfs-node2 9000
+docker exec rustfs-node1 nc -zv rustfs-node3 9000
+docker exec rustfs-node1 nc -zv rustfs-node4 9000
+```
+
+## Troubleshooting
+
+### VolumeNotFound Error
+
+**Symptom:** Error message about `/data/rustfs0` not found.
+
+**Solution:** This example uses `/data/rustfs{1...4}` indexing to match the mounted Docker volumes. Ensure your
+`RUSTFS_VOLUMES` configuration starts at index 1, not 0.
+
+### Health Check Failures
+
+**Symptom:** Containers show as unhealthy.
+
+**Solutions:**
+
+1. Check if `nc` is available: `docker exec rustfs-node1 which nc`
+2. Use alternative health checks (curl/wget) as documented above
+3. Increase `start_period` if nodes need more time to initialize
+
+### Startup Timeouts
+
+**Symptom:** Services timeout waiting for peers.
+
+**Solutions:**
+
+1. Check logs: `docker-compose logs rustfs-node1`
+2. Verify network connectivity: `docker-compose exec rustfs-node1 ping rustfs-node2`
+3. Consider increasing the `sleep` duration in the `docker-compose.yml` `command` directive if a longer delay is needed.
+
+### Permission Issues
+
+**Symptom:** Cannot create directories or write data.
+
+**Solution:** Ensure volumes have correct permissions or set `RUSTFS_UID` and `RUSTFS_GID` environment variables.
+
+## Advanced Configuration
+
+### Custom Credentials
+
+Replace default credentials in production:
+
+```yaml
+environment:
+  - RUSTFS_ACCESS_KEY=your_access_key
+  - RUSTFS_SECRET_KEY=your_secret_key
+```
+
+### TLS Configuration
+
+Add TLS certificates:
+
+```yaml
+volumes:
+  - ./certs:/opt/tls:ro
+environment:
+  - RUSTFS_TLS_PATH=/opt/tls
+```
+
+### Resource Limits
+
+Add resource constraints:
+
+```yaml
+deploy:
+  resources:
+    limits:
+      cpus: '2'
+      memory: 4G
+    reservations:
+      cpus: '1'
+      memory: 2G
+```
+
+## See Also
+
+- [CHECKLIST.md](./CHECKLIST.md) - Step-by-step verification guide
+- [../../console-separation.md](../../console-separation.md) - Console & endpoint service separation guide
+- [../../../examples/docker-comprehensive.yml](../../../examples/docker-comprehensive.yml) - More deployment examples
+- [Issue #618](https://github.com/rustfs/rustfs/issues/618) - Original VolumeNotFound issue
+
+## References
+
+- RustFS Documentation: https://rustfs.io
+- Docker Compose Documentation: https://docs.docker.com/compose/
--- a/docs/examples/mnmd/docker-compose.yml
+++ b/docs/examples/mnmd/docker-compose.yml
@@ -0,0 +1,182 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MNMD (Multi-Node Multi-Drive) Docker Compose Example
+# 4 nodes x 4 drives configuration
+# This example demonstrates a complete, ready-to-use MNMD deployment
+# addressing startup coordination and VolumeNotFound issues.
+
+services:
+  rustfs-node1:
+    image: rustfs/rustfs:latest
+    container_name: rustfs-node1
+    hostname: rustfs-node1
+    environment:
+      # Use service names and correct disk indexing (1..4 to match mounted paths)
+      - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_CMD=rustfs
+    ports:
+      - "9000:9000"  # API endpoint
+      - "9001:9001"  # Console
+    volumes:
+      - node1-data1:/data/rustfs1
+      - node1-data2:/data/rustfs2
+      - node1-data3:/data/rustfs3
+      - node1-data4:/data/rustfs4
+    command: [ "sh", "-c", "sleep 3 && rustfs" ]
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh", "-c",
+          "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health"
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - rustfs-mnmd
+
+  rustfs-node2:
+    image: rustfs/rustfs:latest
+    container_name: rustfs-node2
+    hostname: rustfs-node2
+    environment:
+      - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_CMD=rustfs
+    ports:
+      - "9010:9000"  # API endpoint
+      - "9011:9001"  # Console
+    volumes:
+      - node2-data1:/data/rustfs1
+      - node2-data2:/data/rustfs2
+      - node2-data3:/data/rustfs3
+      - node2-data4:/data/rustfs4
+    command: [ "sh", "-c", "sleep 3 && rustfs" ]
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh", "-c",
+          "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health"
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - rustfs-mnmd
+
+  rustfs-node3:
+    image: rustfs/rustfs:latest
+    container_name: rustfs-node3
+    hostname: rustfs-node3
+    environment:
+      - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_CMD=rustfs
+    ports:
+      - "9020:9000"  # API endpoint
+      - "9021:9001"  # Console
+    volumes:
+      - node3-data1:/data/rustfs1
+      - node3-data2:/data/rustfs2
+      - node3-data3:/data/rustfs3
+      - node3-data4:/data/rustfs4
+    command: [ "sh", "-c", "sleep 3 && rustfs" ]
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh", "-c",
+          "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health"
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - rustfs-mnmd
+
+  rustfs-node4:
+    image: rustfs/rustfs:latest
+    container_name: rustfs-node4
+    hostname: rustfs-node4
+    environment:
+      - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
+      - RUSTFS_ADDRESS=0.0.0.0:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+      - RUSTFS_ACCESS_KEY=rustfsadmin
+      - RUSTFS_SECRET_KEY=rustfsadmin
+      - RUSTFS_CMD=rustfs
+    ports:
+      - "9030:9000"  # API endpoint
+      - "9031:9001"  # Console
+    volumes:
+      - node4-data1:/data/rustfs1
+      - node4-data2:/data/rustfs2
+      - node4-data3:/data/rustfs3
+      - node4-data4:/data/rustfs4
+    command: [ "sh", "-c", "sleep 3 && rustfs" ]
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "sh", "-c",
+          "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health"
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - rustfs-mnmd
+
+networks:
+  rustfs-mnmd:
+    driver: bridge
+
+volumes:
+  node1-data1:
+  node1-data2:
+  node1-data3:
+  node1-data4:
+  node2-data1:
+  node2-data2:
+  node2-data3:
+  node2-data4:
+  node3-data1:
+  node3-data2:
+  node3-data3:
+  node3-data4:
+  node4-data1:
+  node4-data2:
+  node4-data3:
+  node4-data4:
--- a/docs/examples/mnmd/test-deployment.sh
+++ b/docs/examples/mnmd/test-deployment.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# test-deployment.sh - Quick test script for MNMD deployment
+# Usage: ./test-deployment.sh
+
+set -e
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo "========================================="
+echo "RustFS MNMD Deployment Test"
+echo "========================================="
+echo ""
+
+# Test 1: Check if all containers are running
+echo "Test 1: Checking container status..."
+RUNNING=$(docker-compose ps | grep -c "Up" || echo "0")
+if [ "$RUNNING" -eq 4 ]; then
+  echo -e "${GREEN}✓ All 4 containers are running${NC}"
+else
+  echo -e "${RED}✗ Only $RUNNING/4 containers are running${NC}"
+  docker-compose ps
+  exit 1
+fi
+echo ""
+
+# Test 2: Check health status
+echo "Test 2: Checking health status..."
+HEALTHY=0
+for node in rustfs-node1 rustfs-node2 rustfs-node3 rustfs-node4; do
+  STATUS=$(docker inspect "$node" --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
+  if [ "$STATUS" = "healthy" ]; then
+    echo -e "  ${GREEN}✓ $node is healthy${NC}"
+    HEALTHY=$((HEALTHY + 1))
+  elif [ "$STATUS" = "starting" ]; then
+    echo -e "  ${YELLOW}⚠ $node is starting (wait a moment)${NC}"
+  else
+    echo -e "  ${RED}✗ $node status: $STATUS${NC}"
+  fi
+done
+
+if [ "$HEALTHY" -eq 4 ]; then
+  echo -e "${GREEN}✓ All containers are healthy${NC}"
+elif [ "$HEALTHY" -gt 0 ]; then
+  echo -e "${YELLOW}⚠ $HEALTHY/4 containers are healthy (some may still be starting)${NC}"
+else
+  echo -e "${RED}✗ No containers are healthy${NC}"
+  exit 1
+fi
+echo ""
+
+# Test 3: Check API endpoints
+echo "Test 3: Testing API endpoints..."
+PORTS=(9000 9010 9020 9030)
+API_SUCCESS=0
+for port in "${PORTS[@]}"; do
+  if curl -sf http://localhost:${port}/health >/dev/null 2>&1; then
+    echo -e "  ${GREEN}✓ API on port $port is responding${NC}"
+    API_SUCCESS=$((API_SUCCESS + 1))
+  else
+    echo -e "  ${RED}✗ API on port $port is not responding${NC}"
+  fi
+done
+
+if [ "$API_SUCCESS" -eq 4 ]; then
+  echo -e "${GREEN}✓ All API endpoints are working${NC}"
+else
+  echo -e "${YELLOW}⚠ $API_SUCCESS/4 API endpoints are working${NC}"
+fi
+echo ""
+
+# Test 4: Check Console endpoints
+echo "Test 4: Testing Console endpoints..."
+CONSOLE_PORTS=(9001 9011 9021 9031)
+CONSOLE_SUCCESS=0
+for port in "${CONSOLE_PORTS[@]}"; do
+  if curl -sf http://localhost:${port}/health >/dev/null 2>&1; then
+    echo -e "  ${GREEN}✓ Console on port $port is responding${NC}"
+    CONSOLE_SUCCESS=$((CONSOLE_SUCCESS + 1))
+  else
+    echo -e "  ${RED}✗ Console on port $port is not responding${NC}"
+  fi
+done
+
+if [ "$CONSOLE_SUCCESS" -eq 4 ]; then
+  echo -e "${GREEN}✓ All Console endpoints are working${NC}"
+else
+  echo -e "${YELLOW}⚠ $CONSOLE_SUCCESS/4 Console endpoints are working${NC}"
+fi
+echo ""
+
+# Test 5: Check inter-node connectivity
+echo "Test 5: Testing inter-node connectivity..."
+CONN_SUCCESS=0
+for node in rustfs-node2 rustfs-node3 rustfs-node4; do
+  if docker exec rustfs-node1 nc -z "$node" 9000 2>/dev/null; then
+    echo -e "  ${GREEN}✓ node1 → $node connection OK${NC}"
+    CONN_SUCCESS=$((CONN_SUCCESS + 1))
+  else
+    echo -e "  ${RED}✗ node1 → $node connection failed${NC}"
+  fi
+done
+
+if [ "$CONN_SUCCESS" -eq 3 ]; then
+  echo -e "${GREEN}✓ All inter-node connections are working${NC}"
+else
+  echo -e "${YELLOW}⚠ $CONN_SUCCESS/3 inter-node connections are working${NC}"
+fi
+echo ""
+
+# Test 6: Verify data directories
+echo "Test 6: Verifying data directories..."
+DIR_SUCCESS=0
+for i in {1..4}; do
+  if docker exec rustfs-node1 test -d "/data/rustfs${i}"; then
+    DIR_SUCCESS=$((DIR_SUCCESS + 1))
+  else
+    echo -e "  ${RED}✗ /data/rustfs${i} not found in node1${NC}"
+  fi
+done
+
+if [ "$DIR_SUCCESS" -eq 4 ]; then
+  echo -e "${GREEN}✓ All data directories exist${NC}"
+else
+  echo -e "${RED}✗ Only $DIR_SUCCESS/4 data directories exist${NC}"
+fi
+echo ""
+
+# Summary
+echo "========================================="
+echo "Test Summary"
+echo "========================================="
+echo "Containers running: $RUNNING/4"
+echo "Healthy containers: $HEALTHY/4"
+echo "API endpoints: $API_SUCCESS/4"
+echo "Console endpoints: $CONSOLE_SUCCESS/4"
+echo "Inter-node connections: $CONN_SUCCESS/3"
+echo "Data directories: $DIR_SUCCESS/4"
+echo ""
+
+TOTAL=$((RUNNING + HEALTHY + API_SUCCESS + CONSOLE_SUCCESS + CONN_SUCCESS + DIR_SUCCESS))
+MAX_SCORE=23
+
+if [ "$TOTAL" -eq "$MAX_SCORE" ]; then
+  echo -e "${GREEN}✓ All tests passed! Deployment is working correctly.${NC}"
+  exit 0
+elif [ "$TOTAL" -ge 20 ]; then
+  echo -e "${YELLOW}⚠ Most tests passed. Some components may still be starting up.${NC}"
+  echo "  Try running this script again in a few moments."
+  exit 0
+else
+  echo -e "${RED}✗ Some tests failed. Check the output above and logs for details.${NC}"
+  echo "  Run 'docker-compose logs' for more information."
+  exit 1
+fi
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -70,7 +70,6 @@ clap = { workspace = true }
 datafusion = { workspace = true }
 const-str = { workspace = true }
 futures.workspace = true
-hashbrown = { workspace = true }
 hyper.workspace = true
 hyper-util.workspace = true
 http.workspace = true
@@ -79,7 +78,6 @@ matchit = { workspace = true }
 md5.workspace = true
 mime_guess = { workspace = true }
 opentelemetry = { workspace = true }
-percent-encoding = { workspace = true }
 pin-project-lite.workspace = true
 reqwest = { workspace = true }
 rustls = { workspace = true }