From ad990197497482ba7727d16db7065c6bfd4053ff Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Oct 2025 13:15:14 +0800 Subject: [PATCH] Add complete MNMD Docker deployment example with startup coordination and VolumeNotFound fix (#642) * Initial plan * Add MNMD Docker deployment example with 4 nodes x 4 drives - Create docs/examples/mnmd/ directory structure - Add docker-compose.yml with proper disk indexing (1..4) - Add wait-and-start.sh for startup coordination - Add README.md with usage instructions and alternatives - Add CHECKLIST.md with step-by-step verification - Fixes VolumeNotFound issue by using correct volume paths - Implements health checks and startup ordering - Uses service names for stable inter-node addressing Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add docs/examples README as index for deployment examples Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add automated test script for MNMD deployment - Add test-deployment.sh with comprehensive validation - Test container status, health, endpoints, connectivity - Update README to reference test script - Make script executable Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * improve code * improve code * improve dep crates `cargo shear --fix` * upgrade aws-sdk-s3 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme --- Cargo.lock | 124 +++------- Cargo.toml | 11 +- crates/ecstore/Cargo.toml | 5 - crates/utils/Cargo.toml | 4 +- docs/examples/README.md | 60 +++++ docs/examples/mnmd/CHECKLIST.md | 329 ++++++++++++++++++++++++++ docs/examples/mnmd/README.md | 268 +++++++++++++++++++++ docs/examples/mnmd/docker-compose.yml | 182 ++++++++++++++ docs/examples/mnmd/test-deployment.sh | 172 ++++++++++++++ rustfs/Cargo.toml | 2 - 10 files changed, 1050 insertions(+), 107 deletions(-) create mode 100644 docs/examples/README.md create mode 100644 docs/examples/mnmd/CHECKLIST.md create mode 100644 docs/examples/mnmd/README.md create mode 100644 docs/examples/mnmd/docker-compose.yml create mode 100755 docs/examples/mnmd/test-deployment.sh diff --git a/Cargo.lock b/Cargo.lock index 8ba0e6e6..603acb41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1372,9 +1372,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.40" +version = "1.2.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d05d92f4b1fd76aad469d46cdd858ca761576082cd37df81416691e50199fb" +checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" dependencies = [ "find-msvc-tools", "jobserver", @@ -2924,7 +2924,7 @@ dependencies = [ "flatbuffers", "futures", "http 1.3.1", - "md5 0.8.0", + "md5", "rand 0.9.2", "reqwest", "rmp-serde", @@ -3197,9 +3197,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0399f9d26e5191ce32c498bebd31e7a3ceabc2745f0ac54af3f335126c3f24b3" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" [[package]] name = "findshlibs" @@ -4561,12 +4561,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - [[package]] name = "md5" version = "0.8.0" @@ -4814,11 +4808,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.50.1" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5364,12 +5358,12 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ "base64 0.22.1", - "serde", + "serde_core", ] [[package]] @@ -5642,7 +5636,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.6", + "toml_edit 0.23.7", ] [[package]] @@ -6084,9 +6078,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.3" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" +checksum = "4a52d8d02cacdb176ef4678de6c052efb4b3da14b78e4db683a4252762be5433" dependencies = [ "aho-corasick", "memchr", @@ -6096,9 +6090,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" +checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6" dependencies = [ "aho-corasick", "memchr", @@ -6107,15 +6101,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" [[package]] name = "reqwest" @@ -6384,18 +6378,16 @@ dependencies = [ "flatbuffers", "futures", "futures-util", - "hashbrown 0.16.0", "http 1.3.1", "http-body 1.0.1", "hyper 1.7.0", "hyper-util", "libsystemd", "matchit", - "md5 0.8.0", + "md5", "mimalloc", "mime_guess", "opentelemetry", - "percent-encoding", "pin-project-lite", "pprof", "reqwest", @@ -6572,7 +6564,6 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-s3", - "aws-smithy-runtime-api", "aws-smithy-types", "base64 0.22.1", "byteorder", @@ -6583,7 +6574,6 @@ dependencies = [ "enumset", "flatbuffers", "futures", - "futures-util", "glob", "hex-simd", "hmac 0.12.1", @@ -6596,7 +6586,6 @@ dependencies = [ "moka", "nix 0.30.1", "num_cpus", - "once_cell", "parking_lot", "path-absolutize", "pin-project-lite", @@ -6616,7 +6605,6 @@ dependencies = [ "rustfs-policy", "rustfs-protos", "rustfs-rio", - "rustfs-rsc", "rustfs-signer", "rustfs-utils", "rustfs-workers", @@ -6633,7 +6621,6 @@ dependencies = [ "thiserror 2.0.17", "time", "tokio", - "tokio-stream", "tokio-util", "tonic", "tower", @@ -6700,7 +6687,7 @@ dependencies = [ "base64 0.22.1", "chacha20poly1305", "chrono", - "md5 0.8.0", + "md5", "moka", "once_cell", "rand 0.9.2", @@ -6886,33 +6873,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "rustfs-rsc" -version = "2025.506.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8229535cdd7a9d1f5757bd7b588f342e6ea984d66c6b1f6350f9de85d3ce9c25" -dependencies = [ - "async-stream", - "base64 0.22.1", - "bytes", - "chrono", - "crc32fast", - "futures", - "futures-core", - "futures-util", - "hex", - "hmac 0.12.1", - "hyper 1.7.0", - "md5 0.7.0", - "once_cell", - "regex", - "reqwest", - "serde", - "serde-xml-rs", - "sha2 0.10.9", - "urlencoding", -] - [[package]] name = "rustfs-s3select-api" version = "0.0.5" @@ -7478,18 +7438,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde-xml-rs" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782" -dependencies = [ - "log", - "serde", - "thiserror 1.0.69", - "xml-rs", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -8536,9 +8484,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ "serde_core", ] @@ -8559,21 +8507,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.6" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ "indexmap", - "toml_datetime 0.7.2", + "toml_datetime 0.7.3", "toml_parser", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ "winnow", ] @@ -8848,9 +8796,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "tz-rs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1450bf2b99397e72070e7935c89facaa80092ac812502200375f1f7d33c71a1" +checksum = "14eff19b8dc1ace5bf7e4d920b2628ae3837f422ff42210cb1567cbf68b5accf" [[package]] name = "tzdb" @@ -9711,12 +9659,6 @@ dependencies = [ "rustix 1.1.2", ] -[[package]] -name = "xml-rs" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7" - [[package]] name = "xmlparser" version = "0.13.6" @@ -9864,9 +9806,9 @@ dependencies = [ [[package]] name = "zip" -version = "5.1.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" dependencies = [ "aes", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index 521217a4..aa1e811a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,8 +102,7 @@ atomic_enum = "0.3.0" aws-config = { version = "1.8.8" } aws-credential-types = { version = "1.2.8" } aws-smithy-types = { version = "1.3.3" } -aws-smithy-runtime-api = { version = "1.9.1" } -aws-sdk-s3 = { version = "1.106.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } +aws-sdk-s3 = { version = "1.108.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } axum = "0.8.6" axum-extra = "0.10.3" axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false } @@ -163,7 +162,7 @@ mime_guess = "2.0.5" moka = { version = "0.12.11", features = ["future"] } netif = "0.1.6" nix = { version = "0.30.1", features = ["fs"] } -nu-ansi-term = "0.50.1" +nu-ansi-term = "0.50.3" num_cpus = { version = "1.17.0" } nvml-wrapper = "0.11.0" object_store = "0.12.4" @@ -187,7 +186,6 @@ path-absolutize = "3.1.1" path-clean = "1.0.1" blake3 = { version = "1.8.2" } pbkdf2 = "0.12.2" -percent-encoding = "2.3.2" pin-project-lite = "0.2.16" prost = "0.14.1" pretty_assertions = "1.4.1" @@ -196,7 +194,7 @@ rand = "0.9.2" rayon = "1.11.0" rdkafka = { version = "0.38.0", features = ["tokio"] } reed-solomon-simd = { version = "3.0.1" } -regex = { version = "1.11.3" } +regex = { version = "1.12.1" } reqwest = { version = "0.12.23", default-features = false, features = [ "rustls-tls-webpki-roots", "charset", @@ -213,7 +211,6 @@ rsa = "0.9.8" rumqttc = { version = "0.25.0" } rust-embed = { version = "8.7.2" } rustc-hash = { version = "2.1.1" } -rustfs-rsc = "2025.506.1" rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false } rustls-pki-types = "1.12.0" rustls-pemfile = "2.2.0" @@ -277,7 +274,7 @@ wildmatch = { version = "2.5.0", features = ["serde"] } zeroize = { version = "1.8.2", features = ["derive"] } winapi = { version = "0.3.9" } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } -zip = "5.1.1" +zip = "6.0.0" zstd = "0.13.3" diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 0c436eb4..399a7ff6 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -75,7 +75,6 @@ hyper-util.workspace = true hyper-rustls.workspace = true rustls.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal"] } -tokio-stream = { workspace = true } tonic.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } tower.workspace = true @@ -89,8 +88,6 @@ rustfs-madmin.workspace = true rustfs-workers.workspace = true reqwest = { workspace = true } aws-sdk-s3 = { workspace = true } -once_cell = { workspace = true } -rustfs-rsc = { workspace = true } urlencoding = { workspace = true } smallvec = { workspace = true } shadow-rs.workspace = true @@ -99,13 +96,11 @@ rustfs-utils = { workspace = true, features = ["full"] } rustfs-rio.workspace = true rustfs-signer.workspace = true rustfs-checksums.workspace = true -futures-util.workspace = true async-recursion.workspace = true aws-credential-types = { workspace = true } aws-smithy-types = { workspace = true } parking_lot = { workspace = true } moka = { workspace = true } -aws-smithy-runtime-api = { workspace = true } [target.'cfg(not(windows))'.dependencies] nix = { workspace = true } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index bade90c4..6d9b28eb 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -29,7 +29,7 @@ base64-simd = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } bytes = { workspace = true, optional = true } -crc32fast = { workspace = true } +crc32fast = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } futures = { workspace = true, optional = true } hashbrown = { workspace = true, optional = true } @@ -89,7 +89,7 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex", "dep:rand"] crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"] -hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd"] +hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd", "dep:crc32fast"] os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities integration = [] # integration test features sys = ["dep:sysinfo"] # system information features diff --git a/docs/examples/README.md b/docs/examples/README.md new file mode 100644 index 00000000..4224bb5b --- /dev/null +++ b/docs/examples/README.md @@ -0,0 +1,60 @@ +# RustFS Deployment Examples + +This directory contains practical deployment examples and configurations for RustFS. + +## Available Examples + +### [MNMD (Multi-Node Multi-Drive)](./mnmd/) + +Complete Docker Compose example for deploying RustFS in a 4-node, 4-drive-per-node configuration. + +**Features:** +- Proper disk indexing (1..4) to avoid VolumeNotFound errors +- Startup coordination via `wait-and-start.sh` script +- Service discovery using Docker service names +- Health checks with alternatives for different base images +- Comprehensive documentation and verification checklist + +**Use Case:** Production-ready multi-node deployment for high availability and performance. + +**Quick Start:** +```bash +cd docs/examples/mnmd +docker-compose up -d +``` + +**See also:** +- [MNMD README](./mnmd/README.md) - Detailed usage guide +- [MNMD CHECKLIST](./mnmd/CHECKLIST.md) - Step-by-step verification + +## Other Deployment Examples + +For additional deployment examples, see: +- [`examples/`](/examples/) - Root-level examples directory with: + - `docker-quickstart.sh` - Quick start script for basic deployments + - `enhanced-docker-deployment.sh` - Advanced deployment scenarios + - `docker-comprehensive.yml` - Docker Compose with multiple profiles +- [`.docker/compose/`](/.docker/compose/) - Docker Compose configurations: + - `docker-compose.cluster.yaml` - Basic cluster setup + - `docker-compose.observability.yaml` - Observability stack integration + +## Related Documentation + +- [Console & Endpoint Service Separation](../console-separation.md) +- [Environment Variables](../ENVIRONMENT_VARIABLES.md) +- [Performance Testing](../PERFORMANCE_TESTING.md) + +## Contributing + +When adding new examples: +1. Create a dedicated subdirectory under `docs/examples/` +2. Include a comprehensive README.md +3. Provide working configuration files +4. Add verification steps or checklists +5. Document common issues and troubleshooting + +## Support + +For issues or questions: +- GitHub Issues: https://github.com/rustfs/rustfs/issues +- Documentation: https://rustfs.io diff --git a/docs/examples/mnmd/CHECKLIST.md b/docs/examples/mnmd/CHECKLIST.md new file mode 100644 index 00000000..0a08f156 --- /dev/null +++ b/docs/examples/mnmd/CHECKLIST.md @@ -0,0 +1,329 @@ +# MNMD Deployment Checklist + +This checklist provides step-by-step verification for deploying RustFS in MNMD (Multi-Node Multi-Drive) mode using +Docker. + +## Pre-Deployment Checks + +### 1. System Requirements + +- [ ] Docker Engine 20.10+ installed +- [ ] Docker Compose 2.0+ installed +- [ ] At least 8GB RAM available +- [ ] At least 40GB disk space available (for 4 nodes × 4 volumes) + +Verify with: + +```bash +docker --version +docker-compose --version +free -h +df -h +``` + +### 2. File System Checks + +- [ ] Using XFS, ext4, or another suitable filesystem (not NFS for production) +- [ ] File system supports extended attributes + +Verify with: + +```bash +df -T | grep -E '(xfs|ext4)' +``` + +### 3. Permissions and SELinux + +- [ ] Current user is in `docker` group or can run `sudo docker` +- [ ] SELinux is properly configured (if enabled) + +Verify with: + +```bash +groups | grep docker +getenforce # If enabled, should show "Permissive" or "Enforcing" with proper policies +``` + +### 4. Network Configuration + +- [ ] Ports 9000-9031 are available +- [ ] No firewall blocking Docker bridge network + +Verify with: + +```bash +# Check if ports are free +netstat -tuln | grep -E ':(9000|9001|9010|9011|9020|9021|9030|9031)' +# Should return nothing if ports are free +``` + +### 5. Files Present + +- [ ] `docker-compose.yml` exists in current directory + +Verify with: + +```bash +cd docs/examples/mnmd +ls -la +chmod +x wait-and-start.sh # If needed +``` + +## Deployment Steps + +### 1. Start the Cluster + +- [ ] Navigate to the example directory +- [ ] Pull the latest RustFS image +- [ ] Start the cluster + +```bash +cd docs/examples/mnmd +docker-compose pull +docker-compose up -d +``` + +### 2. Monitor Startup + +- [ ] Watch container logs during startup +- [ ] Verify no VolumeNotFound errors +- [ ] Check that peer discovery completes + +```bash +# Watch all logs +docker-compose logs -f + +# Watch specific node +docker-compose logs -f rustfs-node1 + +# Look for successful startup messages +docker-compose logs | grep -i "ready\|listening\|started" +``` + +### 3. Verify Container Status + +- [ ] All 4 containers are running +- [ ] All 4 containers show as healthy + +```bash +docker-compose ps + +# Expected output: 4 containers in "Up" state with "healthy" status +``` + +### 4. Check Health Endpoints + +- [ ] API health endpoints respond on all nodes +- [ ] Console health endpoints respond on all nodes + +```bash +# Test API endpoints +curl http://localhost:9000/health +curl http://localhost:9010/health +curl http://localhost:9020/health +curl http://localhost:9030/health + +# Test Console endpoints +curl http://localhost:9001/health +curl http://localhost:9011/health +curl http://localhost:9021/health +curl http://localhost:9031/health + +# All should return successful health status +``` + +## Post-Deployment Verification + +### 1. In-Container Checks + +- [ ] Data directories exist +- [ ] Directories have correct permissions +- [ ] RustFS process is running + +```bash +# Check node1 +docker exec rustfs-node1 ls -la /data/ +docker exec rustfs-node1 ps aux | grep rustfs + +# Verify all 4 data directories exist +docker exec rustfs-node1 ls -d /data/rustfs{1..4} +``` + +### 2. DNS and Network Validation + +- [ ] Service names resolve correctly +- [ ] Inter-node connectivity works + +```bash +# DNS resolution test +docker exec rustfs-node1 nslookup rustfs-node2 +docker exec rustfs-node1 nslookup rustfs-node3 +docker exec rustfs-node1 nslookup rustfs-node4 + +# Connectivity test (using nc if available) +docker exec rustfs-node1 nc -zv rustfs-node2 9000 +docker exec rustfs-node1 nc -zv rustfs-node3 9000 +docker exec rustfs-node1 nc -zv rustfs-node4 9000 + +# Or using telnet/curl +docker exec rustfs-node1 curl -v http://rustfs-node2:9000/health +``` + +### 3. Volume Configuration Validation + +- [ ] RUSTFS_VOLUMES environment variable is correct +- [ ] All 16 endpoints are configured (4 nodes × 4 drives) + +```bash +# Check environment variable +docker exec rustfs-node1 env | grep RUSTFS_VOLUMES + +# Expected output: +# RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} +``` + +### 4. Cluster Functionality + +- [ ] Can list buckets via API +- [ ] Can create a bucket +- [ ] Can upload an object +- [ ] Can download an object + +```bash +# Configure AWS CLI or s3cmd +export AWS_ACCESS_KEY_ID=rustfsadmin +export AWS_SECRET_ACCESS_KEY=rustfsadmin + +# Using AWS CLI (if installed) +aws --endpoint-url http://localhost:9000 s3 mb s3://test-bucket +aws --endpoint-url http://localhost:9000 s3 ls +echo "test content" > test.txt +aws --endpoint-url http://localhost:9000 s3 cp test.txt s3://test-bucket/ +aws --endpoint-url http://localhost:9000 s3 ls s3://test-bucket/ +aws --endpoint-url http://localhost:9000 s3 cp s3://test-bucket/test.txt downloaded.txt +cat downloaded.txt + +# Or using curl +curl -X PUT http://localhost:9000/test-bucket \ + -H "Host: localhost:9000" \ + --user rustfsadmin:rustfsadmin +``` + +### 5. Healthcheck Verification + +- [ ] Docker reports all services as healthy +- [ ] Healthcheck scripts work in containers + +```bash +# Check Docker health status +docker inspect rustfs-node1 --format='{{.State.Health.Status}}' +docker inspect rustfs-node2 --format='{{.State.Health.Status}}' +docker inspect rustfs-node3 --format='{{.State.Health.Status}}' +docker inspect rustfs-node4 --format='{{.State.Health.Status}}' + +# All should return "healthy" + +# Test healthcheck command manually +docker exec rustfs-node1 nc -z localhost 9000 +echo $? # Should be 0 +``` + +## Troubleshooting Checks + +### If VolumeNotFound Error Occurs + +- [ ] Verify volume indexing starts at 1, not 0 +- [ ] Check that RUSTFS_VOLUMES matches mounted paths +- [ ] Ensure all /data/rustfs{1..4} directories exist + +```bash +# Check mounted volumes +docker inspect rustfs-node1 | jq '.[].Mounts' + +# Verify directories in container +docker exec rustfs-node1 ls -la /data/ +``` + +### If Healthcheck Fails + +- [ ] Check if `nc` is available in the image +- [ ] Try alternative healthcheck (curl/wget) +- [ ] Increase `start_period` in docker-compose.yml + +```bash +# Check if nc is available +docker exec rustfs-node1 which nc + +# Test healthcheck manually +docker exec rustfs-node1 nc -z localhost 9000 + +# Check logs for errors +docker-compose logs rustfs-node1 | grep -i error +``` + +### If Startup Takes Too Long + +- [ ] Check peer discovery timeout in logs +- [ ] Verify network connectivity between nodes +- [ ] Consider increasing timeout in wait-and-start.sh + +```bash +# Check startup logs +docker-compose logs rustfs-node1 | grep -i "waiting\|peer\|timeout" + +# Check network +docker network inspect mnmd_rustfs-mnmd +``` + +### If Containers Crash or Restart + +- [ ] Review container logs +- [ ] Check resource usage (CPU/Memory) +- [ ] Verify no port conflicts + +```bash +# View last crash logs +docker-compose logs --tail=100 rustfs-node1 + +# Check resource usage +docker stats --no-stream + +# Check restart count +docker-compose ps +``` + +## Cleanup Checklist + +When done testing: + +- [ ] Stop the cluster: `docker-compose down` +- [ ] Remove volumes (optional, destroys data): `docker-compose down -v` +- [ ] Clean up dangling images: `docker image prune` +- [ ] Verify ports are released: `netstat -tuln | grep -E ':(9000|9001|9010|9011|9020|9021|9030|9031)'` + +## Production Deployment Additional Checks + +Before deploying to production: + +- [ ] Change default credentials (RUSTFS_ACCESS_KEY, RUSTFS_SECRET_KEY) +- [ ] Configure TLS certificates +- [ ] Set up proper logging and monitoring +- [ ] Configure backups for volumes +- [ ] Review and adjust resource limits +- [ ] Set up external load balancer (if needed) +- [ ] Document disaster recovery procedures +- [ ] Test failover scenarios +- [ ] Verify data persistence after container restart + +## Summary + +This checklist ensures: + +- ✓ Correct disk indexing (1..4 instead of 0..3) +- ✓ Proper startup coordination via wait-and-start.sh +- ✓ Service discovery via Docker service names +- ✓ Health checks function correctly +- ✓ All 16 endpoints (4 nodes × 4 drives) are operational +- ✓ No VolumeNotFound errors occur + +For more details, see [README.md](./README.md) in this directory. diff --git a/docs/examples/mnmd/README.md b/docs/examples/mnmd/README.md new file mode 100644 index 00000000..c8c49293 --- /dev/null +++ b/docs/examples/mnmd/README.md @@ -0,0 +1,268 @@ +# RustFS MNMD (Multi-Node Multi-Drive) Docker Example + +This directory contains a complete, ready-to-use MNMD deployment example for RustFS with 4 nodes and 4 drives per node ( +4x4 configuration). + +## Overview + +This example addresses common deployment issues including: + +- **VolumeNotFound errors** - Fixed by using correct disk indexing (`/data/rustfs{1...4}` instead of + `/data/rustfs{0...3}`) +- **Startup race conditions** - Solved with a simple `sleep` command in each service. +- **Service discovery** - Uses Docker service names (`rustfs-node{1..4}`) instead of hard-coded IPs +- **Health checks** - Implements proper health monitoring with `nc` (with alternatives documented) + +## Quick Start + +From this directory (`docs/examples/mnmd`), run: + +```bash +# Start the cluster +docker-compose up -d + +# Check the status +docker-compose ps + +# View logs +docker-compose logs -f + +# Test the deployment +curl http://localhost:9000/health +curl http://localhost:9001/health + +# Run comprehensive tests +./test-deployment.sh + +# Stop the cluster +docker-compose down + +# Clean up volumes (WARNING: deletes all data) +docker-compose down -v +``` + +## Configuration Details + +### Volume Configuration + +The example uses the following volume configuration: + +```bash +RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} +``` + +This expands to 16 endpoints (4 nodes × 4 drives): + +- Node 1: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4` +- Node 2: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4` +- Node 3: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4` +- Node 4: `/data/rustfs1`, `/data/rustfs2`, `/data/rustfs3`, `/data/rustfs4` + +**Important:** Disk indexing starts at 1 to match the mounted paths (`/data/rustfs1..4`). + +### Port Mappings + +| Node | API Port | Console Port | +|-------|----------|--------------| +| node1 | 9000 | 9001 | +| node2 | 9010 | 9011 | +| node3 | 9020 | 9021 | +| node4 | 9030 | 9031 | + +### Startup Coordination + +To prevent race conditions during startup where nodes might not find each other, a simple `sleep 3` command is added to +each service's command. This provides a brief delay, allowing the network and other services to initialize before RustFS +starts. For more complex scenarios, a more robust health-check dependency or an external entrypoint script might be +required. + +### Health Checks + +Default health check using `nc` (netcat): + +```yaml +healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 9000 || exit 1" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s +``` + +#### Alternative Health Checks + +If your base image lacks `nc`, use one of these alternatives: + +**Using curl:** + +```yaml +healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:9000/health || exit 1" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s +``` + +**Using wget:** + +```yaml +healthcheck: + test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9000/health || exit 1" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s +``` + +### Brace Expansion Alternatives + +If your Docker Compose runtime doesn't support brace expansion (`{1...4}`), replace with explicit endpoints: + +```yaml +environment: + - RUSTFS_VOLUMES=http://rustfs-node1:9000/data/rustfs1,http://rustfs-node1:9000/data/rustfs2,http://rustfs-node1:9000/data/rustfs3,http://rustfs-node1:9000/data/rustfs4,http://rustfs-node2:9000/data/rustfs1,http://rustfs-node2:9000/data/rustfs2,http://rustfs-node2:9000/data/rustfs3,http://rustfs-node2:9000/data/rustfs4,http://rustfs-node3:9000/data/rustfs1,http://rustfs-node3:9000/data/rustfs2,http://rustfs-node3:9000/data/rustfs3,http://rustfs-node3:9000/data/rustfs4,http://rustfs-node4:9000/data/rustfs1,http://rustfs-node4:9000/data/rustfs2,http://rustfs-node4:9000/data/rustfs3,http://rustfs-node4:9000/data/rustfs4 +``` + +## Using RUSTFS_CMD + +The `RUSTFS_CMD` environment variable provides a fallback when no command is specified: + +```yaml +environment: + - RUSTFS_CMD=rustfs # Default fallback command +``` + +This allows the entrypoint to execute the correct command when Docker doesn't provide one. + +## Testing the Deployment + +After starting the cluster, verify it's working: + +### Automated Testing + +Use the provided test script for comprehensive validation: + +```bash +./test-deployment.sh +``` + +This script tests: + +- Container status (4/4 running) +- Health checks (4/4 healthy) +- API endpoints (4 ports) +- Console endpoints (4 ports) +- Inter-node connectivity +- Data directory existence + +### Manual Testing + +For manual verification: + +```bash +# 1. Check all containers are healthy +docker-compose ps + +# 2. Test API endpoints +for port in 9000 9010 9020 9030; do + echo "Testing port $port..." + curl -s http://localhost:${port}/health | jq '.' +done + +# 3. Test console endpoints +for port in 9001 9011 9021 9031; do + echo "Testing console port $port..." + curl -s http://localhost:${port}/health | jq '.' +done + +# 4. Check inter-node connectivity +docker exec rustfs-node1 nc -zv rustfs-node2 9000 +docker exec rustfs-node1 nc -zv rustfs-node3 9000 +docker exec rustfs-node1 nc -zv rustfs-node4 9000 +``` + +## Troubleshooting + +### VolumeNotFound Error + +**Symptom:** Error message about `/data/rustfs0` not found. + +**Solution:** This example uses `/data/rustfs{1...4}` indexing to match the mounted Docker volumes. Ensure your +`RUSTFS_VOLUMES` configuration starts at index 1, not 0. + +### Health Check Failures + +**Symptom:** Containers show as unhealthy. + +**Solutions:** + +1. Check if `nc` is available: `docker exec rustfs-node1 which nc` +2. Use alternative health checks (curl/wget) as documented above +3. Increase `start_period` if nodes need more time to initialize + +### Startup Timeouts + +**Symptom:** Services timeout waiting for peers. + +**Solutions:** + +1. Check logs: `docker-compose logs rustfs-node1` +2. Verify network connectivity: `docker-compose exec rustfs-node1 ping rustfs-node2` +3. Consider increasing the `sleep` duration in the `docker-compose.yml` `command` directive if a longer delay is needed. + +### Permission Issues + +**Symptom:** Cannot create directories or write data. + +**Solution:** Ensure volumes have correct permissions or set `RUSTFS_UID` and `RUSTFS_GID` environment variables. + +## Advanced Configuration + +### Custom Credentials + +Replace default credentials in production: + +```yaml +environment: + - RUSTFS_ACCESS_KEY=your_access_key + - RUSTFS_SECRET_KEY=your_secret_key +``` + +### TLS Configuration + +Add TLS certificates: + +```yaml +volumes: + - ./certs:/opt/tls:ro +environment: + - RUSTFS_TLS_PATH=/opt/tls +``` + +### Resource Limits + +Add resource constraints: + +```yaml +deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '1' + memory: 2G +``` + +## See Also + +- [CHECKLIST.md](./CHECKLIST.md) - Step-by-step verification guide +- [../../console-separation.md](../../console-separation.md) - Console & endpoint service separation guide +- [../../../examples/docker-comprehensive.yml](../../../examples/docker-comprehensive.yml) - More deployment examples +- [Issue #618](https://github.com/rustfs/rustfs/issues/618) - Original VolumeNotFound issue + +## References + +- RustFS Documentation: https://rustfs.io +- Docker Compose Documentation: https://docs.docker.com/compose/ \ No newline at end of file diff --git a/docs/examples/mnmd/docker-compose.yml b/docs/examples/mnmd/docker-compose.yml new file mode 100644 index 00000000..c7634a51 --- /dev/null +++ b/docs/examples/mnmd/docker-compose.yml @@ -0,0 +1,182 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MNMD (Multi-Node Multi-Drive) Docker Compose Example +# 4 nodes x 4 drives configuration +# This example demonstrates a complete, ready-to-use MNMD deployment +# addressing startup coordination and VolumeNotFound issues. + +services: + rustfs-node1: + image: rustfs/rustfs:latest + container_name: rustfs-node1 + hostname: rustfs-node1 + environment: + # Use service names and correct disk indexing (1..4 to match mounted paths) + - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_ACCESS_KEY=rustfsadmin + - RUSTFS_SECRET_KEY=rustfsadmin + - RUSTFS_CMD=rustfs + ports: + - "9000:9000" # API endpoint + - "9001:9001" # Console + volumes: + - node1-data1:/data/rustfs1 + - node1-data2:/data/rustfs2 + - node1-data3:/data/rustfs3 + - node1-data4:/data/rustfs4 + command: [ "sh", "-c", "sleep 3 && rustfs" ] + healthcheck: + test: + [ + "CMD", + "sh", "-c", + "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health" + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - rustfs-mnmd + + rustfs-node2: + image: rustfs/rustfs:latest + container_name: rustfs-node2 + hostname: rustfs-node2 + environment: + - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_ACCESS_KEY=rustfsadmin + - RUSTFS_SECRET_KEY=rustfsadmin + - RUSTFS_CMD=rustfs + ports: + - "9010:9000" # API endpoint + - "9011:9001" # Console + volumes: + - node2-data1:/data/rustfs1 + - node2-data2:/data/rustfs2 + - node2-data3:/data/rustfs3 + - node2-data4:/data/rustfs4 + command: [ "sh", "-c", "sleep 3 && rustfs" ] + healthcheck: + test: + [ + "CMD", + "sh", "-c", + "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health" + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - rustfs-mnmd + + rustfs-node3: + image: rustfs/rustfs:latest + container_name: rustfs-node3 + hostname: rustfs-node3 + environment: + - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_ACCESS_KEY=rustfsadmin + - RUSTFS_SECRET_KEY=rustfsadmin + - RUSTFS_CMD=rustfs + ports: + - "9020:9000" # API endpoint + - "9021:9001" # Console + volumes: + - node3-data1:/data/rustfs1 + - node3-data2:/data/rustfs2 + - node3-data3:/data/rustfs3 + - node3-data4:/data/rustfs4 + command: [ "sh", "-c", "sleep 3 && rustfs" ] + healthcheck: + test: + [ + "CMD", + "sh", "-c", + "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health" + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - rustfs-mnmd + + rustfs-node4: + image: rustfs/rustfs:latest + container_name: rustfs-node4 + hostname: rustfs-node4 + environment: + - RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4} + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_ACCESS_KEY=rustfsadmin + - RUSTFS_SECRET_KEY=rustfsadmin + - RUSTFS_CMD=rustfs + ports: + - "9030:9000" # API endpoint + - "9031:9001" # Console + volumes: + - node4-data1:/data/rustfs1 + - node4-data2:/data/rustfs2 + - node4-data3:/data/rustfs3 + - node4-data4:/data/rustfs4 + command: [ "sh", "-c", "sleep 3 && rustfs" ] + healthcheck: + test: + [ + "CMD", + "sh", "-c", + "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health" + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - rustfs-mnmd + +networks: + rustfs-mnmd: + driver: bridge + +volumes: + node1-data1: + node1-data2: + node1-data3: + node1-data4: + node2-data1: + node2-data2: + node2-data3: + node2-data4: + node3-data1: + node3-data2: + node3-data3: + node3-data4: + node4-data1: + node4-data2: + node4-data3: + node4-data4: diff --git a/docs/examples/mnmd/test-deployment.sh b/docs/examples/mnmd/test-deployment.sh new file mode 100755 index 00000000..40e627d6 --- /dev/null +++ b/docs/examples/mnmd/test-deployment.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# test-deployment.sh - Quick test script for MNMD deployment +# Usage: ./test-deployment.sh + +set -e + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "=========================================" +echo "RustFS MNMD Deployment Test" +echo "=========================================" +echo "" + +# Test 1: Check if all containers are running +echo "Test 1: Checking container status..." +RUNNING=$(docker-compose ps | grep -c "Up" || echo "0") +if [ "$RUNNING" -eq 4 ]; then + echo -e "${GREEN}✓ All 4 containers are running${NC}" +else + echo -e "${RED}✗ Only $RUNNING/4 containers are running${NC}" + docker-compose ps + exit 1 +fi +echo "" + +# Test 2: Check health status +echo "Test 2: Checking health status..." +HEALTHY=0 +for node in rustfs-node1 rustfs-node2 rustfs-node3 rustfs-node4; do + STATUS=$(docker inspect "$node" --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown") + if [ "$STATUS" = "healthy" ]; then + echo -e " ${GREEN}✓ $node is healthy${NC}" + HEALTHY=$((HEALTHY + 1)) + elif [ "$STATUS" = "starting" ]; then + echo -e " ${YELLOW}⚠ $node is starting (wait a moment)${NC}" + else + echo -e " ${RED}✗ $node status: $STATUS${NC}" + fi +done + +if [ "$HEALTHY" -eq 4 ]; then + echo -e "${GREEN}✓ All containers are healthy${NC}" +elif [ "$HEALTHY" -gt 0 ]; then + echo -e "${YELLOW}⚠ $HEALTHY/4 containers are healthy (some may still be starting)${NC}" +else + echo -e "${RED}✗ No containers are healthy${NC}" + exit 1 +fi +echo "" + +# Test 3: Check API endpoints +echo "Test 3: Testing API endpoints..." +PORTS=(9000 9010 9020 9030) +API_SUCCESS=0 +for port in "${PORTS[@]}"; do + if curl -sf http://localhost:${port}/health >/dev/null 2>&1; then + echo -e " ${GREEN}✓ API on port $port is responding${NC}" + API_SUCCESS=$((API_SUCCESS + 1)) + else + echo -e " ${RED}✗ API on port $port is not responding${NC}" + fi +done + +if [ "$API_SUCCESS" -eq 4 ]; then + echo -e "${GREEN}✓ All API endpoints are working${NC}" +else + echo -e "${YELLOW}⚠ $API_SUCCESS/4 API endpoints are working${NC}" +fi +echo "" + +# Test 4: Check Console endpoints +echo "Test 4: Testing Console endpoints..." +CONSOLE_PORTS=(9001 9011 9021 9031) +CONSOLE_SUCCESS=0 +for port in "${CONSOLE_PORTS[@]}"; do + if curl -sf http://localhost:${port}/health >/dev/null 2>&1; then + echo -e " ${GREEN}✓ Console on port $port is responding${NC}" + CONSOLE_SUCCESS=$((CONSOLE_SUCCESS + 1)) + else + echo -e " ${RED}✗ Console on port $port is not responding${NC}" + fi +done + +if [ "$CONSOLE_SUCCESS" -eq 4 ]; then + echo -e "${GREEN}✓ All Console endpoints are working${NC}" +else + echo -e "${YELLOW}⚠ $CONSOLE_SUCCESS/4 Console endpoints are working${NC}" +fi +echo "" + +# Test 5: Check inter-node connectivity +echo "Test 5: Testing inter-node connectivity..." +CONN_SUCCESS=0 +for node in rustfs-node2 rustfs-node3 rustfs-node4; do + if docker exec rustfs-node1 nc -z "$node" 9000 2>/dev/null; then + echo -e " ${GREEN}✓ node1 → $node connection OK${NC}" + CONN_SUCCESS=$((CONN_SUCCESS + 1)) + else + echo -e " ${RED}✗ node1 → $node connection failed${NC}" + fi +done + +if [ "$CONN_SUCCESS" -eq 3 ]; then + echo -e "${GREEN}✓ All inter-node connections are working${NC}" +else + echo -e "${YELLOW}⚠ $CONN_SUCCESS/3 inter-node connections are working${NC}" +fi +echo "" + +# Test 6: Verify data directories +echo "Test 6: Verifying data directories..." +DIR_SUCCESS=0 +for i in {1..4}; do + if docker exec rustfs-node1 test -d "/data/rustfs${i}"; then + DIR_SUCCESS=$((DIR_SUCCESS + 1)) + else + echo -e " ${RED}✗ /data/rustfs${i} not found in node1${NC}" + fi +done + +if [ "$DIR_SUCCESS" -eq 4 ]; then + echo -e "${GREEN}✓ All data directories exist${NC}" +else + echo -e "${RED}✗ Only $DIR_SUCCESS/4 data directories exist${NC}" +fi +echo "" + +# Summary +echo "=========================================" +echo "Test Summary" +echo "=========================================" +echo "Containers running: $RUNNING/4" +echo "Healthy containers: $HEALTHY/4" +echo "API endpoints: $API_SUCCESS/4" +echo "Console endpoints: $CONSOLE_SUCCESS/4" +echo "Inter-node connections: $CONN_SUCCESS/3" +echo "Data directories: $DIR_SUCCESS/4" +echo "" + +TOTAL=$((RUNNING + HEALTHY + API_SUCCESS + CONSOLE_SUCCESS + CONN_SUCCESS + DIR_SUCCESS)) +MAX_SCORE=23 + +if [ "$TOTAL" -eq "$MAX_SCORE" ]; then + echo -e "${GREEN}✓ All tests passed! Deployment is working correctly.${NC}" + exit 0 +elif [ "$TOTAL" -ge 20 ]; then + echo -e "${YELLOW}⚠ Most tests passed. Some components may still be starting up.${NC}" + echo " Try running this script again in a few moments." + exit 0 +else + echo -e "${RED}✗ Some tests failed. Check the output above and logs for details.${NC}" + echo " Run 'docker-compose logs' for more information." + exit 1 +fi diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index c5cccda5..790e9a29 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -70,7 +70,6 @@ clap = { workspace = true } datafusion = { workspace = true } const-str = { workspace = true } futures.workspace = true -hashbrown = { workspace = true } hyper.workspace = true hyper-util.workspace = true http.workspace = true @@ -79,7 +78,6 @@ matchit = { workspace = true } md5.workspace = true mime_guess = { workspace = true } opentelemetry = { workspace = true } -percent-encoding = { workspace = true } pin-project-lite.workspace = true reqwest = { workspace = true } rustls = { workspace = true }