From efae4f52039fb0303d9114a0f0a2fb54a1c37b19 Mon Sep 17 00:00:00 2001 From: overtrue Date: Tue, 17 Jun 2025 22:37:38 +0800 Subject: [PATCH] wip --- .cursorrules | 178 +++++++++---- .docker/Dockerfile.devenv | 5 +- .docker/Dockerfile.rockylinux9.3 | 5 +- .docker/Dockerfile.ubuntu22.04 | 5 +- .docker/cargo.config.toml | 8 - ecstore/BENCHMARK.md | 270 ------------------- ecstore/BENCHMARK_ZH.md | 270 ------------------- ecstore/IMPLEMENTATION_COMPARISON.md | 333 ------------------------ ecstore/IMPLEMENTATION_COMPARISON_ZH.md | 333 ------------------------ 9 files changed, 137 insertions(+), 1270 deletions(-) delete mode 100644 ecstore/BENCHMARK.md delete mode 100644 ecstore/BENCHMARK_ZH.md delete mode 100644 ecstore/IMPLEMENTATION_COMPARISON.md delete mode 100644 ecstore/IMPLEMENTATION_COMPARISON_ZH.md diff --git a/.cursorrules b/.cursorrules index a6b29285..83a208f5 100644 --- a/.cursorrules +++ b/.cursorrules @@ -3,6 +3,7 @@ ## ⚠️ CRITICAL DEVELOPMENT RULES ⚠️ ### 🚨 NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH 🚨 + - **This is the most important rule - NEVER modify code directly on main or master branch** - **Always work on feature branches and use pull requests for all changes** - **Any direct commits to master/main branch are strictly forbidden** @@ -15,23 +16,27 @@ 6. Create a pull request for review ## Project Overview + RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API. The project adopts a modular architecture, supporting erasure coding storage, multi-tenant management, observability, and other enterprise-level features. ## Core Architecture Principles ### 1. Modular Design + - Project uses Cargo workspace structure, containing multiple independent crates - Core modules: `rustfs` (main service), `ecstore` (erasure coding storage), `common` (shared components) - Functional modules: `iam` (identity management), `madmin` (management interface), `crypto` (encryption), etc. - Tool modules: `cli` (command line tool), `crates/*` (utility libraries) ### 2. Asynchronous Programming Pattern + - Comprehensive use of `tokio` async runtime - Prioritize `async/await` syntax - Use `async-trait` for async methods in traits - Avoid blocking operations, use `spawn_blocking` when necessary ### 3. Error Handling Strategy + - **Use modular, type-safe error handling with `thiserror`** - Each module should define its own error type using `thiserror::Error` derive macro - Support error chains and context information through `#[from]` and `#[source]` attributes @@ -54,6 +59,7 @@ RustFS is a high-performance distributed object storage system written in Rust, ## Code Style Guidelines ### 1. Formatting Configuration + ```toml max_width = 130 fn_call_width = 90 @@ -69,21 +75,25 @@ single_line_let_else_max_width = 100 Before every commit, you **MUST**: 1. **Format your code**: + ```bash cargo fmt --all ``` 2. **Verify formatting**: + ```bash cargo fmt --all --check ``` 3. **Pass clippy checks**: + ```bash cargo clippy --all-targets --all-features -- -D warnings ``` 4. **Ensure compilation**: + ```bash cargo check --all-targets ``` @@ -158,6 +168,7 @@ Example output when formatting fails: ``` ### 3. Naming Conventions + - Use `snake_case` for functions, variables, modules - Use `PascalCase` for types, traits, enums - Constants use `SCREAMING_SNAKE_CASE` @@ -167,6 +178,7 @@ Example output when formatting fails: - Choose names that clearly express the purpose and intent ### 4. Type Declaration Guidelines + - **Prefer type inference over explicit type declarations** when the type is obvious from context - Let the Rust compiler infer types whenever possible to reduce verbosity and improve maintainability - Only specify types explicitly when: @@ -176,6 +188,7 @@ Example output when formatting fails: - Needed to resolve ambiguity between multiple possible types **Good examples (prefer these):** + ```rust // Compiler can infer the type let items = vec![1, 2, 3, 4]; @@ -187,6 +200,7 @@ let filtered: Vec<_> = items.iter().filter(|&&x| x > 2).collect(); ``` **Avoid unnecessary explicit types:** + ```rust // Unnecessary - type is obvious let items: Vec = vec![1, 2, 3, 4]; @@ -195,6 +209,7 @@ let result: ProcessResult = process_data(&input); ``` **When explicit types are beneficial:** + ```rust // API boundaries - always specify types pub fn process_data(input: &[u8]) -> Result { ... } @@ -207,6 +222,7 @@ let cache: HashMap>> = HashMap::new(); ``` ### 5. Documentation Comments + - Public APIs must have documentation comments - Use `///` for documentation comments - Complex functions add `# Examples` and `# Parameters` descriptions @@ -215,6 +231,7 @@ let cache: HashMap>> = HashMap::new(); - Avoid meaningless comments like "debug 111" or placeholder text ### 6. Import Guidelines + - Standard library imports first - Third-party crate imports in the middle - Project internal imports last @@ -223,6 +240,7 @@ let cache: HashMap>> = HashMap::new(); ## Asynchronous Programming Guidelines ### 1. Trait Definition + ```rust #[async_trait::async_trait] pub trait StorageAPI: Send + Sync { @@ -231,6 +249,7 @@ pub trait StorageAPI: Send + Sync { ``` ### 2. Error Handling + ```rust // Use ? operator to propagate errors async fn example_function() -> Result<()> { @@ -241,6 +260,7 @@ async fn example_function() -> Result<()> { ``` ### 3. Concurrency Control + - Use `Arc` and `Mutex`/`RwLock` for shared state management - Prioritize async locks from `tokio::sync` - Avoid holding locks for long periods @@ -248,6 +268,7 @@ async fn example_function() -> Result<()> { ## Logging and Tracing Guidelines ### 1. Tracing Usage + ```rust #[tracing::instrument(skip(self, data))] async fn process_data(&self, data: &[u8]) -> Result<()> { @@ -257,6 +278,7 @@ async fn process_data(&self, data: &[u8]) -> Result<()> { ``` ### 2. Log Levels + - `error!`: System errors requiring immediate attention - `warn!`: Warning information that may affect functionality - `info!`: Important business information @@ -264,6 +286,7 @@ async fn process_data(&self, data: &[u8]) -> Result<()> { - `trace!`: Detailed execution paths ### 3. Structured Logging + ```rust info!( counter.rustfs_api_requests_total = 1_u64, @@ -276,22 +299,23 @@ info!( ## Error Handling Guidelines ### 1. Error Type Definition + ```rust // Use thiserror for module-specific error types #[derive(thiserror::Error, Debug)] pub enum MyError { #[error("IO error: {0}")] Io(#[from] std::io::Error), - + #[error("Storage error: {0}")] Storage(#[from] ecstore::error::StorageError), - + #[error("Custom error: {message}")] Custom { message: String }, - + #[error("File not found: {path}")] FileNotFound { path: String }, - + #[error("Invalid configuration: {0}")] InvalidConfig(String), } @@ -301,6 +325,7 @@ pub type Result = core::result::Result; ``` ### 2. Error Helper Methods + ```rust impl MyError { /// Create error from any compatible error type @@ -314,6 +339,7 @@ impl MyError { ``` ### 3. Error Conversion Between Modules + ```rust // Convert between different module error types impl From for MyError { @@ -340,6 +366,7 @@ impl From for ecstore::error::StorageError { ``` ### 4. Error Context and Propagation + ```rust // Use ? operator for clean error propagation async fn example_function() -> Result<()> { @@ -351,14 +378,15 @@ async fn example_function() -> Result<()> { // Add context to errors fn process_with_context(path: &str) -> Result<()> { std::fs::read(path) - .map_err(|e| MyError::Custom { - message: format!("Failed to read {}: {}", path, e) + .map_err(|e| MyError::Custom { + message: format!("Failed to read {}: {}", path, e) })?; Ok(()) } ``` ### 5. API Error Conversion (S3 Example) + ```rust // Convert storage errors to API-specific errors use s3s::{S3Error, S3ErrorCode}; @@ -404,6 +432,7 @@ impl From for S3Error { ### 6. Error Handling Best Practices #### Pattern Matching and Error Classification + ```rust // Use pattern matching for specific error handling async fn handle_storage_operation() -> Result<()> { @@ -415,8 +444,8 @@ async fn handle_storage_operation() -> Result<()> { } Err(ecstore::error::StorageError::BucketNotFound(bucket)) => { error!("Bucket not found: {}", bucket); - Err(MyError::Custom { - message: format!("Bucket {} does not exist", bucket) + Err(MyError::Custom { + message: format!("Bucket {} does not exist", bucket) }) } Err(e) => { @@ -428,30 +457,32 @@ async fn handle_storage_operation() -> Result<()> { ``` #### Error Aggregation and Reporting + ```rust // Collect and report multiple errors pub fn validate_configuration(config: &Config) -> Result<()> { let mut errors = Vec::new(); - + if config.bucket_name.is_empty() { errors.push("Bucket name cannot be empty"); } - + if config.region.is_empty() { errors.push("Region must be specified"); } - + if !errors.is_empty() { return Err(MyError::Custom { message: format!("Configuration validation failed: {}", errors.join(", ")) }); } - + Ok(()) } ``` #### Contextual Error Information + ```rust // Add operation context to errors #[tracing::instrument(skip(self))] @@ -468,11 +499,13 @@ async fn upload_file(&self, bucket: &str, key: &str, data: Vec) -> Result<() ## Performance Optimization Guidelines ### 1. Memory Management + - Use `Bytes` instead of `Vec` for zero-copy operations - Avoid unnecessary cloning, use reference passing - Use `Arc` for sharing large objects ### 2. Concurrency Optimization + ```rust // Use join_all for concurrent operations let futures = disks.iter().map(|disk| disk.operation()); @@ -480,12 +513,14 @@ let results = join_all(futures).await; ``` ### 3. Caching Strategy + - Use `lazy_static` or `OnceCell` for global caching - Implement LRU cache to avoid memory leaks ## Testing Guidelines ### 1. Unit Tests + ```rust #[cfg(test)] mod tests { @@ -507,10 +542,10 @@ mod tests { #[test] fn test_error_conversion() { use ecstore::error::StorageError; - + let storage_err = StorageError::BucketNotFound("test-bucket".to_string()); let api_err: ApiError = storage_err.into(); - + assert_eq!(api_err.code, S3ErrorCode::NoSuchBucket); assert!(api_err.message.contains("test-bucket")); assert!(api_err.source.is_some()); @@ -520,7 +555,7 @@ mod tests { fn test_error_types() { let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found"); let my_err = MyError::Io(io_err); - + // Test error matching match my_err { MyError::Io(_) => {}, // Expected @@ -532,7 +567,7 @@ mod tests { fn test_error_context() { let result = process_with_context("nonexistent_file.txt"); assert!(result.is_err()); - + let err = result.unwrap_err(); match err { MyError::Custom { message } => { @@ -546,10 +581,12 @@ mod tests { ``` ### 2. Integration Tests + - Use `e2e_test` module for end-to-end testing - Simulate real storage environments ### 3. Test Quality Standards + - Write meaningful test cases that verify actual functionality - Avoid placeholder or debug content like "debug 111", "test test", etc. - Use descriptive test names that clearly indicate what is being tested @@ -559,9 +596,11 @@ mod tests { ## Cross-Platform Compatibility Guidelines ### 1. CPU Architecture Compatibility + - **Always consider multi-platform and different CPU architecture compatibility** when writing code - Support major architectures: x86_64, aarch64 (ARM64), and other target platforms - Use conditional compilation for architecture-specific code: + ```rust #[cfg(target_arch = "x86_64")] fn optimized_x86_64_function() { /* x86_64 specific implementation */ } @@ -574,16 +613,19 @@ fn generic_function() { /* Generic fallback implementation */ } ``` ### 2. Platform-Specific Dependencies + - Use feature flags for platform-specific dependencies - Provide fallback implementations for unsupported platforms - Test on multiple architectures in CI/CD pipeline ### 3. Endianness Considerations + - Use explicit byte order conversion when dealing with binary data - Prefer `to_le_bytes()`, `from_le_bytes()` for consistent little-endian format - Use `byteorder` crate for complex binary format handling ### 4. SIMD and Performance Optimizations + - Use portable SIMD libraries like `wide` or `packed_simd` - Provide fallback implementations for non-SIMD architectures - Use runtime feature detection when appropriate @@ -591,10 +633,12 @@ fn generic_function() { /* Generic fallback implementation */ } ## Security Guidelines ### 1. Memory Safety + - Disable `unsafe` code (workspace.lints.rust.unsafe_code = "deny") - Use `rustls` instead of `openssl` ### 2. Authentication and Authorization + ```rust // Use IAM system for permission checks let identity = iam.authenticate(&access_key, &secret_key).await?; @@ -604,11 +648,13 @@ iam.authorize(&identity, &action, &resource).await?; ## Configuration Management Guidelines ### 1. Environment Variables + - Use `RUSTFS_` prefix - Support both configuration files and environment variables - Provide reasonable default values ### 2. Configuration Structure + ```rust #[derive(Debug, Deserialize, Clone)] pub struct Config { @@ -622,10 +668,12 @@ pub struct Config { ## Dependency Management Guidelines ### 1. Workspace Dependencies + - Manage versions uniformly at workspace level - Use `workspace = true` to inherit configuration ### 2. Feature Flags + ```rust [features] default = ["file"] @@ -636,15 +684,18 @@ kafka = ["dep:rdkafka"] ## Deployment and Operations Guidelines ### 1. Containerization + - Provide Dockerfile and docker-compose configuration - Support multi-stage builds to optimize image size ### 2. Observability + - Integrate OpenTelemetry for distributed tracing - Support Prometheus metrics collection - Provide Grafana dashboards ### 3. Health Checks + ```rust // Implement health check endpoint async fn health_check() -> Result { @@ -655,6 +706,7 @@ async fn health_check() -> Result { ## Code Review Checklist ### 1. **Code Formatting and Quality (MANDATORY)** + - [ ] **Code is properly formatted** (`cargo fmt --all --check` passes) - [ ] **All clippy warnings are resolved** (`cargo clippy --all-targets --all-features -- -D warnings` passes) - [ ] **Code compiles successfully** (`cargo check --all-targets` passes) @@ -662,27 +714,32 @@ async fn health_check() -> Result { - [ ] **No formatting-related changes** mixed with functional changes (separate commits) ### 2. Functionality + - [ ] Are all error cases properly handled? - [ ] Is there appropriate logging? - [ ] Is there necessary test coverage? ### 3. Performance + - [ ] Are unnecessary memory allocations avoided? - [ ] Are async operations used correctly? - [ ] Are there potential deadlock risks? ### 4. Security + - [ ] Are input parameters properly validated? - [ ] Are there appropriate permission checks? - [ ] Is information leakage avoided? ### 5. Cross-Platform Compatibility + - [ ] Does the code work on different CPU architectures (x86_64, aarch64)? - [ ] Are platform-specific features properly gated with conditional compilation? - [ ] Is byte order handling correct for binary data? - [ ] Are there appropriate fallback implementations for unsupported platforms? ### 6. Code Commits and Documentation + - [ ] Does it comply with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)? - [ ] Are commit messages concise and under 72 characters for the title line? - [ ] Commit titles should be concise and in English, avoid Chinese @@ -691,6 +748,7 @@ async fn health_check() -> Result { ## Common Patterns and Best Practices ### 1. Resource Management + ```rust // Use RAII pattern for resource management pub struct ResourceGuard { @@ -705,6 +763,7 @@ impl Drop for ResourceGuard { ``` ### 2. Dependency Injection + ```rust // Use dependency injection pattern pub struct Service { @@ -714,6 +773,7 @@ pub struct Service { ``` ### 3. Graceful Shutdown + ```rust // Implement graceful shutdown async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) { @@ -732,16 +792,19 @@ async fn shutdown_gracefully(shutdown_rx: &mut Receiver<()>) { ## Domain-Specific Guidelines ### 1. Storage Operations + - All storage operations must support erasure coding - Implement read/write quorum mechanisms - Support data integrity verification ### 2. Network Communication + - Use gRPC for internal service communication - HTTP/HTTPS support for S3-compatible API - Implement connection pooling and retry mechanisms ### 3. Metadata Management + - Use FlatBuffers for serialization - Support version control and migration - Implement metadata caching @@ -751,11 +814,12 @@ These rules should serve as guiding principles when developing the RustFS projec ### 4. Code Operations #### Branch Management - - **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨** - - **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️** - - **Always work on feature branches - NO EXCEPTIONS** - - Always check the .cursorrules file before starting to ensure you understand the project guidelines - - **MANDATORY workflow for ALL changes:** + +- **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨** +- **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️** +- **Always work on feature branches - NO EXCEPTIONS** +- Always check the .cursorrules file before starting to ensure you understand the project guidelines +- **MANDATORY workflow for ALL changes:** 1. `git checkout main` (switch to main branch) 2. `git pull` (get latest changes) 3. `git checkout -b feat/your-feature-name` (create and switch to feature branch) @@ -763,28 +827,54 @@ These rules should serve as guiding principles when developing the RustFS projec 5. Test thoroughly before committing 6. Commit and push to the feature branch 7. Create a pull request for code review - - Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc. - - **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master** - - Ensure all changes are made on feature branches and merged through pull requests +- Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc. +- **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master** +- Ensure all changes are made on feature branches and merged through pull requests #### Development Workflow - - Use English for all code comments, documentation, and variable names - - Write meaningful and descriptive names for variables, functions, and methods - - Avoid meaningless test content like "debug 111" or placeholder values - - Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues - - Ensure each change provides sufficient test cases to guarantee code correctness - - Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness - - When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing - - **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks** - - After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) - - **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed - - After each development completion, first git push to remote repository - - After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) - - Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) - - **Always provide PR descriptions in English** after completing any changes, including: - - Clear and concise title following Conventional Commits format - - Detailed description of what was changed and why - - List of key changes and improvements - - Any breaking changes or migration notes if applicable - - Testing information and verification steps - - **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying + +- Use English for all code comments, documentation, and variable names +- Write meaningful and descriptive names for variables, functions, and methods +- Avoid meaningless test content like "debug 111" or placeholder values +- Before each change, carefully read the existing code to ensure you understand the code structure and implementation, do not break existing logic implementation, do not introduce new issues +- Ensure each change provides sufficient test cases to guarantee code correctness +- Do not arbitrarily modify numbers and constants in test cases, carefully analyze their meaning to ensure test case correctness +- When writing or modifying tests, check existing test cases to ensure they have scientific naming and rigorous logic testing, if not compliant, modify test cases to ensure scientific and rigorous testing +- **Before committing any changes, run `cargo clippy --all-targets --all-features -- -D warnings` to ensure all code passes Clippy checks** +- After each development completion, first git add . then git commit -m "feat: feature description" or "fix: issue description", ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) +- **Keep commit messages concise and under 72 characters** for the title line, use body for detailed explanations if needed +- After each development completion, first git push to remote repository +- After each change completion, summarize the changes, do not create summary files, provide a brief change description, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) +- Provide change descriptions needed for PR in the conversation, ensure compliance with [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) +- **Always provide PR descriptions in English** after completing any changes, including: + - Clear and concise title following Conventional Commits format + - Detailed description of what was changed and why + - List of key changes and improvements + - Any breaking changes or migration notes if applicable + - Testing information and verification steps +- **Provide PR descriptions in copyable markdown format** enclosed in code blocks for easy one-click copying + +## 🚫 AI 文档生成限制 + +### 禁止生成总结文档 + +- **严格禁止创建任何形式的AI生成总结文档** +- **不得创建包含大量表情符号、详细格式化表格和典型AI风格的文档** +- **不得在项目中生成以下类型的文档：** + - 基准测试总结文档（BENCHMARK*.md） + - 实现对比分析文档（IMPLEMENTATION_COMPARISON*.md） + - 性能分析报告文档 + - 架构总结文档 + - 功能对比文档 + - 任何带有大量表情符号和格式化内容的文档 +- **如果需要文档，请只在用户明确要求时创建，并保持简洁实用的风格** +- **文档应当专注于实际需要的信息，避免过度格式化和装饰性内容** +- **任何发现的AI生成总结文档都应该立即删除** + +### 允许的文档类型 + +- README.md（项目介绍，保持简洁） +- 技术文档（仅在明确需要时创建） +- 用户手册（仅在明确需要时创建） +- API文档（从代码生成） +- 变更日志（CHANGELOG.md） diff --git a/.docker/Dockerfile.devenv b/.docker/Dockerfile.devenv index fee6c2dd..2b028c8f 100644 --- a/.docker/Dockerfile.devenv +++ b/.docker/Dockerfile.devenv @@ -18,10 +18,7 @@ RUN wget https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux. && mv flatc /usr/local/bin/ && chmod +x /usr/local/bin/flatc && rm -rf Linux.flatc.binary.g++-13.zip # install rust -ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" -ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" -RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ - && sh rustup-init.sh -y && rm -rf rustup-init.sh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh COPY .docker/cargo.config.toml /root/.cargo/config.toml diff --git a/.docker/Dockerfile.rockylinux9.3 b/.docker/Dockerfile.rockylinux9.3 index f677aabe..43ab1dcb 100644 --- a/.docker/Dockerfile.rockylinux9.3 +++ b/.docker/Dockerfile.rockylinux9.3 @@ -25,10 +25,7 @@ RUN wget https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux. && rm -rf Linux.flatc.binary.g++-13.zip # install rust -ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" -ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" -RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ - && sh rustup-init.sh -y && rm -rf rustup-init.sh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh COPY .docker/cargo.config.toml /root/.cargo/config.toml diff --git a/.docker/Dockerfile.ubuntu22.04 b/.docker/Dockerfile.ubuntu22.04 index 2cb9689c..3f438400 100644 --- a/.docker/Dockerfile.ubuntu22.04 +++ b/.docker/Dockerfile.ubuntu22.04 @@ -18,10 +18,7 @@ RUN wget https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux. && mv flatc /usr/local/bin/ && chmod +x /usr/local/bin/flatc && rm -rf Linux.flatc.binary.g++-13.zip # install rust -ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" -ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" -RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ - && sh rustup-init.sh -y && rm -rf rustup-init.sh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh COPY .docker/cargo.config.toml /root/.cargo/config.toml diff --git a/.docker/cargo.config.toml b/.docker/cargo.config.toml index ef2fa863..fc6904fe 100644 --- a/.docker/cargo.config.toml +++ b/.docker/cargo.config.toml @@ -1,13 +1,5 @@ [source.crates-io] registry = "https://github.com/rust-lang/crates.io-index" -replace-with = 'rsproxy-sparse' - -[source.rsproxy] -registry = "https://rsproxy.cn/crates.io-index" -[registries.rsproxy] -index = "https://rsproxy.cn/crates.io-index" -[source.rsproxy-sparse] -registry = "sparse+https://rsproxy.cn/index/" [net] git-fetch-with-cli = true diff --git a/ecstore/BENCHMARK.md b/ecstore/BENCHMARK.md deleted file mode 100644 index 5a420dc8..00000000 --- a/ecstore/BENCHMARK.md +++ /dev/null @@ -1,270 +0,0 @@ -# Reed-Solomon Erasure Coding Performance Benchmark - -This directory contains a comprehensive benchmark suite for comparing the performance of different Reed-Solomon implementations. - -## 📊 Test Overview - -### Supported Implementation Modes - -#### 🏛️ Pure Erasure Mode (Default, Recommended) -- **Stable and Reliable**: Uses mature reed-solomon-erasure implementation -- **Wide Compatibility**: Supports arbitrary shard sizes -- **Memory Efficient**: Optimized memory usage patterns -- **Predictable**: Performance insensitive to shard size -- **Use Case**: Default choice for production environments, suitable for most application scenarios - -#### 🎯 SIMD Mode (`reed-solomon-simd` feature) -- **High Performance Optimization**: Uses SIMD instruction sets for high-performance encoding/decoding -- **Performance Oriented**: Focuses on maximizing processing performance -- **Target Scenarios**: High-performance scenarios for large data processing -- **Use Case**: Scenarios requiring maximum performance, suitable for handling large amounts of data - -### Test Dimensions - -- **Encoding Performance** - Speed of encoding data into erasure code shards -- **Decoding Performance** - Speed of recovering original data from erasure code shards -- **Shard Size Sensitivity** - Impact of different shard sizes on performance -- **Erasure Code Configuration** - Performance impact of different data/parity shard ratios -- **SIMD Mode Performance** - Performance characteristics of SIMD optimization -- **Concurrency Performance** - Performance in multi-threaded environments -- **Memory Efficiency** - Memory usage patterns and efficiency -- **Error Recovery Capability** - Recovery performance under different numbers of lost shards - -## 🚀 Quick Start - -### Run Quick Tests - -```bash -# Run quick performance comparison tests (default pure Erasure mode) -./run_benchmarks.sh quick -``` - -### Run Complete Comparison Tests - -```bash -# Run detailed implementation comparison tests -./run_benchmarks.sh comparison -``` - -### Run Specific Mode Tests - -```bash -# Test default pure erasure mode (recommended) -./run_benchmarks.sh erasure - -# Test SIMD mode -./run_benchmarks.sh simd -``` - -## 📈 Manual Benchmark Execution - -### Basic Usage - -```bash -# Run all benchmarks (default pure erasure mode) -cargo bench - -# Run specific benchmark files -cargo bench --bench erasure_benchmark -cargo bench --bench comparison_benchmark -``` - -### Compare Different Implementation Modes - -```bash -# Test default pure erasure mode -cargo bench --bench comparison_benchmark - -# Test SIMD mode -cargo bench --bench comparison_benchmark \ - --features reed-solomon-simd - -# Save baseline for comparison -cargo bench --bench comparison_benchmark \ - -- --save-baseline erasure_baseline - -# Compare SIMD mode performance with baseline -cargo bench --bench comparison_benchmark \ - --features reed-solomon-simd \ - -- --baseline erasure_baseline -``` - -### Filter Specific Tests - -```bash -# Run only encoding tests -cargo bench encode - -# Run only decoding tests -cargo bench decode - -# Run tests for specific data sizes -cargo bench 1MB - -# Run tests for specific configurations -cargo bench "4+2" -``` - -## 📊 View Results - -### HTML Reports - -Benchmark results automatically generate HTML reports: - -```bash -# Start local server to view reports -cd target/criterion -python3 -m http.server 8080 - -# Access in browser -open http://localhost:8080/report/index.html -``` - -### Command Line Output - -Benchmarks display in terminal: -- Operations per second (ops/sec) -- Throughput (MB/s) -- Latency statistics (mean, standard deviation, percentiles) -- Performance trend changes - -## 🔧 Test Configuration - -### Data Sizes - -- **Small Data**: 1KB, 8KB - Test small file scenarios -- **Medium Data**: 64KB, 256KB - Test common file sizes -- **Large Data**: 1MB, 4MB - Test large file processing and SIMD optimization -- **Very Large Data**: 16MB+ - Test high throughput scenarios - -### Erasure Code Configurations - -- **(4,2)** - Common configuration, 33% redundancy -- **(6,3)** - 50% redundancy, balanced performance and reliability -- **(8,4)** - 50% redundancy, more parallelism -- **(10,5)**, **(12,6)** - High parallelism configurations - -### Shard Sizes - -Test different shard sizes from 32 bytes to 8KB, with special focus on: -- **Memory Alignment**: 64, 128, 256 bytes - Impact of memory alignment on performance -- **Cache Friendly**: 1KB, 2KB, 4KB - CPU cache-friendly sizes - -## 📝 Interpreting Test Results - -### Performance Metrics - -1. **Throughput** - - Unit: MB/s or GB/s - - Measures data processing speed - - Higher is better - -2. **Latency** - - Unit: microseconds (μs) or milliseconds (ms) - - Measures single operation time - - Lower is better - -3. **CPU Efficiency** - - Bytes processed per CPU cycle - - Reflects algorithm efficiency - -### Expected Results - -**Pure Erasure Mode (Default)**: -- Stable performance, insensitive to shard size -- Best compatibility, supports all configurations -- Stable and predictable memory usage - -**SIMD Mode (`reed-solomon-simd` feature)**: -- High-performance SIMD optimized implementation -- Suitable for large data processing scenarios -- Focuses on maximizing performance - -**Shard Size Sensitivity**: -- SIMD mode may be more sensitive to shard sizes -- Pure Erasure mode relatively insensitive to shard size - -**Memory Usage**: -- SIMD mode may have specific memory alignment requirements -- Pure Erasure mode has more stable memory usage - -## 🛠️ Custom Testing - -### Adding New Test Scenarios - -Edit `benches/erasure_benchmark.rs` or `benches/comparison_benchmark.rs`: - -```rust -// Add new test configuration -let configs = vec![ - // Your custom configuration - BenchConfig::new(10, 4, 2048 * 1024, 2048 * 1024), // 10+4, 2MB -]; -``` - -### Adjust Test Parameters - -```rust -// Modify sampling and test time -group.sample_size(20); // Sample count -group.measurement_time(Duration::from_secs(10)); // Test duration -``` - -## 🐛 Troubleshooting - -### Common Issues - -1. **Compilation Errors**: Ensure correct dependencies are installed -```bash -cargo update -cargo build --all-features -``` - -2. **Performance Anomalies**: Check if running in correct mode -```bash -# Check current configuration -cargo bench --bench comparison_benchmark -- --help -``` - -3. **Tests Taking Too Long**: Adjust test parameters -```bash -# Use shorter test duration -cargo bench -- --quick -``` - -### Performance Analysis - -Use tools like `perf` for detailed performance analysis: - -```bash -# Analyze CPU usage -cargo bench --bench comparison_benchmark & -perf record -p $(pgrep -f comparison_benchmark) -perf report -``` - -## 🤝 Contributing - -Welcome to submit new benchmark scenarios or optimization suggestions: - -1. Fork the project -2. Create feature branch: `git checkout -b feature/new-benchmark` -3. Add test cases -4. Commit changes: `git commit -m 'Add new benchmark for XYZ'` -5. Push to branch: `git push origin feature/new-benchmark` -6. Create Pull Request - -## 📚 References - -- [reed-solomon-erasure crate](https://crates.io/crates/reed-solomon-erasure) -- [reed-solomon-simd crate](https://crates.io/crates/reed-solomon-simd) -- [Criterion.rs benchmark framework](https://bheisler.github.io/criterion.rs/book/) -- [Reed-Solomon error correction principles](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) - ---- - -💡 **Tips**: -- Recommend using the default pure Erasure mode, which provides stable performance across various scenarios -- Consider SIMD mode for high-performance requirements -- Benchmark results may vary based on hardware, operating system, and compiler versions -- Suggest running tests in target deployment environment for most accurate performance data \ No newline at end of file diff --git a/ecstore/BENCHMARK_ZH.md b/ecstore/BENCHMARK_ZH.md deleted file mode 100644 index 88355ed6..00000000 --- a/ecstore/BENCHMARK_ZH.md +++ /dev/null @@ -1,270 +0,0 @@ -# Reed-Solomon 纠删码性能基准测试 - -本目录包含了比较不同 Reed-Solomon 实现性能的综合基准测试套件。 - -## 📊 测试概述 - -### 支持的实现模式 - -#### 🏛️ 纯 Erasure 模式（默认，推荐） -- **稳定可靠**: 使用成熟的 reed-solomon-erasure 实现 -- **广泛兼容**: 支持任意分片大小 -- **内存高效**: 优化的内存使用模式 -- **可预测性**: 性能对分片大小不敏感 -- **使用场景**: 生产环境默认选择，适合大多数应用场景 - -#### 🎯 SIMD模式（`reed-solomon-simd` feature） -- **高性能优化**: 使用SIMD指令集进行高性能编码解码 -- **性能导向**: 专注于最大化处理性能 -- **适用场景**: 大数据量处理的高性能场景 -- **使用场景**: 需要最大化性能的场景，适合处理大量数据 - -### 测试维度 - -- **编码性能** - 数据编码成纠删码分片的速度 -- **解码性能** - 从纠删码分片恢复原始数据的速度 -- **分片大小敏感性** - 不同分片大小对性能的影响 -- **纠删码配置** - 不同数据/奇偶分片比例的性能影响 -- **SIMD模式性能** - SIMD优化的性能表现 -- **并发性能** - 多线程环境下的性能表现 -- **内存效率** - 内存使用模式和效率 -- **错误恢复能力** - 不同丢失分片数量下的恢复性能 - -## 🚀 快速开始 - -### 运行快速测试 - -```bash -# 运行快速性能对比测试（默认纯Erasure模式） -./run_benchmarks.sh quick -``` - -### 运行完整对比测试 - -```bash -# 运行详细的实现对比测试 -./run_benchmarks.sh comparison -``` - -### 运行特定模式的测试 - -```bash -# 测试默认纯 erasure 模式（推荐） -./run_benchmarks.sh erasure - -# 测试SIMD模式 -./run_benchmarks.sh simd -``` - -## 📈 手动运行基准测试 - -### 基本使用 - -```bash -# 运行所有基准测试（默认纯 erasure 模式） -cargo bench - -# 运行特定的基准测试文件 -cargo bench --bench erasure_benchmark -cargo bench --bench comparison_benchmark -``` - -### 对比不同实现模式 - -```bash -# 测试默认纯 erasure 模式 -cargo bench --bench comparison_benchmark - -# 测试SIMD模式 -cargo bench --bench comparison_benchmark \ - --features reed-solomon-simd - -# 保存基线进行对比 -cargo bench --bench comparison_benchmark \ - -- --save-baseline erasure_baseline - -# 与基线比较SIMD模式性能 -cargo bench --bench comparison_benchmark \ - --features reed-solomon-simd \ - -- --baseline erasure_baseline -``` - -### 过滤特定测试 - -```bash -# 只运行编码测试 -cargo bench encode - -# 只运行解码测试 -cargo bench decode - -# 只运行特定数据大小的测试 -cargo bench 1MB - -# 只运行特定配置的测试 -cargo bench "4+2" -``` - -## 📊 查看结果 - -### HTML 报告 - -基准测试结果会自动生成 HTML 报告： - -```bash -# 启动本地服务器查看报告 -cd target/criterion -python3 -m http.server 8080 - -# 在浏览器中访问 -open http://localhost:8080/report/index.html -``` - -### 命令行输出 - -基准测试会在终端显示： -- 每秒操作数 (ops/sec) -- 吞吐量 (MB/s) -- 延迟统计 (平均值、标准差、百分位数) -- 性能变化趋势 - -## 🔧 测试配置 - -### 数据大小 - -- **小数据**: 1KB, 8KB - 测试小文件场景 -- **中等数据**: 64KB, 256KB - 测试常见文件大小 -- **大数据**: 1MB, 4MB - 测试大文件处理和 SIMD 优化 -- **超大数据**: 16MB+ - 测试高吞吐量场景 - -### 纠删码配置 - -- **(4,2)** - 常用配置，33% 冗余 -- **(6,3)** - 50% 冗余，平衡性能和可靠性 -- **(8,4)** - 50% 冗余，更多并行度 -- **(10,5)**, **(12,6)** - 高并行度配置 - -### 分片大小 - -测试从 32 字节到 8KB 的不同分片大小，特别关注： -- **内存对齐**: 64, 128, 256 字节 - 内存对齐对性能的影响 -- **Cache 友好**: 1KB, 2KB, 4KB - CPU 缓存友好的大小 - -## 📝 解读测试结果 - -### 性能指标 - -1. **吞吐量 (Throughput)** - - 单位: MB/s 或 GB/s - - 衡量数据处理速度 - - 越高越好 - -2. **延迟 (Latency)** - - 单位: 微秒 (μs) 或毫秒 (ms) - - 衡量单次操作时间 - - 越低越好 - -3. **CPU 效率** - - 每 CPU 周期处理的字节数 - - 反映算法效率 - -### 预期结果 - -**纯 Erasure 模式（默认）**: -- 性能稳定，对分片大小不敏感 -- 兼容性最佳，支持所有配置 -- 内存使用稳定可预测 - -**SIMD模式（`reed-solomon-simd` feature）**: -- 高性能SIMD优化实现 -- 适合大数据量处理场景 -- 专注于最大化性能 - -**分片大小敏感性**: -- SIMD模式对分片大小可能更敏感 -- 纯 Erasure 模式对分片大小相对不敏感 - -**内存使用**: -- SIMD模式可能有特定的内存对齐要求 -- 纯 Erasure 模式内存使用更稳定 - -## 🛠️ 自定义测试 - -### 添加新的测试场景 - -编辑 `benches/erasure_benchmark.rs` 或 `benches/comparison_benchmark.rs`： - -```rust -// 添加新的测试配置 -let configs = vec![ - // 你的自定义配置 - BenchConfig::new(10, 4, 2048 * 1024, 2048 * 1024), // 10+4, 2MB -]; -``` - -### 调整测试参数 - -```rust -// 修改采样和测试时间 -group.sample_size(20); // 样本数量 -group.measurement_time(Duration::from_secs(10)); // 测试时间 -``` - -## 🐛 故障排除 - -### 常见问题 - -1. **编译错误**: 确保安装了正确的依赖 -```bash -cargo update -cargo build --all-features -``` - -2. **性能异常**: 检查是否在正确的模式下运行 -```bash -# 检查当前配置 -cargo bench --bench comparison_benchmark -- --help -``` - -3. **测试时间过长**: 调整测试参数 -```bash -# 使用更短的测试时间 -cargo bench -- --quick -``` - -### 性能分析 - -使用 `perf` 等工具进行更详细的性能分析： - -```bash -# 分析 CPU 使用情况 -cargo bench --bench comparison_benchmark & -perf record -p $(pgrep -f comparison_benchmark) -perf report -``` - -## 🤝 贡献 - -欢迎提交新的基准测试场景或优化建议： - -1. Fork 项目 -2. 创建特性分支: `git checkout -b feature/new-benchmark` -3. 添加测试用例 -4. 提交更改: `git commit -m 'Add new benchmark for XYZ'` -5. 推送到分支: `git push origin feature/new-benchmark` -6. 创建 Pull Request - -## 📚 参考资料 - -- [reed-solomon-erasure crate](https://crates.io/crates/reed-solomon-erasure) -- [reed-solomon-simd crate](https://crates.io/crates/reed-solomon-simd) -- [Criterion.rs 基准测试框架](https://bheisler.github.io/criterion.rs/book/) -- [Reed-Solomon 纠删码原理](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) - ---- - -💡 **提示**: -- 推荐使用默认的纯Erasure模式，它在各种场景下都有稳定的表现 -- 对于高性能需求可以考虑SIMD模式 -- 基准测试结果可能因硬件、操作系统和编译器版本而异 -- 建议在目标部署环境中运行测试以获得最准确的性能数据 \ No newline at end of file diff --git a/ecstore/IMPLEMENTATION_COMPARISON.md b/ecstore/IMPLEMENTATION_COMPARISON.md deleted file mode 100644 index 1e5c2d32..00000000 --- a/ecstore/IMPLEMENTATION_COMPARISON.md +++ /dev/null @@ -1,333 +0,0 @@ -# Reed-Solomon Implementation Comparison Analysis - -## 🔍 Issue Analysis - -With the optimized SIMD mode design, we provide high-performance Reed-Solomon implementation. The system can now deliver optimal performance across different scenarios. - -## 📊 Implementation Mode Comparison - -### 🏛️ Pure Erasure Mode (Default, Recommended) - -**Default Configuration**: No features specified, uses stable reed-solomon-erasure implementation - -**Characteristics**: -- ✅ **Wide Compatibility**: Supports any shard size from byte-level to GB-level -- 📈 **Stable Performance**: Performance insensitive to shard size, predictable -- 🔧 **Production Ready**: Mature and stable implementation, widely used in production -- 💾 **Memory Efficient**: Optimized memory usage patterns -- 🎯 **Consistency**: Completely consistent behavior across all scenarios - -**Use Cases**: -- Default choice for most production environments -- Systems requiring completely consistent and predictable performance behavior -- Performance-change-sensitive systems -- Scenarios mainly processing small files or small shards -- Systems requiring strict memory usage control - -### 🎯 SIMD Mode (`reed-solomon-simd` feature) - -**Configuration**: `--features reed-solomon-simd` - -**Characteristics**: -- 🚀 **High-Performance SIMD**: Uses SIMD instruction sets for high-performance encoding/decoding -- 🎯 **Performance Oriented**: Focuses on maximizing processing performance -- ⚡ **Large Data Optimization**: Suitable for high-throughput scenarios with large data processing -- 🏎️ **Speed Priority**: Designed for performance-critical applications - -**Use Cases**: -- Application scenarios requiring maximum performance -- High-throughput systems processing large amounts of data -- Scenarios with extremely high performance requirements -- CPU-intensive workloads - -## 📏 Shard Size vs Performance Comparison - -Performance across different configurations: - -| Data Size | Config | Shard Size | Pure Erasure Mode (Default) | SIMD Mode Strategy | Performance Comparison | -|-----------|--------|------------|----------------------------|-------------------|----------------------| -| 1KB | 4+2 | 256 bytes | Erasure implementation | SIMD implementation | SIMD may be faster | -| 1KB | 6+3 | 171 bytes | Erasure implementation | SIMD implementation | SIMD may be faster | -| 1KB | 8+4 | 128 bytes | Erasure implementation | SIMD implementation | SIMD may be faster | -| 64KB | 4+2 | 16KB | Erasure implementation | SIMD optimization | SIMD mode faster | -| 64KB | 6+3 | 10.7KB | Erasure implementation | SIMD optimization | SIMD mode faster | -| 1MB | 4+2 | 256KB | Erasure implementation | SIMD optimization | SIMD mode significantly faster | -| 16MB | 8+4 | 2MB | Erasure implementation | SIMD optimization | SIMD mode substantially faster | - -## 🎯 Benchmark Results Interpretation - -### Pure Erasure Mode Example (Default) ✅ - -``` -encode_comparison/implementation/1KB_6+3_erasure - time: [245.67 ns 256.78 ns 267.89 ns] - thrpt: [3.73 GiB/s 3.89 GiB/s 4.07 GiB/s] - -💡 Consistent Erasure performance - All configurations use the same implementation -``` - -``` -encode_comparison/implementation/64KB_4+2_erasure - time: [2.3456 μs 2.4567 μs 2.5678 μs] - thrpt: [23.89 GiB/s 24.65 GiB/s 25.43 GiB/s] - -💡 Stable and reliable performance - Suitable for most production scenarios -``` - -### SIMD Mode Success Examples ✅ - -**Large Shard SIMD Optimization**: -``` -encode_comparison/implementation/64KB_4+2_simd - time: [1.2345 μs 1.2567 μs 1.2789 μs] - thrpt: [47.89 GiB/s 48.65 GiB/s 49.43 GiB/s] - -💡 Using SIMD optimization - Shard size: 16KB, high-performance processing -``` - -**Small Shard SIMD Processing**: -``` -encode_comparison/implementation/1KB_6+3_simd - time: [234.56 ns 245.67 ns 256.78 ns] - thrpt: [3.89 GiB/s 4.07 GiB/s 4.26 GiB/s] - -💡 SIMD processing small shards - Shard size: 171 bytes -``` - -## 🛠️ Usage Guide - -### Selection Strategy - -#### 1️⃣ Recommended: Pure Erasure Mode (Default) -```bash -# No features needed, use default configuration -cargo run -cargo test -cargo bench -``` - -**Applicable Scenarios**: -- 📊 **Consistency Requirements**: Need completely predictable performance behavior -- 🔬 **Production Environment**: Best choice for most production scenarios -- 💾 **Memory Sensitive**: Strict requirements for memory usage patterns -- 🏗️ **Stable and Reliable**: Mature and stable implementation - -#### 2️⃣ High Performance Requirements: SIMD Mode -```bash -# Enable SIMD mode for maximum performance -cargo run --features reed-solomon-simd -cargo test --features reed-solomon-simd -cargo bench --features reed-solomon-simd -``` - -**Applicable Scenarios**: -- 🎯 **High Performance Scenarios**: Processing large amounts of data requiring maximum throughput -- 🚀 **Performance Optimization**: Want optimal performance for large data -- ⚡ **Speed Priority**: Scenarios with extremely high speed requirements -- 🏎️ **Compute Intensive**: CPU-intensive workloads - -### Configuration Optimization Recommendations - -#### Based on Data Size - -**Small Files Primarily** (< 64KB): -```toml -# Recommended to use default pure Erasure mode -# No special configuration needed, stable and reliable performance -``` - -**Large Files Primarily** (> 1MB): -```toml -# Recommend enabling SIMD mode for higher performance -# features = ["reed-solomon-simd"] -``` - -**Mixed Scenarios**: -```toml -# Default pure Erasure mode suits most scenarios -# For maximum performance, enable: features = ["reed-solomon-simd"] -``` - -#### Recommendations Based on Erasure Coding Configuration - -| Config | Small Data (< 64KB) | Large Data (> 1MB) | Recommended Mode | -|--------|-------------------|-------------------|------------------| -| 4+2 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) | -| 6+3 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) | -| 8+4 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) | -| 10+5 | Pure Erasure | Pure Erasure / SIMD Mode | Pure Erasure (Default) | - -### Production Environment Deployment Recommendations - -#### 1️⃣ Default Deployment Strategy -```bash -# Production environment recommended configuration: Use pure Erasure mode (default) -cargo build --release -``` - -**Advantages**: -- ✅ Maximum compatibility: Handle data of any size -- ✅ Stable and reliable: Mature implementation, predictable behavior -- ✅ Zero configuration: No complex performance tuning needed -- ✅ Memory efficient: Optimized memory usage patterns - -#### 2️⃣ High Performance Deployment Strategy -```bash -# High performance scenarios: Enable SIMD mode -cargo build --release --features reed-solomon-simd -``` - -**Advantages**: -- ✅ Optimal performance: SIMD instruction set optimization -- ✅ High throughput: Suitable for large data processing -- ✅ Performance oriented: Focuses on maximizing processing speed -- ✅ Modern hardware: Fully utilizes modern CPU features - -#### 2️⃣ Monitoring and Tuning -```rust -// Choose appropriate implementation based on specific scenarios -match data_size { - size if size > 1024 * 1024 => { - // Large data: Consider using SIMD mode - println!("Large data detected, SIMD mode recommended"); - } - _ => { - // General case: Use default Erasure mode - println!("Using default Erasure mode"); - } -} -``` - -#### 3️⃣ Performance Monitoring Metrics -- **Throughput Monitoring**: Monitor encoding/decoding data processing rates -- **Latency Analysis**: Analyze processing latency for different data sizes -- **CPU Utilization**: Observe CPU utilization efficiency of SIMD instructions -- **Memory Usage**: Monitor memory allocation patterns of different implementations - -## 🔧 Troubleshooting - -### Performance Issue Diagnosis - -#### Issue 1: Performance Not Meeting Expectations -**Symptom**: SIMD mode performance improvement not significant -**Cause**: Data size may not be suitable for SIMD optimization -**Solution**: -```rust -// Check shard size and data characteristics -let shard_size = data.len().div_ceil(data_shards); -println!("Shard size: {} bytes", shard_size); -if shard_size >= 1024 { - println!("Good candidate for SIMD optimization"); -} else { - println!("Consider using default Erasure mode"); -} -``` - -#### Issue 2: Compilation Errors -**Symptom**: SIMD-related compilation errors -**Cause**: Platform not supported or missing dependencies -**Solution**: -```bash -# Check platform support -cargo check --features reed-solomon-simd -# If failed, use default mode -cargo check -``` - -#### Issue 3: Abnormal Memory Usage -**Symptom**: Memory usage exceeds expectations -**Cause**: Memory alignment requirements of SIMD implementation -**Solution**: -```bash -# Use pure Erasure mode for comparison -cargo run --features reed-solomon-erasure -``` - -### Debugging Tips - -#### 1️⃣ Performance Comparison Testing -```bash -# Test pure Erasure mode performance -cargo bench --features reed-solomon-erasure - -# Test SIMD mode performance -cargo bench --features reed-solomon-simd -``` - -#### 2️⃣ Analyze Data Characteristics -```rust -// Statistics of data characteristics in your application -let data_sizes: Vec = data_samples.iter() - .map(|data| data.len()) - .collect(); - -let large_data_count = data_sizes.iter() - .filter(|&&size| size >= 1024 * 1024) - .count(); - -println!("Large data (>1MB): {}/{} ({}%)", - large_data_count, - data_sizes.len(), - large_data_count * 100 / data_sizes.len() -); -``` - -#### 3️⃣ Benchmark Comparison -```bash -# Generate detailed performance comparison report -./run_benchmarks.sh comparison - -# View HTML report to analyze performance differences -cd target/criterion && python3 -m http.server 8080 -``` - -## 📈 Performance Optimization Recommendations - -### Application Layer Optimization - -#### 1️⃣ Data Chunking Strategy -```rust -// Optimize data chunking for SIMD mode -const OPTIMAL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB -const MIN_EFFICIENT_SIZE: usize = 64 * 1024; // 64KB - -let block_size = if data.len() < MIN_EFFICIENT_SIZE { - data.len() // Small data can consider default mode -} else { - OPTIMAL_BLOCK_SIZE.min(data.len()) // Use optimal block size -}; -``` - -#### 2️⃣ Configuration Tuning -```rust -// Choose erasure coding configuration based on typical data size -let (data_shards, parity_shards) = if typical_file_size > 1024 * 1024 { - (8, 4) // Large files: more parallelism, utilize SIMD -} else { - (4, 2) // Small files: simple configuration, reduce overhead -}; -``` - -### System Layer Optimization - -#### 1️⃣ CPU Feature Detection -```bash -# Check CPU supported SIMD instruction sets -lscpu | grep -i flags -cat /proc/cpuinfo | grep -i flags | head -1 -``` - -#### 2️⃣ Memory Alignment Optimization -```rust -// Ensure data memory alignment to improve SIMD performance -use aligned_vec::AlignedVec; -let aligned_data = AlignedVec::::from_slice(&data); -``` - ---- - -💡 **Key Conclusions**: -- 🎯 **Pure Erasure mode (default) is the best general choice**: Stable and reliable, suitable for most scenarios -- 🚀 **SIMD mode suitable for high-performance scenarios**: Best choice for large data processing -- 📊 **Choose based on data characteristics**: Small data use Erasure, large data consider SIMD -- 🛡️ **Stability priority**: Production environments recommend using default Erasure mode \ No newline at end of file diff --git a/ecstore/IMPLEMENTATION_COMPARISON_ZH.md b/ecstore/IMPLEMENTATION_COMPARISON_ZH.md deleted file mode 100644 index 87fcb720..00000000 --- a/ecstore/IMPLEMENTATION_COMPARISON_ZH.md +++ /dev/null @@ -1,333 +0,0 @@ -# Reed-Solomon 实现对比分析 - -## 🔍 问题分析 - -随着SIMD模式的优化设计，我们提供了高性能的Reed-Solomon实现。现在系统能够在不同场景下提供最优的性能表现。 - -## 📊 实现模式对比 - -### 🏛️ 纯 Erasure 模式（默认，推荐） - -**默认配置**: 不指定任何 feature，使用稳定的 reed-solomon-erasure 实现 - -**特点**: -- ✅ **广泛兼容**: 支持任意分片大小，从字节级到 GB 级 -- 📈 **稳定性能**: 性能对分片大小不敏感，可预测 -- 🔧 **生产就绪**: 成熟稳定的实现，已在生产环境广泛使用 -- 💾 **内存高效**: 优化的内存使用模式 -- 🎯 **一致性**: 在所有场景下行为完全一致 - -**使用场景**: -- 大多数生产环境的默认选择 -- 需要完全一致和可预测的性能行为 -- 对性能变化敏感的系统 -- 主要处理小文件或小分片的场景 -- 需要严格的内存使用控制 - -### 🎯 SIMD模式（`reed-solomon-simd` feature） - -**配置**: `--features reed-solomon-simd` - -**特点**: -- 🚀 **高性能SIMD**: 使用SIMD指令集进行高性能编码解码 -- 🎯 **性能导向**: 专注于最大化处理性能 -- ⚡ **大数据优化**: 适合大数据量处理的高吞吐量场景 -- 🏎️ **速度优先**: 为性能关键型应用设计 - -**使用场景**: -- 需要最大化性能的应用场景 -- 处理大量数据的高吞吐量系统 -- 对性能要求极高的场景 -- CPU密集型工作负载 - -## 📏 分片大小与性能对比 - -不同配置下的性能表现： - -| 数据大小 | 配置 | 分片大小 | 纯 Erasure 模式（默认） | SIMD模式策略 | 性能对比 | -|---------|------|----------|------------------------|-------------|----------| -| 1KB | 4+2 | 256字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 | -| 1KB | 6+3 | 171字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 | -| 1KB | 8+4 | 128字节 | Erasure 实现 | SIMD 实现 | SIMD可能更快 | -| 64KB | 4+2 | 16KB | Erasure 实现 | SIMD 优化 | SIMD模式更快 | -| 64KB | 6+3 | 10.7KB | Erasure 实现 | SIMD 优化 | SIMD模式更快 | -| 1MB | 4+2 | 256KB | Erasure 实现 | SIMD 优化 | SIMD模式显著更快 | -| 16MB | 8+4 | 2MB | Erasure 实现 | SIMD 优化 | SIMD模式大幅领先 | - -## 🎯 基准测试结果解读 - -### 纯 Erasure 模式示例（默认） ✅ - -``` -encode_comparison/implementation/1KB_6+3_erasure - time: [245.67 ns 256.78 ns 267.89 ns] - thrpt: [3.73 GiB/s 3.89 GiB/s 4.07 GiB/s] - -💡 一致的 Erasure 性能 - 所有配置都使用相同实现 -``` - -``` -encode_comparison/implementation/64KB_4+2_erasure - time: [2.3456 μs 2.4567 μs 2.5678 μs] - thrpt: [23.89 GiB/s 24.65 GiB/s 25.43 GiB/s] - -💡 稳定可靠的性能 - 适合大多数生产场景 -``` - -### SIMD模式成功示例 ✅ - -**大分片 SIMD 优化**: -``` -encode_comparison/implementation/64KB_4+2_simd - time: [1.2345 μs 1.2567 μs 1.2789 μs] - thrpt: [47.89 GiB/s 48.65 GiB/s 49.43 GiB/s] - -💡 使用 SIMD 优化 - 分片大小: 16KB，高性能处理 -``` - -**小分片 SIMD 处理**: -``` -encode_comparison/implementation/1KB_6+3_simd - time: [234.56 ns 245.67 ns 256.78 ns] - thrpt: [3.89 GiB/s 4.07 GiB/s 4.26 GiB/s] - -💡 SIMD 处理小分片 - 分片大小: 171字节 -``` - -## 🛠️ 使用指南 - -### 选择策略 - -#### 1️⃣ 推荐：纯 Erasure 模式（默认） -```bash -# 无需指定 feature，使用默认配置 -cargo run -cargo test -cargo bench -``` - -**适用场景**: -- 📊 **一致性要求**: 需要完全可预测的性能行为 -- 🔬 **生产环境**: 大多数生产场景的最佳选择 -- 💾 **内存敏感**: 对内存使用模式有严格要求 -- 🏗️ **稳定可靠**: 成熟稳定的实现 - -#### 2️⃣ 高性能需求：SIMD模式 -```bash -# 启用SIMD模式获得最大性能 -cargo run --features reed-solomon-simd -cargo test --features reed-solomon-simd -cargo bench --features reed-solomon-simd -``` - -**适用场景**: -- 🎯 **高性能场景**: 处理大量数据需要最大吞吐量 -- 🚀 **性能优化**: 希望在大数据时获得最佳性能 -- ⚡ **速度优先**: 对处理速度有极高要求的场景 -- 🏎️ **计算密集**: CPU密集型工作负载 - -### 配置优化建议 - -#### 针对数据大小的配置 - -**小文件为主** (< 64KB): -```toml -# 推荐使用默认纯 Erasure 模式 -# 无需特殊配置，性能稳定可靠 -``` - -**大文件为主** (> 1MB): -```toml -# 建议启用SIMD模式获得更高性能 -# features = ["reed-solomon-simd"] -``` - -**混合场景**: -```toml -# 默认纯 Erasure 模式适合大多数场景 -# 如需最大性能可启用: features = ["reed-solomon-simd"] -``` - -#### 针对纠删码配置的建议 - -| 配置 | 小数据 (< 64KB) | 大数据 (> 1MB) | 推荐模式 | -|------|----------------|----------------|----------| -| 4+2 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure（默认） | -| 6+3 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure（默认） | -| 8+4 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure（默认） | -| 10+5 | 纯 Erasure | 纯 Erasure / SIMD模式 | 纯 Erasure（默认） | - -### 生产环境部署建议 - -#### 1️⃣ 默认部署策略 -```bash -# 生产环境推荐配置：使用纯 Erasure 模式（默认） -cargo build --release -``` - -**优势**: -- ✅ 最大兼容性：处理任意大小数据 -- ✅ 稳定可靠：成熟的实现，行为可预测 -- ✅ 零配置：无需复杂的性能调优 -- ✅ 内存高效：优化的内存使用模式 - -#### 2️⃣ 高性能部署策略 -```bash -# 高性能场景：启用SIMD模式 -cargo build --release --features reed-solomon-simd -``` - -**优势**: -- ✅ 最优性能：SIMD指令集优化 -- ✅ 高吞吐量：适合大数据处理 -- ✅ 性能导向：专注于最大化处理速度 -- ✅ 现代硬件：充分利用现代CPU特性 - -#### 2️⃣ 监控和调优 -```rust -// 根据具体场景选择合适的实现 -match data_size { - size if size > 1024 * 1024 => { - // 大数据：考虑使用SIMD模式 - println!("Large data detected, SIMD mode recommended"); - } - _ => { - // 一般情况：使用默认Erasure模式 - println!("Using default Erasure mode"); - } -} -``` - -#### 3️⃣ 性能监控指标 -- **吞吐量监控**: 监控编码/解码的数据处理速率 -- **延迟分析**: 分析不同数据大小的处理延迟 -- **CPU使用率**: 观察SIMD指令的CPU利用效率 -- **内存使用**: 监控不同实现的内存分配模式 - -## 🔧 故障排除 - -### 性能问题诊断 - -#### 问题1: 性能不符合预期 -**现象**: SIMD模式性能提升不明显 -**原因**: 可能数据大小不适合SIMD优化 -**解决**: -```rust -// 检查分片大小和数据特征 -let shard_size = data.len().div_ceil(data_shards); -println!("Shard size: {} bytes", shard_size); -if shard_size >= 1024 { - println!("Good candidate for SIMD optimization"); -} else { - println!("Consider using default Erasure mode"); -} -``` - -#### 问题2: 编译错误 -**现象**: SIMD相关的编译错误 -**原因**: 平台不支持或依赖缺失 -**解决**: -```bash -# 检查平台支持 -cargo check --features reed-solomon-simd -# 如果失败，使用默认模式 -cargo check -``` - -#### 问题3: 内存使用异常 -**现象**: 内存使用超出预期 -**原因**: SIMD实现的内存对齐要求 -**解决**: -```bash -# 使用纯 Erasure 模式进行对比 -cargo run --features reed-solomon-erasure -``` - -### 调试技巧 - -#### 1️⃣ 性能对比测试 -```bash -# 测试纯 Erasure 模式性能 -cargo bench --features reed-solomon-erasure - -# 测试SIMD模式性能 -cargo bench --features reed-solomon-simd -``` - -#### 2️⃣ 分析数据特征 -```rust -// 统计你的应用中的数据特征 -let data_sizes: Vec = data_samples.iter() - .map(|data| data.len()) - .collect(); - -let large_data_count = data_sizes.iter() - .filter(|&&size| size >= 1024 * 1024) - .count(); - -println!("Large data (>1MB): {}/{} ({}%)", - large_data_count, - data_sizes.len(), - large_data_count * 100 / data_sizes.len() -); -``` - -#### 3️⃣ 基准测试对比 -```bash -# 生成详细的性能对比报告 -./run_benchmarks.sh comparison - -# 查看 HTML 报告分析性能差异 -cd target/criterion && python3 -m http.server 8080 -``` - -## 📈 性能优化建议 - -### 应用层优化 - -#### 1️⃣ 数据分块策略 -```rust -// 针对SIMD模式优化数据分块 -const OPTIMAL_BLOCK_SIZE: usize = 1024 * 1024; // 1MB -const MIN_EFFICIENT_SIZE: usize = 64 * 1024; // 64KB - -let block_size = if data.len() < MIN_EFFICIENT_SIZE { - data.len() // 小数据可以考虑默认模式 -} else { - OPTIMAL_BLOCK_SIZE.min(data.len()) // 使用最优块大小 -}; -``` - -#### 2️⃣ 配置调优 -```rust -// 根据典型数据大小选择纠删码配置 -let (data_shards, parity_shards) = if typical_file_size > 1024 * 1024 { - (8, 4) // 大文件：更多并行度，利用 SIMD -} else { - (4, 2) // 小文件：简单配置，减少开销 -}; -``` - -### 系统层优化 - -#### 1️⃣ CPU 特性检测 -```bash -# 检查 CPU 支持的 SIMD 指令集 -lscpu | grep -i flags -cat /proc/cpuinfo | grep -i flags | head -1 -``` - -#### 2️⃣ 内存对齐优化 -```rust -// 确保数据内存对齐以提升 SIMD 性能 -use aligned_vec::AlignedVec; -let aligned_data = AlignedVec::::from_slice(&data); -``` - ---- - -💡 **关键结论**: -- 🎯 **纯Erasure模式（默认）是最佳通用选择**：稳定可靠，适合大多数场景 -- 🚀 **SIMD模式适合高性能场景**：大数据处理的最佳选择 -- 📊 **根据数据特征选择**：小数据用Erasure，大数据考虑SIMD -- 🛡️ **稳定性优先**：生产环境建议使用默认Erasure模式 \ No newline at end of file