fix: add missing OSSUTIL_BIN variable in linux case branch (#141)

* fix: improve ossutil install logic in GitHub Actions workflow

* wip

* wip

* fix: add missing OSSUTIL_BIN variable in linux case branch

.cursorrules

@@ -5,15 +5,18 @@
 ### 🚨 NEVER COMMIT DIRECTLY TO MASTER/MAIN BRANCH 🚨
 
 - **This is the most important rule - NEVER modify code directly on main or master branch**
+- **ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO EXCEPTIONS**
 - **Always work on feature branches and use pull requests for all changes**
 - **Any direct commits to master/main branch are strictly forbidden**
+- **Pull requests are the ONLY way to merge code to main branch**
 - Before starting any development, always:
   1. `git checkout main` (switch to main branch)
   2. `git pull` (get latest changes)
   3. `git checkout -b feat/your-feature-name` (create and switch to feature branch)
   4. Make your changes on the feature branch
   5. Commit and push to the feature branch
-  6. Create a pull request for review
+  6. **Create a pull request for review - THIS IS MANDATORY**
+  7. **Wait for PR approval and merge through GitHub interface only**
 
 ## Project Overview
 
@@ -817,6 +820,7 @@ These rules should serve as guiding principles when developing the RustFS projec
 
 - **🚨 CRITICAL: NEVER modify code directly on main or master branch - THIS IS ABSOLUTELY FORBIDDEN 🚨**
 - **⚠️ ANY DIRECT COMMITS TO MASTER/MAIN WILL BE REJECTED AND MUST BE REVERTED IMMEDIATELY ⚠️**
+- **🔒 ALL CHANGES MUST GO THROUGH PULL REQUESTS - NO DIRECT COMMITS TO MAIN UNDER ANY CIRCUMSTANCES 🔒**
 - **Always work on feature branches - NO EXCEPTIONS**
 - Always check the .cursorrules file before starting to ensure you understand the project guidelines
 - **MANDATORY workflow for ALL changes:**
@@ -826,13 +830,39 @@ These rules should serve as guiding principles when developing the RustFS projec
    4. Make your changes ONLY on the feature branch
    5. Test thoroughly before committing
    6. Commit and push to the feature branch
-   7. Create a pull request for code review
+   7. **Create a pull request for code review - THIS IS THE ONLY WAY TO MERGE TO MAIN**
+   8. **Wait for PR approval before merging - NEVER merge your own PRs without review**
 - Use descriptive branch names following the pattern: `feat/feature-name`, `fix/issue-name`, `refactor/component-name`, etc.
 - **Double-check current branch before ANY commit: `git branch` to ensure you're NOT on main/master**
-- Ensure all changes are made on feature branches and merged through pull requests
+- **Pull Request Requirements:**
+  - All changes must be submitted via PR regardless of size or urgency
+  - PRs must include comprehensive description and testing information
+  - PRs must pass all CI/CD checks before merging
+  - PRs require at least one approval from code reviewers
+  - Even hotfixes and emergency changes must go through PR process
+- **Enforcement:**
+  - Main branch should be protected with branch protection rules
+  - Direct pushes to main should be blocked by repository settings
+  - Any accidental direct commits to main must be immediately reverted via PR
 
 #### Development Workflow
 
+## 🎯 **Core Development Principles**
+
+- **🔴 Every change must be precise - don't modify unless you're confident**
+  - Carefully analyze code logic and ensure complete understanding before making changes
+  - When uncertain, prefer asking users or consulting documentation over blind modifications
+  - Use small iterative steps, modify only necessary parts at a time
+  - Evaluate impact scope before changes to ensure no new issues are introduced
+
+- **🚀 GitHub PR creation prioritizes gh command usage**
+  - Prefer using `gh pr create` command to create Pull Requests
+  - Avoid having users manually create PRs through web interface
+  - Provide clear and professional PR titles and descriptions
+  - Using `gh` commands ensures better integration and automation
+
+## 📝 **Code Quality Requirements**
+
 - Use English for all code comments, documentation, and variable names
 - Write meaningful and descriptive names for variables, functions, and methods
 - Avoid meaningless test content like "debug 111" or placeholder values

.github/actions/setup/action.yml

@@ -72,6 +72,7 @@ runs:
       uses: arduino/setup-protoc@v3
       with:
         version: "31.1"
+        repo-token: ${{ inputs.github-token }}
 
     - name: Install flatc
       uses: Nugine/setup-flatc@v1

.github/workflows/build.yml

@@ -18,22 +18,36 @@ on:
   push:
     tags: ["*"]
     branches: [main]
-    paths:
-      - "rustfs/**"
-      - "cli/**"
-      - "crates/**"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/workflows/build.yml"
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - ".github/**"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "LICENSE*"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".gitignore"
+      - ".dockerignore"
   pull_request:
     branches: [main]
-    paths:
-      - "rustfs/**"
-      - "cli/**"
-      - "crates/**"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/workflows/build.yml"
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - ".github/**"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "LICENSE*"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".gitignore"
+      - ".dockerignore"
   schedule:
     - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
   workflow_dispatch:
@@ -51,26 +65,9 @@ env:
   CARGO_INCREMENTAL: 0
 
 jobs:
-  # First layer: GitHub Actions level optimization (handling duplicates and concurrency)
-  skip-duplicate:
-    name: Skip Duplicate Actions
-    runs-on: ubuntu-latest
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - name: Skip duplicate actions
-        id: skip_check
-        uses: fkirc/skip-duplicate-actions@v5
-        with:
-          concurrent_skipping: "same_content_newer"
-          cancel_others: true
-          paths_ignore: '["*.md", "docs/**", "deploy/**", "scripts/dev_*.sh"]'
-
   # Second layer: Business logic level checks (handling build strategy)
   build-check:
     name: Build Strategy Check
-    needs: skip-duplicate
-    if: needs.skip-duplicate.outputs.should_skip != 'true'
     runs-on: ubuntu-latest
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
@@ -91,9 +88,11 @@ jobs:
             build_type="development"
           fi
 
+          # Always build for tag pushes (version releases)
           if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
             should_build=true
             build_type="release"
+            echo "🏷️  Tag detected: forcing release build"
           fi
 
           echo "should_build=$should_build" >> $GITHUB_OUTPUT
@@ -103,8 +102,8 @@ jobs:
   # Build RustFS binaries
   build-rustfs:
     name: Build RustFS
-    needs: [skip-duplicate, build-check]
-    if: needs.skip-duplicate.outputs.should_skip != 'true' && needs.build-check.outputs.should_build == 'true'
+    needs: [build-check]
+    if: needs.build-check.outputs.should_build == 'true'
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     env:
@@ -113,15 +112,33 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # Linux builds
           - os: ubuntu-latest
             target: x86_64-unknown-linux-musl
             cross: false
+            platform: linux
           - os: ubuntu-latest
             target: aarch64-unknown-linux-musl
             cross: true
+            platform: linux
+          # macOS builds
           - os: macos-latest
             target: aarch64-apple-darwin
             cross: false
+            platform: macos
+          - os: macos-latest
+            target: x86_64-apple-darwin
+            cross: false
+            platform: macos
+          # # Windows builds (temporarily disabled)
+          # - os: windows-latest
+          #   target: x86_64-pc-windows-msvc
+          #   cross: false
+          #   platform: windows
+          # - os: windows-latest
+          #   target: aarch64-pc-windows-msvc
+          #   cross: true
+          #   platform: windows
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -141,149 +158,149 @@ jobs:
       - name: Download static console assets
         run: |
           mkdir -p ./rustfs/static
-          curl -L "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" \
-            -o console.zip --retry 3 --retry-delay 5 --max-time 300
-          unzip -o console.zip -d ./rustfs/static
-          rm console.zip
+          if [[ "${{ matrix.platform }}" == "windows" ]]; then
+            curl.exe -L "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" -o console.zip --retry 3 --retry-delay 5 --max-time 300
+            if [[ $? -eq 0 ]]; then
+              unzip -o console.zip -d ./rustfs/static
+              rm console.zip
+            else
+              echo "Warning: Failed to download console assets, continuing without them"
+              echo "// Static assets not available" > ./rustfs/static/empty.txt
+            fi
+          else
+            curl -L "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" \
+              -o console.zip --retry 3 --retry-delay 5 --max-time 300
+            if [[ $? -eq 0 ]]; then
+              unzip -o console.zip -d ./rustfs/static
+              rm console.zip
+            else
+              echo "Warning: Failed to download console assets, continuing without them"
+              echo "// Static assets not available" > ./rustfs/static/empty.txt
+            fi
+          fi
 
       - name: Build RustFS
         run: |
+          # Force rebuild by touching build.rs
           touch rustfs/build.rs
 
           if [[ "${{ matrix.cross }}" == "true" ]]; then
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
+            if [[ "${{ matrix.platform }}" == "windows" ]]; then
+              # Use cross for Windows ARM64
+              cargo install cross --git https://github.com/cross-rs/cross
+              cross build --release --target ${{ matrix.target }} -p rustfs --bins
+            else
+              # Use zigbuild for Linux ARM64
+              cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
+            fi
           else
             cargo build --release --target ${{ matrix.target }} -p rustfs --bins
           fi
 
       - name: Create release package
         id: package
+        shell: bash
         run: |
           PACKAGE_NAME="rustfs-${{ matrix.target }}"
-          mkdir -p "${PACKAGE_NAME}"/{bin,docs}
 
-          # Copy binary
-          if [[ "${{ matrix.target }}" == *"windows"* ]]; then
-            cp target/${{ matrix.target }}/release/rustfs.exe "${PACKAGE_NAME}/bin/"
-          else
-            cp target/${{ matrix.target }}/release/rustfs "${PACKAGE_NAME}/bin/"
-            chmod +x "${PACKAGE_NAME}/bin/rustfs"
+          # Create zip packages for all platforms
+          # Ensure zip is available
+          if ! command -v zip &> /dev/null; then
+            if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+              sudo apt-get update && sudo apt-get install -y zip
+            fi
           fi
 
-          # Copy documentation
-          [ -f "LICENSE" ] && cp LICENSE "${PACKAGE_NAME}/docs/"
-          [ -f "README.md" ] && cp README.md "${PACKAGE_NAME}/docs/"
-
-          # Create archive
-          tar -czf "${PACKAGE_NAME}.tar.gz" "${PACKAGE_NAME}"
-
+          cd target/${{ matrix.target }}/release
+          zip "../../../${PACKAGE_NAME}.zip" rustfs
+          cd ../../..
           echo "package_name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT
-          echo "Package created: ${PACKAGE_NAME}.tar.gz"
+          echo "package_file=${PACKAGE_NAME}.zip" >> $GITHUB_OUTPUT
+          echo "Package created: ${PACKAGE_NAME}.zip"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.package.outputs.package_name }}
-          path: ${{ steps.package.outputs.package_name }}.tar.gz
+          path: ${{ steps.package.outputs.package_file }}
           retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 7 }}
 
-  # Build GUI (only for releases)
-  build-gui:
-    name: Build GUI
-    needs: [skip-duplicate, build-check, build-rustfs]
-    if: needs.skip-duplicate.outputs.should_skip != 'true' && needs.build-check.outputs.build_type == 'release'
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            target: x86_64-unknown-linux-musl
-            platform: linux
-          - os: macos-latest
-            target: aarch64-apple-darwin
-            platform: macos
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          target: ${{ matrix.target }}
-          cache-shared-key: gui-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Download RustFS binary
-        uses: actions/download-artifact@v4
-        with:
-          name: rustfs-${{ matrix.target }}
-          path: ./artifacts
-
-      - name: Prepare embedded binary
-        run: |
-          mkdir -p ./cli/rustfs-gui/embedded-rustfs/
-          tar -xzf ./artifacts/rustfs-${{ matrix.target }}.tar.gz -C ./artifacts/
-          cp ./artifacts/rustfs-${{ matrix.target }}/bin/rustfs ./cli/rustfs-gui/embedded-rustfs/
-
-      - name: Install Dioxus CLI
-        uses: taiki-e/cache-cargo-install-action@v2
-        with:
-          tool: dioxus-cli
-
-      - name: Build GUI
-        working-directory: ./cli/rustfs-gui
+      - name: Upload to Aliyun OSS
+        if: needs.build-check.outputs.build_type == 'release' && env.OSS_ACCESS_KEY_ID != ''
+        env:
+          OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
+          OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
+          OSS_REGION: cn-beijing
+          OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
         run: |
+          # Install ossutil (platform-specific)
+          OSSUTIL_VERSION="2.1.1"
           case "${{ matrix.platform }}" in
-            "linux")
-              dx bundle --platform linux --package-types deb --package-types appimage --release
+            linux)
+              if [[ "$(uname -m)" == "arm64" ]]; then
+                ARCH="arm64"
+              else
+                ARCH="amd64"
+              fi
+              OSSUTIL_ZIP="ossutil-${OSSUTIL_VERSION}-linux-${ARCH}.zip"
+              OSSUTIL_DIR="ossutil-${OSSUTIL_VERSION}-linux-${ARCH}"
+
+              curl -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}"
+              unzip "$OSSUTIL_ZIP"
+              mv "${OSSUTIL_DIR}/ossutil" /usr/local/bin/
+              rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
+              chmod +x /usr/local/bin/ossutil
+              OSSUTIL_BIN=ossutil
               ;;
-            "macos")
-              dx bundle --platform macos --package-types dmg --release
+            macos)
+              if [[ "$(uname -m)" == "arm64" ]]; then
+                ARCH="arm64"
+              else
+                ARCH="amd64"
+              fi
+              OSSUTIL_ZIP="ossutil-${OSSUTIL_VERSION}-mac-${ARCH}.zip"
+              OSSUTIL_DIR="ossutil-${OSSUTIL_VERSION}-mac-${ARCH}"
+
+              curl -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}"
+              unzip "$OSSUTIL_ZIP"
+              mv "${OSSUTIL_DIR}/ossutil" /usr/local/bin/
+              rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
+              chmod +x /usr/local/bin/ossutil
+              OSSUTIL_BIN=ossutil
               ;;
           esac
 
-      - name: Package GUI
-        id: gui_package
-        run: |
-          GUI_PACKAGE="rustfs-gui-${{ matrix.target }}"
-          mkdir -p "${GUI_PACKAGE}"
+          # Upload the package file directly to OSS
+          echo "Uploading ${{ steps.package.outputs.package_file }} to OSS..."
+          $OSSUTIL_BIN cp "${{ steps.package.outputs.package_file }}" oss://rustfs-artifacts/artifacts/rustfs/ --force
 
-          # Copy GUI bundles
-          if [[ -d "cli/rustfs-gui/dist/bundle" ]]; then
-            cp -r cli/rustfs-gui/dist/bundle/* "${GUI_PACKAGE}/"
+          # Create latest.json (only for the first Linux build to avoid duplication)
+          if [[ "${{ matrix.target }}" == "x86_64-unknown-linux-musl" ]]; then
+            VERSION="${GITHUB_REF#refs/tags/v}"
+            echo "{\"version\":\"${VERSION}\",\"release_date\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > latest.json
+            $OSSUTIL_BIN cp latest.json oss://rustfs-version/latest.json --force
           fi
 
-          tar -czf "${GUI_PACKAGE}.tar.gz" "${GUI_PACKAGE}"
-          echo "gui_package=${GUI_PACKAGE}" >> $GITHUB_OUTPUT
-
-      - name: Upload GUI artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.gui_package.outputs.gui_package }}
-          path: ${{ steps.gui_package.outputs.gui_package }}.tar.gz
-          retention-days: 30
-
   # Release management
   release:
     name: GitHub Release
-    needs: [skip-duplicate, build-check, build-rustfs, build-gui]
-    if: always() && needs.skip-duplicate.outputs.should_skip != 'true' && needs.build-check.outputs.build_type == 'release'
+    needs: [build-check, build-rustfs]
+    if: always() && needs.build-check.outputs.build_type == 'release'
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Download all artifacts
         uses: actions/download-artifact@v4
         with:
           path: ./release-artifacts
 
-      - name: Prepare release
+      - name: Prepare release assets
         id: release_prep
         run: |
           VERSION="${GITHUB_REF#refs/tags/}"
@@ -294,72 +311,180 @@ jobs:
 
           # Organize artifacts
           mkdir -p ./release-files
-          find ./release-artifacts -name "*.tar.gz" -exec cp {} ./release-files/ \;
 
-          # Create release notes
-          cat > release_notes.md << EOF
-          ## RustFS ${VERSION_CLEAN}
+          # Copy all artifacts (.zip files)
+          find ./release-artifacts -name "*.zip" -exec cp {} ./release-files/ \;
 
-          ### 🚀 Downloads
+          # Generate checksums for all files
+          cd ./release-files
+          if ls *.zip >/dev/null 2>&1; then
+            sha256sum *.zip >> SHA256SUMS
+            sha512sum *.zip >> SHA512SUMS
+          fi
+          cd ..
 
-          **Linux:**
-          - \`rustfs-x86_64-unknown-linux-musl.tar.gz\` - Linux x86_64 (static)
-          - \`rustfs-aarch64-unknown-linux-musl.tar.gz\` - Linux ARM64 (static)
-
-          **macOS:**
-          - \`rustfs-aarch64-apple-darwin.tar.gz\` - macOS Apple Silicon
-
-          **GUI Applications:**
-          - \`rustfs-gui-*.tar.gz\` - GUI applications
-
-          ### 📦 Installation
-
-          1. Download the appropriate binary for your platform
-          2. Extract: \`tar -xzf rustfs-*.tar.gz\`
-          3. Install: \`sudo cp rustfs-*/bin/rustfs /usr/local/bin/\`
-
-          ### 🔗 Mirror Downloads
-
-          - [OSS Mirror](https://rustfs-artifacts.oss-cn-beijing.aliyuncs.com/artifacts/rustfs/)
-          EOF
+          # Display what we're releasing
+          echo "=== Release Files ==="
+          ls -la ./release-files/
 
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
-        with:
-          tag_name: ${{ steps.release_prep.outputs.version }}
-          name: "RustFS ${{ steps.release_prep.outputs.version_clean }}"
-          body_path: release_notes.md
-          files: ./release-files/*.tar.gz
-          draft: false
-          prerelease: ${{ contains(steps.release_prep.outputs.version, 'alpha') || contains(steps.release_prep.outputs.version, 'beta') || contains(steps.release_prep.outputs.version, 'rc') }}
-
-  # Upload to OSS (optional)
-  upload-oss:
-    name: Upload to OSS
-    needs: [skip-duplicate, build-check, build-rustfs]
-    if: always() && needs.skip-duplicate.outputs.should_skip != 'true' && needs.build-check.outputs.build_type == 'release' && needs.build-rustfs.result == 'success'
-    runs-on: ubuntu-latest
-    env:
-      OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
-      OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: ./artifacts
-
-      - name: Upload to Aliyun OSS
-        if: ${{ env.OSS_ACCESS_KEY_ID != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
-          # Install ossutil
-          curl -o ossutil.zip https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-linux-amd64.zip
-          unzip ossutil.zip
-          sudo mv ossutil-*/ossutil /usr/local/bin/
+          VERSION="${{ steps.release_prep.outputs.version }}"
+          VERSION_CLEAN="${{ steps.release_prep.outputs.version_clean }}"
 
-          # Upload files
-          find ./artifacts -name "*.tar.gz" -exec ossutil cp {} oss://rustfs-artifacts/artifacts/rustfs/ --force \;
+          # Check if release already exists
+          if gh release view "$VERSION" >/dev/null 2>&1; then
+            echo "Release $VERSION already exists, skipping creation"
+          else
+            # Get release notes from tag message
+            RELEASE_NOTES=$(git tag -l --format='%(contents)' "${VERSION}")
+            if [[ -z "$RELEASE_NOTES" || "$RELEASE_NOTES" =~ ^[[:space:]]*$ ]]; then
+              RELEASE_NOTES="Release ${VERSION_CLEAN}"
+            fi
 
-          # Create latest.json
-          VERSION="${GITHUB_REF#refs/tags/v}"
-          echo "{\"version\":\"${VERSION}\",\"release_date\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > latest.json
-          ossutil cp latest.json oss://rustfs-version/latest.json --force
+            # Determine if this is a prerelease
+            PRERELEASE_FLAG=""
+            if [[ "$VERSION" == *"alpha"* ]] || [[ "$VERSION" == *"beta"* ]] || [[ "$VERSION" == *"rc"* ]]; then
+              PRERELEASE_FLAG="--prerelease"
+            fi
+
+            # Create the release only if it doesn't exist
+            gh release create "$VERSION" \
+              --title "RustFS $VERSION_CLEAN" \
+              --notes "$RELEASE_NOTES" \
+              $PRERELEASE_FLAG
+          fi
+
+      - name: Upload release assets
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          VERSION="${{ steps.release_prep.outputs.version }}"
+
+          cd ./release-files
+
+          # Upload all binary files
+          for file in *.zip; do
+            if [[ -f "$file" ]]; then
+              echo "Uploading $file..."
+              gh release upload "$VERSION" "$file" --clobber
+            fi
+          done
+
+          # Upload checksum files
+          if [[ -f "SHA256SUMS" ]]; then
+            echo "Uploading SHA256SUMS..."
+            gh release upload "$VERSION" "SHA256SUMS" --clobber
+          fi
+
+          if [[ -f "SHA512SUMS" ]]; then
+            echo "Uploading SHA512SUMS..."
+            gh release upload "$VERSION" "SHA512SUMS" --clobber
+          fi
+
+      - name: Update release notes
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          VERSION="${{ steps.release_prep.outputs.version }}"
+          VERSION_CLEAN="${{ steps.release_prep.outputs.version_clean }}"
+
+          # Check if release already has custom notes (not auto-generated)
+          EXISTING_NOTES=$(gh release view "$VERSION" --json body --jq '.body' 2>/dev/null || echo "")
+
+          # Only update if release notes are empty or auto-generated
+          if [[ -z "$EXISTING_NOTES" ]] || [[ "$EXISTING_NOTES" == *"Release ${VERSION_CLEAN}"* ]]; then
+            echo "Updating release notes for $VERSION"
+
+            # Get original release notes from tag
+            ORIGINAL_NOTES=$(git tag -l --format='%(contents)' "${VERSION}")
+            if [[ -z "$ORIGINAL_NOTES" || "$ORIGINAL_NOTES" =~ ^[[:space:]]*$ ]]; then
+              ORIGINAL_NOTES="Release ${VERSION_CLEAN}"
+            fi
+
+            # Create comprehensive release notes
+            cat > enhanced_notes.md << EOF
+            ## RustFS ${VERSION_CLEAN}
+
+            ${ORIGINAL_NOTES}
+
+            ---
+
+            ### 🚀 Quick Download
+
+            **Linux (Static Binaries - No Dependencies):**
+            \`\`\`bash
+            # x86_64 (Intel/AMD)
+            curl -LO https://github.com/rustfs/rustfs/releases/download/${VERSION}/rustfs-x86_64-unknown-linux-musl.zip
+            unzip rustfs-x86_64-unknown-linux-musl.zip
+            sudo mv rustfs /usr/local/bin/
+
+            # ARM64 (Graviton, Apple Silicon VMs)
+            curl -LO https://github.com/rustfs/rustfs/releases/download/${VERSION}/rustfs-aarch64-unknown-linux-musl.zip
+            unzip rustfs-aarch64-unknown-linux-musl.zip
+            sudo mv rustfs /usr/local/bin/
+            \`\`\`
+
+            **macOS:**
+            \`\`\`bash
+            # Apple Silicon (M1/M2/M3)
+            curl -LO https://github.com/rustfs/rustfs/releases/download/${VERSION}/rustfs-aarch64-apple-darwin.zip
+            unzip rustfs-aarch64-apple-darwin.zip
+            sudo mv rustfs /usr/local/bin/
+
+            # Intel
+            curl -LO https://github.com/rustfs/rustfs/releases/download/${VERSION}/rustfs-x86_64-apple-darwin.zip
+            unzip rustfs-x86_64-apple-darwin.zip
+            sudo mv rustfs /usr/local/bin/
+            \`\`\`
+
+            ### 📁 Available Downloads
+
+            | Platform | Architecture | File | Description |
+            |----------|-------------|------|-------------|
+            | Linux | x86_64 | \`rustfs-x86_64-unknown-linux-musl.zip\` | Static binary, no dependencies |
+            | Linux | ARM64 | \`rustfs-aarch64-unknown-linux-musl.zip\` | Static binary, no dependencies |
+            | macOS | Apple Silicon | \`rustfs-aarch64-apple-darwin.zip\` | Native binary, ZIP archive |
+            | macOS | Intel | \`rustfs-x86_64-apple-darwin.zip\` | Native binary, ZIP archive |
+
+            ### 🔐 Verification
+
+            Download checksums and verify your download:
+            \`\`\`bash
+            # Download checksums
+            curl -LO https://github.com/rustfs/rustfs/releases/download/${VERSION}/SHA256SUMS
+
+            # Verify (Linux)
+            sha256sum -c SHA256SUMS --ignore-missing
+
+            # Verify (macOS)
+            shasum -a 256 -c SHA256SUMS --ignore-missing
+            \`\`\`
+
+            ### 🛠️ System Requirements
+
+            - **Linux**: Any distribution with glibc 2.17+ (CentOS 7+, Ubuntu 16.04+)
+            - **macOS**: 10.15+ (Catalina or later)
+            - **Windows**: Windows 10 version 1809 or later
+
+            ### 📚 Documentation
+
+            - [Installation Guide](https://github.com/rustfs/rustfs#installation)
+            - [Quick Start](https://github.com/rustfs/rustfs#quick-start)
+            - [Configuration](https://github.com/rustfs/rustfs/blob/main/docs/)
+            - [API Documentation](https://docs.rs/rustfs)
+
+            ### 🆘 Support
+
+            - 🐛 [Report Issues](https://github.com/rustfs/rustfs/issues)
+            - 💬 [Community Discussions](https://github.com/rustfs/rustfs/discussions)
+            - 📖 [Documentation](https://github.com/rustfs/rustfs/tree/main/docs)
+            EOF
+
+            # Update the release with enhanced notes
+            gh release edit "$VERSION" --notes-file enhanced_notes.md
+          else
+            echo "Release $VERSION already has custom notes, skipping update to preserve manual edits"
+          fi

.github/workflows/ci.yml

@@ -80,6 +80,8 @@ jobs:
           concurrent_skipping: "same_content_newer"
           cancel_others: true
           paths_ignore: '["*.md", "docs/**", "deploy/**"]'
+          # Never skip release events and tag pushes
+          do_not_skip: '["release", "push"]'
 
   test-and-lint:
     name: Test and Lint

.github/workflows/docker.yml

@@ -18,24 +18,36 @@ on:
   push:
     tags: ["*"]
     branches: [main]
-    paths:
-      - "rustfs/**"
-      - "crates/**"
-      - "Dockerfile*"
-      - ".docker/**"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/workflows/docker.yml"
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - ".github/**"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "LICENSE*"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".gitignore"
+      - ".dockerignore"
   pull_request:
     branches: [main]
-    paths:
-      - "rustfs/**"
-      - "crates/**"
-      - "Dockerfile*"
-      - ".docker/**"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/workflows/docker.yml"
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - ".github/**"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "LICENSE*"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".gitignore"
+      - ".dockerignore"
   workflow_dispatch:
     inputs:
       push_images:

.github/workflows/performance.yml

@@ -18,10 +18,10 @@ on:
   push:
     branches: [main]
     paths:
-      - "rustfs/**"
-      - "crates/**"
-      - "Cargo.toml"
-      - "Cargo.lock"
+      - '**/*.rs'
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      - '.github/workflows/performance.yml'
   workflow_dispatch:
     inputs:
       profile_duration:

Cargo.lock

@@ -1171,7 +1171,7 @@ dependencies = [
  "bitflags 2.9.1",
  "cexpr",
  "clang-sys",
- "itertools 0.12.1",
+ "itertools 0.11.0",
  "lazy_static",
  "lazycell",
  "log",
@@ -3469,7 +3469,7 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
 
 [[package]]
 name = "e2e_test"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "bytes",
  "flatbuffers 25.2.10",
@@ -5023,15 +5023,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -5341,7 +5332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
 dependencies = [
  "cfg-if",
- "windows-targets 0.53.0",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -7839,7 +7830,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-trait",
  "atoi",
@@ -7908,7 +7899,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-appauth"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "base64-simd",
  "rsa",
@@ -7918,7 +7909,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-common"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "lazy_static",
  "tokio",
@@ -7927,7 +7918,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-config"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "const-str",
  "serde",
@@ -7936,7 +7927,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-crypto"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "aes-gcm",
  "argon2",
@@ -7954,7 +7945,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-ecstore"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-channel",
  "async-trait",
@@ -8029,7 +8020,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-filemeta"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "byteorder",
  "bytes",
@@ -8050,7 +8041,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-gui"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "chrono",
  "dioxus",
@@ -8071,7 +8062,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-iam"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -8095,7 +8086,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-lock"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-trait",
  "lazy_static",
@@ -8112,7 +8103,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-madmin"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "chrono",
  "humantime",
@@ -8124,7 +8115,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-notify"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-trait",
  "axum",
@@ -8153,7 +8144,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-obs"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-trait",
  "chrono",
@@ -8186,7 +8177,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-policy"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "base64-simd",
  "ipnetwork",
@@ -8205,7 +8196,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-protos"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "flatbuffers 25.2.10",
  "prost",
@@ -8216,7 +8207,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-rio"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "aes-gcm",
  "bytes",
@@ -8264,7 +8255,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-api"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -8288,7 +8279,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-query"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -8306,7 +8297,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-signer"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "bytes",
  "http 1.3.1",
@@ -8324,7 +8315,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-utils"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "base64-simd",
  "blake3",
@@ -8368,7 +8359,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-workers"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "tokio",
  "tracing",
@@ -8376,7 +8367,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-zip"
-version = "0.0.3"
+version = "0.0.5"
 dependencies = [
  "async-compression",
  "tokio",
@@ -9847,9 +9838,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.46.0"
+version = "1.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4"
+checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
 dependencies = [
  "backtrace",
  "bytes",
@@ -11129,29 +11120,13 @@ dependencies = [
  "windows_aarch64_gnullvm 0.52.6",
  "windows_aarch64_msvc 0.52.6",
  "windows_i686_gnu 0.52.6",
- "windows_i686_gnullvm 0.52.6",
+ "windows_i686_gnullvm",
  "windows_i686_msvc 0.52.6",
  "windows_x86_64_gnu 0.52.6",
  "windows_x86_64_gnullvm 0.52.6",
  "windows_x86_64_msvc 0.52.6",
 ]
 
-[[package]]
-name = "windows-targets"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b"
-dependencies = [
- "windows_aarch64_gnullvm 0.53.0",
- "windows_aarch64_msvc 0.53.0",
- "windows_i686_gnu 0.53.0",
- "windows_i686_gnullvm 0.53.0",
- "windows_i686_msvc 0.53.0",
- "windows_x86_64_gnu 0.53.0",
- "windows_x86_64_gnullvm 0.53.0",
- "windows_x86_64_msvc 0.53.0",
-]
-
 [[package]]
 name = "windows-threading"
 version = "0.1.0"
@@ -11188,12 +11163,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.2"
@@ -11212,12 +11181,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.2"
@@ -11236,24 +11199,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
-
 [[package]]
 name = "windows_i686_gnullvm"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.2"
@@ -11272,12 +11223,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.2"
@@ -11296,12 +11241,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.2"
@@ -11320,12 +11259,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.2"
@@ -11344,12 +11277,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.53.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
-
 [[package]]
 name = "winnow"
 version = "0.5.40"

Cargo.toml

@@ -44,7 +44,11 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.85"
-version = "0.0.3"
+version = "0.0.5"
+homepage = "https://rustfs.com"
+description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
+keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
+categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
 
 [workspace.lints.rust]
 unsafe_code = "deny"
@@ -52,28 +56,33 @@ unsafe_code = "deny"
 [workspace.lints.clippy]
 all = "warn"
 
+[patch.crates-io]
+rustfs-utils = { path = "crates/utils" }
+rustfs-filemeta = { path = "crates/filemeta" }
+rustfs-rio = { path = "crates/rio" }
+
 [workspace.dependencies]
-rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.3" }
-rustfs-appauth = { path = "crates/appauth", version = "0.0.3" }
-rustfs-common = { path = "crates/common", version = "0.0.3" }
-rustfs-crypto = { path = "crates/crypto", version = "0.0.3" }
-rustfs-ecstore = { path = "crates/ecstore", version = "0.0.3" }
-rustfs-iam = { path = "crates/iam", version = "0.0.3" }
-rustfs-lock = { path = "crates/lock", version = "0.0.3" }
-rustfs-madmin = { path = "crates/madmin", version = "0.0.3" }
-rustfs-policy = { path = "crates/policy", version = "0.0.3" }
-rustfs-protos = { path = "crates/protos", version = "0.0.3" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.3" }
-rustfs = { path = "./rustfs", version = "0.0.3" }
-rustfs-zip = { path = "./crates/zip", version = "0.0.3" }
-rustfs-config = { path = "./crates/config", version = "0.0.3" }
-rustfs-obs = { path = "crates/obs", version = "0.0.3" }
-rustfs-notify = { path = "crates/notify", version = "0.0.3" }
-rustfs-utils = { path = "crates/utils", version = "0.0.3" }
-rustfs-rio = { path = "crates/rio", version = "0.0.3" }
-rustfs-filemeta = { path = "crates/filemeta", version = "0.0.3" }
-rustfs-signer = { path = "crates/signer", version = "0.0.3" }
-rustfs-workers = { path = "crates/workers", version = "0.0.3" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
+rustfs-appauth = { path = "crates/appauth", version = "0.0.5" }
+rustfs-common = { path = "crates/common", version = "0.0.5" }
+rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
+rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
+rustfs-iam = { path = "crates/iam", version = "0.0.5" }
+rustfs-lock = { path = "crates/lock", version = "0.0.5" }
+rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
+rustfs-policy = { path = "crates/policy", version = "0.0.5" }
+rustfs-protos = { path = "crates/protos", version = "0.0.5" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
+rustfs = { path = "./rustfs", version = "0.0.5" }
+rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
+rustfs-config = { path = "./crates/config", version = "0.0.5" }
+rustfs-obs = { path = "crates/obs", version = "0.0.5" }
+rustfs-notify = { path = "crates/notify", version = "0.0.5" }
+rustfs-utils = { path = "crates/utils", version = "0.0.5" }
+rustfs-rio = { path = "crates/rio", version = "0.0.5" }
+rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
+rustfs-signer = { path = "crates/signer", version = "0.0.5" }
+rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 aes-gcm = { version = "0.10.3", features = ["std"] }
 arc-swap = "1.7.1"
 argon2 = { version = "0.5.3", features = ["std"] }
@@ -171,7 +180,6 @@ pbkdf2 = "0.12.2"
 percent-encoding = "2.3.1"
 pin-project-lite = "0.2.16"
 prost = "0.13.5"
-prost-build = "0.13.5"
 quick-xml = "0.37.5"
 rand = "0.9.1"
 rdkafka = { version = "0.37.0", features = ["tokio"] }
@@ -225,7 +233,7 @@ time = { version = "0.3.41", features = [
     "macros",
     "serde",
 ] }
-tokio = { version = "1.46.0", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.46.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.2", default-features = false }
 tokio-stream = { version = "0.1.17" }
 tokio-tar = "0.3.1"

Dockerfile

@@ -20,9 +20,10 @@ RUN apk add -U --no-cache \
     bash \
     unzip
 
-RUN curl -Lo /tmp/rustfs.zip https://dl.rustfs.com/artifacts/rustfs/rustfs-release-x86_64-unknown-linux-musl.latest.zip && \
-    unzip /tmp/rustfs.zip -d /tmp && \
-    mv /tmp/rustfs-release-x86_64-unknown-linux-musl/bin/rustfs /rustfs && \
+
+RUN curl -Lo /tmp/rustfs.zip https://dl.rustfs.com/artifacts/rustfs/rustfs-x86_64-unknown-linux-musl.zip && \
+    unzip -o /tmp/rustfs.zip -d /tmp && \
+    mv /tmp/rustfs /rustfs && \
     chmod +x /rustfs && \
     rm -rf /tmp/*
 

README.md

@@ -1,7 +1,9 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
+
 <p align="center">RustFS is a high-performance distributed object storage software built using Rust</p>
 
+
 <p align="center">
   <a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
   <a href="https://github.com/rustfs/rustfs/actions/workflows/docker.yml"><img alt="Build and Push Docker Images" src="https://github.com/rustfs/rustfs/actions/workflows/docker.yml/badge.svg" /></a>
@@ -17,11 +19,22 @@
 </p>
 
 <p align="center">
-English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简体中文</a>
+English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简体中文</a> | 
+  <!-- Keep these links. Translations will automatically update with the README. -->
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=de">Deutsch</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=es">Español</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=fr">français</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=ja">日本語</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=ko">한국어</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=pt">Português</a> | 
+  <a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
 </p>
 
 RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature, support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation, RustFS provides faster speed and safer distributed features for high-performance object storage.
 
+
+> ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**
+
 ## Features
 
 - **High Performance**: Built with Rust, ensuring speed and efficiency.

cli/rustfs-gui/rustfs-logo.svg

@@ -0,0 +1,15 @@
+<svg width="1558" height="260" viewBox="0 0 1558 260" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_0_3)">
+<path d="M1288.5 112.905H1159.75V58.4404H1262L1270 0L1074 0V260H1159.75V162.997H1296.95L1288.5 112.905Z" fill="#0196D0"/>
+<path d="M1058.62 58.4404V0H789V58.4404H881.133V260H966.885V58.4404H1058.62Z" fill="#0196D0"/>
+<path d="M521 179.102V0L454.973 15V161C454.973 181.124 452.084 193.146 443.5 202C434.916 211.257 419.318 214.5 400.5 214.5C381.022 214.5 366.744 210.854 357.5 202C348.916 193.548 346.357 175.721 346.357 156V0L280 15V175.48C280 208.08 290.234 229.412 309.712 241.486C329.19 253.56 358.903 260 400.5 260C440.447 260 470.159 253.56 490.297 241.486C510.766 229.412 521 208.483 521 179.102Z" fill="#0196D0"/>
+<path d="M172.84 84.2813C172.84 97.7982 168.249 107.737 158.41 113.303C149.883 118.471 137.092 121.254 120.693 122.049V162.997C129.876 163.792 138.076 166.177 144.307 176.514L184.647 260H265L225.316 180.489C213.181 155.046 201.374 149.48 178.744 143.517C212.197 138.349 241.386 118.471 241.386 73.1499C241.386 53.2722 233.843 30.2141 218.756 17.8899C203.998 5.56575 183.991 0 159.394 0H120.693V48.5015H127.58C142.23 48.5015 153.6 51.4169 161.689 57.2477C169.233 62.8135 172.84 71.5596 172.84 84.2813ZM120.693 122.049C119.163 122.049 117.741 122.049 116.43 122.049H68.5457V48.5015H120.693V0H0V260H70.5137V162.997H110.526C113.806 162.997 117.741 162.997 120.693 162.997V122.049Z" fill="#0196D0"/>
+<path d="M774 179.297C774 160.829 766.671 144.669 752.013 131.972C738.127 119.66 712.025 110.169 673.708 103.5C662.136 101.191 651.722 99.6523 643.235 97.3437C586.532 84.6467 594.632 52.7118 650.564 52.7118C680.651 52.7118 709.582 61.946 738.127 66.9478C742.37 67.7174 743.913 68.1021 744.298 68.1021L750.47 12.697C720.383 3.46282 684.895 0 654.036 0C616.619 0 587.689 6.54088 567.245 19.2379C546.801 31.9349 536 57.7137 536 82.3382C536 103.5 543.715 119.66 559.916 131.972C575.731 143.515 604.276 152.749 645.55 160.059C658.279 162.368 668.694 163.907 676.794 166.215C685.023 168.524 691.066 170.704 694.924 172.756C702.253 176.604 706.11 182.375 706.11 188.531C706.11 196.611 701.481 202.767 692.224 207C664.836 220.081 587.689 212.001 556.83 198.15L543.715 247.784C547.186 248.169 552.972 249.323 559.916 250.477C616.619 259.327 690.681 270.869 741.212 238.935C762.814 225.468 774 206.23 774 179.297Z" fill="#0196D0"/>
+<path d="M1558 179.568C1558 160.383 1550.42 144.268 1535.67 131.99C1521.32 119.968 1494.34 110.631 1454.74 103.981C1442.38 101.679 1432.01 99.3764 1422.84 97.8416C1422.44 97.8416 1422.04 97.8416 1422.04 97.4579V112.422L1361.04 75.2038L1422.04 38.3692V52.9496C1424.7 52.9496 1427.49 52.9496 1430.41 52.9496C1461.51 52.9496 1491.42 62.5419 1521.32 67.5299C1525.31 67.9136 1526.9 67.9136 1527.3 67.9136L1533.68 12.6619C1502.98 3.83692 1465.9 0 1434 0C1395.33 0 1365.43 6.52277 1345.09 19.5683C1323.16 32.6139 1312 57.9376 1312 82.8776C1312 103.981 1320.37 120.096 1336.72 131.607C1353.46 143.885 1382.97 153.093 1425.23 160.383C1434 161.535 1441.18 162.686 1447.56 164.22L1448.36 150.791L1507.36 190.312L1445.57 224.844L1445.96 212.949C1409.68 215.635 1357.45 209.112 1333.53 197.985L1320.37 247.482C1323.56 248.249 1329.54 248.633 1336.72 250.551C1395.33 259.376 1471.88 270.887 1524.11 238.657C1546.84 225.611 1558 205.659 1558 179.568Z" fill="#0196D0"/>
+</g>
+<defs>
+<clipPath id="clip0_0_3">
+<rect width="1558" height="260" fill="white"/>
+</clipPath>
+</defs>
+</svg>

crates/appauth/Cargo.toml

@@ -19,6 +19,10 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Application authentication and authorization for RustFS, providing secure access control and user management."
+keywords = ["authentication", "authorization", "security", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "authentication"]
 
 [dependencies]
 base64-simd = { workspace = true }

crates/appauth/README.md

@@ -3,7 +3,7 @@
 # RustFS AppAuth - Application Authentication
 
 <p align="center">
-  <strong>Secure application authentication and authorization for RustFS object storage</strong>
+  <strong>Application-level authentication and authorization module for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,461 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS AppAuth** provides secure application authentication and authorization mechanisms for the [RustFS](https://rustfs.com) distributed object storage system. It implements modern cryptographic standards including RSA-based authentication, JWT tokens, and secure session management for application-level access control.
-
-> **Note:** This is a security-critical submodule of RustFS that provides essential application authentication capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS AppAuth** provides application-level authentication and authorization capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔐 Authentication Methods
-
-- **RSA Authentication**: Public-key cryptography for secure authentication
-- **JWT Tokens**: JSON Web Token support for stateless authentication
-- **API Keys**: Simple API key-based authentication
-- **Session Management**: Secure session handling and lifecycle management
-
-### 🛡️ Security Features
-
-- **Cryptographic Signing**: RSA digital signatures for request validation
-- **Token Encryption**: Encrypted token storage and transmission
-- **Key Rotation**: Automatic key rotation and management
-- **Audit Logging**: Comprehensive authentication event logging
-
-### 🚀 Performance Features
-
-- **Base64 Optimization**: High-performance base64 encoding/decoding
-- **Token Caching**: Efficient token validation caching
-- **Parallel Verification**: Concurrent authentication processing
-- **Hardware Acceleration**: Leverage CPU crypto extensions
-
-### 🔧 Integration Features
-
-- **S3 Compatibility**: AWS S3-compatible authentication
-- **Multi-Tenant**: Support for multiple application tenants
-- **Permission Mapping**: Fine-grained permission assignment
-- **External Integration**: LDAP, OAuth, and custom authentication providers
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-appauth = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Authentication Setup
-
-```rust
-use rustfs_appauth::{AppAuthenticator, AuthConfig, AuthMethod};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Configure authentication
-    let config = AuthConfig {
-        auth_method: AuthMethod::RSA,
-        key_size: 2048,
-        token_expiry: Duration::from_hours(24),
-        enable_caching: true,
-        audit_logging: true,
-    };
-
-    // Initialize authenticator
-    let authenticator = AppAuthenticator::new(config).await?;
-
-    // Generate application credentials
-    let app_credentials = authenticator.generate_app_credentials("my-app").await?;
-
-    println!("App ID: {}", app_credentials.app_id);
-    println!("Public Key: {}", app_credentials.public_key);
-
-    Ok(())
-}
-```
-
-### RSA-Based Authentication
-
-```rust
-use rustfs_appauth::{RSAAuthenticator, AuthRequest, AuthResponse};
-
-async fn rsa_authentication_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create RSA authenticator
-    let rsa_auth = RSAAuthenticator::new(2048).await?;
-
-    // Generate key pair for application
-    let (private_key, public_key) = rsa_auth.generate_keypair().await?;
-
-    // Register application
-    let app_id = rsa_auth.register_application("my-storage-app", &public_key).await?;
-    println!("Application registered with ID: {}", app_id);
-
-    // Create authentication request
-    let auth_request = AuthRequest {
-        app_id: app_id.clone(),
-        timestamp: chrono::Utc::now(),
-        request_data: b"GET /bucket/object".to_vec(),
-    };
-
-    // Sign request with private key
-    let signed_request = rsa_auth.sign_request(&auth_request, &private_key).await?;
-
-    // Verify authentication
-    let auth_response = rsa_auth.authenticate(&signed_request).await?;
-
-    match auth_response {
-        AuthResponse::Success { session_token, permissions } => {
-            println!("Authentication successful!");
-            println!("Session token: {}", session_token);
-            println!("Permissions: {:?}", permissions);
-        }
-        AuthResponse::Failed { reason } => {
-            println!("Authentication failed: {}", reason);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### JWT Token Management
-
-```rust
-use rustfs_appauth::{JWTManager, TokenClaims, TokenRequest};
-
-async fn jwt_management_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create JWT manager
-    let jwt_manager = JWTManager::new("your-secret-key").await?;
-
-    // Create token claims
-    let claims = TokenClaims {
-        app_id: "my-app".to_string(),
-        user_id: Some("user123".to_string()),
-        permissions: vec![
-            "read:bucket".to_string(),
-            "write:bucket".to_string(),
-        ],
-        expires_at: chrono::Utc::now() + chrono::Duration::hours(24),
-        issued_at: chrono::Utc::now(),
-    };
-
-    // Generate JWT token
-    let token = jwt_manager.generate_token(&claims).await?;
-    println!("Generated token: {}", token);
-
-    // Validate token
-    let validation_result = jwt_manager.validate_token(&token).await?;
-
-    match validation_result {
-        Ok(validated_claims) => {
-            println!("Token valid for app: {}", validated_claims.app_id);
-            println!("Permissions: {:?}", validated_claims.permissions);
-        }
-        Err(e) => {
-            println!("Token validation failed: {}", e);
-        }
-    }
-
-    // Refresh token
-    let refreshed_token = jwt_manager.refresh_token(&token).await?;
-    println!("Refreshed token: {}", refreshed_token);
-
-    Ok(())
-}
-```
-
-### API Key Authentication
-
-```rust
-use rustfs_appauth::{APIKeyManager, APIKeyConfig, KeyPermissions};
-
-async fn api_key_authentication() -> Result<(), Box<dyn std::error::Error>> {
-    let api_key_manager = APIKeyManager::new().await?;
-
-    // Create API key configuration
-    let key_config = APIKeyConfig {
-        app_name: "storage-client".to_string(),
-        permissions: KeyPermissions {
-            read_buckets: vec!["public-*".to_string()],
-            write_buckets: vec!["uploads".to_string()],
-            admin_access: false,
-        },
-        expires_at: Some(chrono::Utc::now() + chrono::Duration::days(90)),
-        rate_limit: Some(1000), // requests per hour
-    };
-
-    // Generate API key
-    let api_key = api_key_manager.generate_key(&key_config).await?;
-    println!("Generated API key: {}", api_key.key);
-    println!("Key ID: {}", api_key.key_id);
-
-    // Authenticate with API key
-    let auth_result = api_key_manager.authenticate(&api_key.key).await?;
-
-    if auth_result.is_valid {
-        println!("API key authentication successful");
-        println!("Rate limit remaining: {}", auth_result.rate_limit_remaining);
-    }
-
-    // List API keys for application
-    let keys = api_key_manager.list_keys("storage-client").await?;
-    for key in keys {
-        println!("Key: {} - Status: {} - Expires: {:?}",
-            key.key_id, key.status, key.expires_at);
-    }
-
-    // Revoke API key
-    api_key_manager.revoke_key(&api_key.key_id).await?;
-    println!("API key revoked successfully");
-
-    Ok(())
-}
-```
-
-### Session Management
-
-```rust
-use rustfs_appauth::{SessionManager, SessionConfig, SessionInfo};
-
-async fn session_management_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Configure session management
-    let session_config = SessionConfig {
-        session_timeout: Duration::from_hours(8),
-        max_sessions_per_app: 10,
-        require_refresh: true,
-        secure_cookies: true,
-    };
-
-    let session_manager = SessionManager::new(session_config).await?;
-
-    // Create new session
-    let session_info = SessionInfo {
-        app_id: "web-app".to_string(),
-        user_id: Some("user456".to_string()),
-        ip_address: "192.168.1.100".to_string(),
-        user_agent: "RustFS-Client/1.0".to_string(),
-    };
-
-    let session = session_manager.create_session(&session_info).await?;
-    println!("Session created: {}", session.session_id);
-
-    // Validate session
-    let validation = session_manager.validate_session(&session.session_id).await?;
-
-    if validation.is_valid {
-        println!("Session is valid, expires at: {}", validation.expires_at);
-    }
-
-    // Refresh session
-    session_manager.refresh_session(&session.session_id).await?;
-    println!("Session refreshed");
-
-    // Get active sessions
-    let active_sessions = session_manager.get_active_sessions("web-app").await?;
-    println!("Active sessions: {}", active_sessions.len());
-
-    // Terminate session
-    session_manager.terminate_session(&session.session_id).await?;
-    println!("Session terminated");
-
-    Ok(())
-}
-```
-
-### Multi-Tenant Authentication
-
-```rust
-use rustfs_appauth::{MultiTenantAuth, TenantConfig, TenantPermissions};
-
-async fn multi_tenant_auth_example() -> Result<(), Box<dyn std::error::Error>> {
-    let multi_tenant_auth = MultiTenantAuth::new().await?;
-
-    // Create tenant configurations
-    let tenant1_config = TenantConfig {
-        tenant_id: "company-a".to_string(),
-        name: "Company A".to_string(),
-        permissions: TenantPermissions {
-            max_buckets: 100,
-            max_storage_gb: 1000,
-            allowed_regions: vec!["us-east-1".to_string(), "us-west-2".to_string()],
-        },
-        auth_methods: vec![AuthMethod::RSA, AuthMethod::JWT],
-    };
-
-    let tenant2_config = TenantConfig {
-        tenant_id: "company-b".to_string(),
-        name: "Company B".to_string(),
-        permissions: TenantPermissions {
-            max_buckets: 50,
-            max_storage_gb: 500,
-            allowed_regions: vec!["eu-west-1".to_string()],
-        },
-        auth_methods: vec![AuthMethod::APIKey],
-    };
-
-    // Register tenants
-    multi_tenant_auth.register_tenant(&tenant1_config).await?;
-    multi_tenant_auth.register_tenant(&tenant2_config).await?;
-
-    // Authenticate application for specific tenant
-    let auth_request = TenantAuthRequest {
-        tenant_id: "company-a".to_string(),
-        app_id: "app-1".to_string(),
-        credentials: AuthCredentials::RSA {
-            signature: "signed-data".to_string(),
-            public_key: "public-key-data".to_string(),
-        },
-    };
-
-    let auth_result = multi_tenant_auth.authenticate(&auth_request).await?;
-
-    if auth_result.is_authenticated {
-        println!("Multi-tenant authentication successful");
-        println!("Tenant: {}", auth_result.tenant_id);
-        println!("Permissions: {:?}", auth_result.permissions);
-    }
-
-    Ok(())
-}
-```
-
-### Authentication Middleware
-
-```rust
-use rustfs_appauth::{AuthMiddleware, AuthContext, MiddlewareConfig};
-use axum::{Router, middleware, Extension};
-
-async fn setup_auth_middleware() -> Result<Router, Box<dyn std::error::Error>> {
-    // Configure authentication middleware
-    let middleware_config = MiddlewareConfig {
-        skip_paths: vec!["/health".to_string(), "/metrics".to_string()],
-        require_auth: true,
-        audit_requests: true,
-    };
-
-    let auth_middleware = AuthMiddleware::new(middleware_config).await?;
-
-    // Create router with authentication middleware
-    let app = Router::new()
-        .route("/api/buckets", axum::routing::get(list_buckets))
-        .route("/api/objects", axum::routing::post(upload_object))
-        .layer(middleware::from_fn(auth_middleware.authenticate))
-        .layer(Extension(auth_middleware));
-
-    Ok(app)
-}
-
-async fn list_buckets(
-    Extension(auth_context): Extension<AuthContext>,
-) -> Result<String, Box<dyn std::error::Error>> {
-    // Use authentication context
-    println!("Authenticated app: {}", auth_context.app_id);
-    println!("Permissions: {:?}", auth_context.permissions);
-
-    // Your bucket listing logic here
-    Ok("Bucket list".to_string())
-}
-```
-
-## 🏗️ Architecture
-
-### AppAuth Architecture
-
-```
-AppAuth Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Authentication API                       │
-├─────────────────────────────────────────────────────────────┤
-│   RSA Auth    │   JWT Tokens  │   API Keys   │   Sessions   │
-├─────────────────────────────────────────────────────────────┤
-│              Cryptographic Operations                       │
-├─────────────────────────────────────────────────────────────┤
-│  Signing/     │   Token       │   Key        │   Session    │
-│  Verification │   Management  │   Management │   Storage    │
-├─────────────────────────────────────────────────────────────┤
-│              Security Infrastructure                        │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Authentication Methods
-
-| Method | Security Level | Use Case | Performance |
-|--------|----------------|----------|-------------|
-| RSA | High | Enterprise applications | Medium |
-| JWT | Medium-High | Web applications | High |
-| API Key | Medium | Service-to-service | Very High |
-| Session | Medium | Interactive applications | High |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test RSA authentication
-cargo test rsa_auth
-
-# Test JWT tokens
-cargo test jwt_tokens
-
-# Test API key management
-cargo test api_keys
-
-# Test session management
-cargo test sessions
-
-# Integration tests
-cargo test --test integration
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: RSA cryptographic libraries
-- **Security**: Secure key storage recommended
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS Signer](../signer) - Request signing
-- [RustFS Crypto](../crypto) - Cryptographic operations
+- JWT-based authentication with secure token management
+- RBAC (Role-Based Access Control) for fine-grained permissions
+- Multi-tenant application isolation and management
+- OAuth 2.0 and OpenID Connect integration
+- API key management and rotation
+- Session management with configurable expiration
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [AppAuth API Reference](https://docs.rustfs.com/appauth/)
-- [Security Guide](https://docs.rustfs.com/security/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔐 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/appauth/src/token.rs

@@ -23,14 +23,14 @@ use std::io::{Error, Result};
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct Token {
-    pub name: String, // 应用 ID
-    pub expired: u64, // 到期时间 (UNIX 时间戳)
+    pub name: String, // Application ID
+    pub expired: u64, // Expiry time (UNIX timestamp)
 }
 
-// 公钥生成 Token
-// [token] Token 对象
-// [key] 公钥字符串
-// 返回 base64 处理的加密字符串
+/// Public key generation Token
+/// [token] Token object
+/// [key] Public key string
+/// Returns the encrypted string processed by base64
 pub fn gencode(token: &Token, key: &str) -> Result<String> {
     let data = serde_json::to_vec(token)?;
     let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
@@ -38,10 +38,10 @@ pub fn gencode(token: &Token, key: &str) -> Result<String> {
     Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
 }
 
-// 私钥解析 Token
-// [token] base64 处理的加密字符串
-// [key] 私钥字符串
-// 返回 Token 对象
+/// Private key resolution Token
+/// [token] Encrypted string processed by base64
+/// [key] Private key string
+/// Return to the Token object
 pub fn parse(token: &str, key: &str) -> Result<Token> {
     let encrypted_data = base64_simd::URL_SAFE_NO_PAD
         .decode_to_vec(token.as_bytes())

crates/common/Cargo.toml

@@ -19,6 +19,10 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "Common utilities and data structures for RustFS, providing shared functionality across the project."
+keywords = ["common", "utilities", "data-structures", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "data-structures"]
 
 [lints]
 workspace = true

crates/common/README.md

@@ -3,7 +3,7 @@
 # RustFS Common - Shared Components
 
 <p align="center">
-  <strong>Common types, utilities, and shared components for RustFS distributed object storage</strong>
+  <strong>Shared components and common utilities module for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,279 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Common** provides shared components, types, and utilities used across all RustFS modules. This foundational library ensures consistency, reduces code duplication, and provides essential building blocks for the [RustFS](https://rustfs.com) distributed object storage system.
-
-> **Note:** This is a foundational submodule of RustFS that provides essential shared components for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Common** provides shared components and common utilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔧 Core Types
-
-- **Common Data Structures**: Shared types and enums
-- **Error Handling**: Unified error types and utilities
-- **Result Types**: Consistent result handling patterns
-- **Constants**: System-wide constants and defaults
-
-### 🛠️ Utilities
-
-- **Async Helpers**: Common async patterns and utilities
-- **Serialization**: Shared serialization utilities
-- **Logging**: Common logging and tracing setup
-- **Metrics**: Shared metrics and observability
-
-### 🌐 Network Components
-
-- **gRPC Common**: Shared gRPC types and utilities
-- **Protocol Helpers**: Common protocol implementations
-- **Connection Management**: Shared connection utilities
-- **Request/Response Types**: Common API types
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-common = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Common Types
-
-```rust
-use rustfs_common::{Result, Error, ObjectInfo, BucketInfo};
-
-fn main() -> Result<()> {
-    // Use common result type
-    let result = some_operation()?;
-
-    // Use common object info
-    let object = ObjectInfo {
-        name: "example.txt".to_string(),
-        size: 1024,
-        etag: "d41d8cd98f00b204e9800998ecf8427e".to_string(),
-        last_modified: chrono::Utc::now(),
-        content_type: "text/plain".to_string(),
-    };
-
-    println!("Object: {} ({} bytes)", object.name, object.size);
-    Ok(())
-}
-```
-
-### Error Handling
-
-```rust
-use rustfs_common::{Error, ErrorKind, Result};
-
-fn example_operation() -> Result<String> {
-    // Different error types
-    match some_condition {
-        true => Ok("Success".to_string()),
-        false => Err(Error::new(
-            ErrorKind::InvalidInput,
-            "Invalid operation parameters"
-        )),
-    }
-}
-
-fn handle_errors() {
-    match example_operation() {
-        Ok(value) => println!("Success: {}", value),
-        Err(e) => {
-            match e.kind() {
-                ErrorKind::InvalidInput => println!("Input error: {}", e),
-                ErrorKind::NotFound => println!("Not found: {}", e),
-                ErrorKind::PermissionDenied => println!("Access denied: {}", e),
-                _ => println!("Other error: {}", e),
-            }
-        }
-    }
-}
-```
-
-### Async Utilities
-
-```rust
-use rustfs_common::async_utils::{timeout_with_default, retry_with_backoff, spawn_task};
-use std::time::Duration;
-
-async fn async_operations() -> Result<()> {
-    // Timeout with default value
-    let result = timeout_with_default(
-        Duration::from_secs(5),
-        expensive_operation(),
-        "default_value".to_string()
-    ).await;
-
-    // Retry with exponential backoff
-    let result = retry_with_backoff(
-        3, // max attempts
-        Duration::from_millis(100), // initial delay
-        || async { fallible_operation().await }
-    ).await?;
-
-    // Spawn background task
-    spawn_task("background-worker", async {
-        background_work().await;
-    });
-
-    Ok(())
-}
-```
-
-### Metrics and Observability
-
-```rust
-use rustfs_common::metrics::{Counter, Histogram, Gauge, MetricsRegistry};
-
-fn setup_metrics() -> Result<()> {
-    let registry = MetricsRegistry::new();
-
-    // Create metrics
-    let requests_total = Counter::new("requests_total", "Total number of requests")?;
-    let request_duration = Histogram::new(
-        "request_duration_seconds",
-        "Request duration in seconds"
-    )?;
-    let active_connections = Gauge::new(
-        "active_connections",
-        "Number of active connections"
-    )?;
-
-    // Register metrics
-    registry.register(Box::new(requests_total))?;
-    registry.register(Box::new(request_duration))?;
-    registry.register(Box::new(active_connections))?;
-
-    Ok(())
-}
-```
-
-### gRPC Common Types
-
-```rust
-use rustfs_common::grpc::{GrpcResult, GrpcError, TonicStatus};
-use tonic::{Request, Response, Status};
-
-async fn grpc_service_example(
-    request: Request<MyRequest>
-) -> GrpcResult<MyResponse> {
-    let req = request.into_inner();
-
-    // Validate request
-    if req.name.is_empty() {
-        return Err(GrpcError::invalid_argument("Name cannot be empty"));
-    }
-
-    // Process request
-    let response = MyResponse {
-        result: format!("Processed: {}", req.name),
-        status: "success".to_string(),
-    };
-
-    Ok(Response::new(response))
-}
-
-// Error conversion
-impl From<Error> for Status {
-    fn from(err: Error) -> Self {
-        match err.kind() {
-            ErrorKind::NotFound => Status::not_found(err.to_string()),
-            ErrorKind::PermissionDenied => Status::permission_denied(err.to_string()),
-            ErrorKind::InvalidInput => Status::invalid_argument(err.to_string()),
-            _ => Status::internal(err.to_string()),
-        }
-    }
-}
-```
-
-## 🏗️ Architecture
-
-### Common Module Structure
-
-```
-Common Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Public API Layer                         │
-├─────────────────────────────────────────────────────────────┤
-│  Core Types    │   Error Types   │   Result Types           │
-├─────────────────────────────────────────────────────────────┤
-│  Async Utils   │   Metrics       │   gRPC Common            │
-├─────────────────────────────────────────────────────────────┤
-│  Constants     │   Serialization │   Logging                │
-├─────────────────────────────────────────────────────────────┤
-│              Foundation Types                                │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Core Components
-
-| Component | Purpose | Usage |
-|-----------|---------|-------|
-| Types | Common data structures | Shared across all modules |
-| Errors | Unified error handling | Consistent error reporting |
-| Async Utils | Async patterns | Common async operations |
-| Metrics | Observability | Performance monitoring |
-| gRPC | Protocol support | Service communication |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test specific components
-cargo test types
-cargo test errors
-cargo test async_utils
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Minimal, focused on essential functionality
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Utils](../utils) - Utility functions
-- [RustFS Config](../config) - Configuration management
+- Shared data structures and type definitions
+- Common error handling and result types
+- Utility functions used across modules
+- Configuration structures and validation
+- Logging and tracing infrastructure
+- Cross-platform compatibility helpers
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Common API Reference](https://docs.rustfs.com/common/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔧 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/config/Cargo.toml

@@ -19,6 +19,10 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Configuration management for RustFS, providing a centralized way to manage application settings and features."
+keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "config"]
 
 [dependencies]
 const-str = { workspace = true, optional = true }

crates/config/README.md

@@ -3,7 +3,7 @@
 # RustFS Config - Configuration Management
 
 <p align="center">
-  <strong>Centralized configuration management for RustFS distributed object storage</strong>
+  <strong>Configuration management and validation module for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,388 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Config** is the configuration management module for the [RustFS](https://rustfs.com) distributed object storage system. It provides centralized configuration handling, environment-based configuration loading, validation, and runtime configuration updates for all RustFS components.
-
-> **Note:** This is a foundational submodule of RustFS that provides essential configuration management capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Config** provides configuration management and validation capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### ⚙️ Configuration Management
-
-- **Multi-Format Support**: JSON, YAML, TOML configuration formats
-- **Environment Variables**: Automatic environment variable override
-- **Default Values**: Comprehensive default configuration
-- **Validation**: Configuration validation and error reporting
-
-### 🔧 Advanced Features
-
-- **Hot Reload**: Runtime configuration updates without restart
-- **Profile Support**: Environment-specific configuration profiles
-- **Secret Management**: Secure handling of sensitive configuration
-- **Configuration Merging**: Hierarchical configuration composition
-
-### 🛠️ Developer Features
-
-- **Type Safety**: Strongly typed configuration structures
-- **Documentation**: Auto-generated configuration documentation
-- **CLI Integration**: Command-line configuration override
-- **Testing Support**: Configuration mocking for tests
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-config = "0.1.0"
-
-# With specific features
-rustfs-config = { version = "0.1.0", features = ["constants", "notify"] }
-```
-
-### Feature Flags
-
-Available features:
-
-- `constants` - Configuration constants and compile-time values
-- `notify` - Configuration change notification support
-- `observability` - Observability and metrics configuration
-- `default` - Core configuration functionality
-
-## 🔧 Usage
-
-### Basic Configuration Loading
-
-```rust
-use rustfs_config::{Config, ConfigBuilder, ConfigFormat};
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Load configuration from file
-    let config = Config::from_file("config.yaml")?;
-
-    // Load with environment overrides
-    let config = ConfigBuilder::new()
-        .add_file("config.yaml")
-        .add_env_prefix("RUSTFS")
-        .build()?;
-
-    // Access configuration values
-    println!("Server address: {}", config.server.address);
-    println!("Storage path: {}", config.storage.path);
-
-    Ok(())
-}
-```
-
-### Environment-Based Configuration
-
-```rust
-use rustfs_config::{Config, Environment};
-
-async fn load_environment_config() -> Result<(), Box<dyn std::error::Error>> {
-    // Load configuration based on environment
-    let env = Environment::detect()?;
-    let config = Config::for_environment(env).await?;
-
-    match env {
-        Environment::Development => {
-            println!("Using development configuration");
-            println!("Debug mode: {}", config.debug.enabled);
-        }
-        Environment::Production => {
-            println!("Using production configuration");
-            println!("Log level: {}", config.logging.level);
-        }
-        Environment::Testing => {
-            println!("Using test configuration");
-            println!("Test database: {}", config.database.test_url);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Configuration Structure
-
-```rust
-use rustfs_config::{Config, ServerConfig, StorageConfig, SecurityConfig};
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct ApplicationConfig {
-    pub server: ServerConfig,
-    pub storage: StorageConfig,
-    pub security: SecurityConfig,
-    pub logging: LoggingConfig,
-    pub monitoring: MonitoringConfig,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct ServerConfig {
-    pub address: String,
-    pub port: u16,
-    pub workers: usize,
-    pub timeout: std::time::Duration,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct StorageConfig {
-    pub path: String,
-    pub max_size: u64,
-    pub compression: bool,
-    pub erasure_coding: ErasureCodingConfig,
-}
-
-fn load_typed_config() -> Result<ApplicationConfig, Box<dyn std::error::Error>> {
-    let config: ApplicationConfig = Config::builder()
-        .add_file("config.yaml")
-        .add_env_prefix("RUSTFS")
-        .set_default("server.port", 9000)?
-        .set_default("server.workers", 4)?
-        .build_typed()?;
-
-    Ok(config)
-}
-```
-
-### Configuration Validation
-
-```rust
-use rustfs_config::{Config, ValidationError, Validator};
-
-#[derive(Debug)]
-pub struct ConfigValidator;
-
-impl Validator<ApplicationConfig> for ConfigValidator {
-    fn validate(&self, config: &ApplicationConfig) -> Result<(), ValidationError> {
-        // Validate server configuration
-        if config.server.port < 1024 {
-            return Err(ValidationError::new("server.port", "Port must be >= 1024"));
-        }
-
-        if config.server.workers == 0 {
-            return Err(ValidationError::new("server.workers", "Workers must be > 0"));
-        }
-
-        // Validate storage configuration
-        if !std::path::Path::new(&config.storage.path).exists() {
-            return Err(ValidationError::new("storage.path", "Storage path does not exist"));
-        }
-
-        // Validate erasure coding parameters
-        if config.storage.erasure_coding.data_drives + config.storage.erasure_coding.parity_drives > 16 {
-            return Err(ValidationError::new("storage.erasure_coding", "Total drives cannot exceed 16"));
-        }
-
-        Ok(())
-    }
-}
-
-fn validate_configuration() -> Result<(), Box<dyn std::error::Error>> {
-    let config: ApplicationConfig = Config::load_with_validation(
-        "config.yaml",
-        ConfigValidator,
-    )?;
-
-    println!("Configuration is valid!");
-    Ok(())
-}
-```
-
-### Hot Configuration Reload
-
-```rust
-use rustfs_config::{ConfigWatcher, ConfigEvent};
-use tokio::sync::mpsc;
-
-async fn watch_configuration_changes() -> Result<(), Box<dyn std::error::Error>> {
-    let (tx, mut rx) = mpsc::channel::<ConfigEvent>(100);
-
-    // Start configuration watcher
-    let watcher = ConfigWatcher::new("config.yaml", tx)?;
-    watcher.start().await?;
-
-    // Handle configuration changes
-    while let Some(event) = rx.recv().await {
-        match event {
-            ConfigEvent::Changed(new_config) => {
-                println!("Configuration changed, reloading...");
-                // Apply new configuration
-                apply_configuration(new_config).await?;
-            }
-            ConfigEvent::Error(err) => {
-                eprintln!("Configuration error: {}", err);
-            }
-        }
-    }
-
-    Ok(())
-}
-
-async fn apply_configuration(config: ApplicationConfig) -> Result<(), Box<dyn std::error::Error>> {
-    // Update server configuration
-    // Update storage configuration
-    // Update security settings
-    // etc.
-    Ok(())
-}
-```
-
-### Configuration Profiles
-
-```rust
-use rustfs_config::{Config, Profile, ProfileManager};
-
-fn load_profile_based_config() -> Result<(), Box<dyn std::error::Error>> {
-    let profile_manager = ProfileManager::new("configs/")?;
-
-    // Load specific profile
-    let config = profile_manager.load_profile("production")?;
-
-    // Load with fallback
-    let config = profile_manager
-        .load_profile("staging")
-        .or_else(|_| profile_manager.load_profile("default"))?;
-
-    // Merge multiple profiles
-    let config = profile_manager
-        .merge_profiles(&["base", "production", "regional"])?;
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Configuration Architecture
-
-```
-Config Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Configuration API                        │
-├─────────────────────────────────────────────────────────────┤
-│   File Loader   │   Env Loader    │   CLI Parser           │
-├─────────────────────────────────────────────────────────────┤
-│              Configuration Merger                           │
-├─────────────────────────────────────────────────────────────┤
-│   Validation    │    Watching     │   Hot Reload           │
-├─────────────────────────────────────────────────────────────┤
-│              Type System Integration                        │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Configuration Sources
-
-| Source | Priority | Format | Example |
-|--------|----------|---------|---------|
-| Command Line | 1 (Highest) | Key-Value | `--server.port=8080` |
-| Environment Variables | 2 | Key-Value | `RUSTFS_SERVER_PORT=8080` |
-| Configuration File | 3 | JSON/YAML/TOML | `config.yaml` |
-| Default Values | 4 (Lowest) | Code | Compile-time defaults |
-
-## 📋 Configuration Reference
-
-### Server Configuration
-
-```yaml
-server:
-  address: "0.0.0.0"
-  port: 9000
-  workers: 4
-  timeout: "30s"
-  tls:
-    enabled: true
-    cert_file: "/etc/ssl/server.crt"
-    key_file: "/etc/ssl/server.key"
-```
-
-### Storage Configuration
-
-```yaml
-storage:
-  path: "/var/lib/rustfs"
-  max_size: "1TB"
-  compression: true
-  erasure_coding:
-    data_drives: 8
-    parity_drives: 4
-    stripe_size: "1MB"
-```
-
-### Security Configuration
-
-```yaml
-security:
-  auth:
-    enabled: true
-    method: "jwt"
-    secret_key: "${JWT_SECRET}"
-  encryption:
-    algorithm: "AES-256-GCM"
-    key_rotation_interval: "24h"
-```
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test configuration loading
-cargo test config_loading
-
-# Test validation
-cargo test validation
-
-# Test hot reload
-cargo test hot_reload
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Minimal external dependencies
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Utils](../utils) - Utility functions
-- [RustFS Common](../common) - Common types and utilities
+- Multi-format configuration support (TOML, YAML, JSON, ENV)
+- Environment variable integration and override
+- Configuration validation and type safety
+- Hot-reload capabilities for dynamic updates
+- Default value management and fallbacks
+- Secure credential handling and encryption
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Config API Reference](https://docs.rustfs.com/config/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with ⚙️ by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/crypto/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Cryptography and security features for RustFS, providing encryption, hashing, and secure authentication mechanisms."
+keywords = ["cryptography", "encryption", "hashing", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "cryptography"]
+documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
 
 [lints]
 workspace = true

crates/crypto/README.md

@@ -1,9 +1,9 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS Crypto Module
+# RustFS Crypto - Cryptographic Operations
 
 <p align="center">
-  <strong>High-performance cryptographic module for RustFS distributed object storage</strong>
+  <strong>High-performance cryptographic operations module for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,313 +17,21 @@
 
 ## 📖 Overview
 
-The **RustFS Crypto Module** is a core cryptographic component of the [RustFS](https://rustfs.com) distributed object storage system. This module provides secure, high-performance encryption and decryption capabilities, JWT token management, and cross-platform cryptographic operations designed specifically for enterprise-grade storage systems.
-
-> **Note:** This is a submodule of RustFS and is designed to work seamlessly within the RustFS ecosystem. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Crypto** provides high-performance cryptographic operations for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔐 Encryption & Decryption
-
-- **Multiple Algorithms**: Support for AES-GCM, ChaCha20Poly1305, and PBKDF2
-- **Key Derivation**: Argon2id and PBKDF2 for secure key generation
-- **Memory Safety**: Built with Rust's memory safety guarantees
-- **Cross-Platform**: Optimized for x86_64, aarch64, s390x, and other architectures
-
-### 🎫 JWT Management
-
-- **Token Generation**: Secure JWT token creation with HS512 algorithm
-- **Token Validation**: Robust JWT token verification and decoding
-- **Claims Management**: Flexible claims handling with JSON support
-
-### 🛡️ Security Features
-
-- **FIPS Compliance**: Optional FIPS 140-2 compatible mode
-- **Hardware Acceleration**: Automatic detection and utilization of CPU crypto extensions
-- **Secure Random**: Cryptographically secure random number generation
-- **Side-Channel Protection**: Resistant to timing attacks
-
-### 🚀 Performance
-
-- **Zero-Copy Operations**: Efficient memory usage with `Bytes` support
-- **Async/Await**: Full async support for non-blocking operations
-- **Hardware Optimization**: CPU-specific optimizations for better performance
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-crypto = "0.1.0"
-```
-
-### Feature Flags
-
-```toml
-[dependencies]
-rustfs-crypto = { version = "0.1.0", features = ["crypto", "fips"] }
-```
-
-Available features:
-
-- `crypto` (default): Enable all cryptographic functions
-- `fips`: Enable FIPS 140-2 compliance mode
-- `default`: Includes both `crypto` and `fips`
-
-## 🔧 Usage
-
-### Basic Encryption/Decryption
-
-```rust
-use rustfs_crypto::{encrypt_data, decrypt_data};
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let password = b"my_secure_password";
-    let data = b"sensitive information";
-
-    // Encrypt data
-    let encrypted = encrypt_data(password, data)?;
-    println!("Encrypted {} bytes", encrypted.len());
-
-    // Decrypt data
-    let decrypted = decrypt_data(password, &encrypted)?;
-    assert_eq!(data, decrypted.as_slice());
-    println!("Successfully decrypted data");
-
-    Ok(())
-}
-```
-
-### JWT Token Management
-
-```rust
-use rustfs_crypto::{jwt_encode, jwt_decode};
-use serde_json::json;
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let secret = b"jwt_secret_key";
-    let claims = json!({
-        "sub": "user123",
-        "exp": 1234567890,
-        "iat": 1234567890
-    });
-
-    // Create JWT token
-    let token = jwt_encode(secret, &claims)?;
-    println!("Generated token: {}", token);
-
-    // Verify and decode token
-    let decoded = jwt_decode(&token, secret)?;
-    println!("Decoded claims: {:?}", decoded.claims);
-
-    Ok(())
-}
-```
-
-### Advanced Usage with Custom Configuration
-
-```rust
-use rustfs_crypto::{encrypt_data, decrypt_data, Error};
-
-#[cfg(feature = "crypto")]
-fn secure_storage_example() -> Result<(), Error> {
-    // Large data encryption
-    let large_data = vec![0u8; 1024 * 1024]; // 1MB
-    let password = b"complex_password_123!@#";
-
-    // Encrypt with automatic algorithm selection
-    let encrypted = encrypt_data(password, &large_data)?;
-
-    // Decrypt and verify
-    let decrypted = decrypt_data(password, &encrypted)?;
-    assert_eq!(large_data.len(), decrypted.len());
-
-    println!("Successfully processed {} bytes", large_data.len());
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Supported Encryption Algorithms
-
-| Algorithm | Key Derivation | Use Case | FIPS Compliant |
-|-----------|---------------|----------|----------------|
-| AES-GCM | Argon2id | General purpose, hardware accelerated | ✅ |
-| ChaCha20Poly1305 | Argon2id | Software-only environments | ❌ |
-| AES-GCM | PBKDF2 | FIPS compliance required | ✅ |
-
-### Cross-Platform Support
-
-The module automatically detects and optimizes for:
-
-- **x86/x86_64**: AES-NI and PCLMULQDQ instructions
-- **aarch64**: ARM Crypto Extensions
-- **s390x**: IBM Z Crypto Extensions
-- **Other architectures**: Fallback to software implementations
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run tests with all features
-cargo test --all-features
-
-# Run benchmarks
-cargo bench
-
-# Test cross-platform compatibility
-cargo test --target x86_64-unknown-linux-gnu
-cargo test --target aarch64-unknown-linux-gnu
-```
-
-## 📊 Performance
-
-The crypto module is designed for high-performance scenarios:
-
-- **Encryption Speed**: Up to 2GB/s on modern hardware
-- **Memory Usage**: Minimal heap allocation with zero-copy operations
-- **CPU Utilization**: Automatic hardware acceleration detection
-- **Scalability**: Thread-safe operations for concurrent access
-
-## 🤝 Integration with RustFS
-
-This module is specifically designed to integrate with other RustFS components:
-
-- **Storage Layer**: Provides encryption for object storage
-- **Authentication**: JWT tokens for API authentication
-- **Configuration**: Secure configuration data encryption
-- **Metadata**: Encrypted metadata storage
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Architectures**: x86_64, aarch64, s390x, and more
-
-## 🔒 Security Considerations
-
-- All cryptographic operations use industry-standard algorithms
-- Key derivation follows best practices (Argon2id, PBKDF2)
-- Memory is securely cleared after use
-- Timing attack resistance is built-in
-- Hardware security modules (HSM) support planned
-
-## 🐛 Known Issues
-
-- Hardware acceleration detection may not work on all virtualized environments
-- FIPS mode requires additional system-level configuration
-- Some older CPU architectures may have reduced performance
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS Policy](../policy) - Policy engine
+- AES-GCM encryption with hardware acceleration
+- RSA and ECDSA digital signature support
+- Secure hash functions (SHA-256, BLAKE3)
+- Key derivation and management utilities
+- Stream ciphers for large data encryption
+- Hardware security module integration
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [API Reference](https://docs.rustfs.com/crypto/)
-- [Security Guide](https://docs.rustfs.com/security/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details on:
-
-- Code style and formatting requirements
-- Testing procedures and coverage
-- Security considerations for cryptographic code
-- Pull request process and review guidelines
-
-### Development Setup
-
-```bash
-# Clone the repository
-git clone https://github.com/rustfs/rustfs.git
-cd rustfs
-
-# Navigate to crypto module
-cd crates/crypto
-
-# Install dependencies
-cargo build
-
-# Run tests
-cargo test
-
-# Format code
-cargo fmt
-
-# Run linter
-cargo clippy
-```
-
-## 💬 Getting Help
-
-- **Documentation**: [docs.rustfs.com](https://docs.rustfs.com)
-- **Issues**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-- **Security**: Report security issues to <security@rustfs.com>
-
-## 📞 Contact
-
-- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Business**: <hello@rustfs.com>
-- **Jobs**: <jobs@rustfs.com>
-- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-
-## 👥 Contributors
-
-This module is maintained by the RustFS team and community contributors. Special thanks to all who have contributed to making RustFS cryptography secure and efficient.
-
-<a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=rustfs/rustfs" />
-</a>
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
-```
-Copyright 2024 RustFS Team
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-```
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with ❤️ by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/ecstore/Cargo.toml

@@ -19,6 +19,12 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "Erasure coding storage backend for RustFS, providing efficient data storage and retrieval with redundancy."
+keywords = ["erasure-coding", "storage", "rustfs", "Minio", "solomon"]
+categories = ["web-programming", "development-tools", "filesystem"]
+documentation = "https://docs.rs/rustfs-ecstore/latest/rustfs_ecstore/"
+
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lints]

crates/ecstore/README.md

@@ -1,6 +1,6 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS ECStore - Erasure Coding Storage Engine
+# RustFS ECStore - Erasure Coding Storage
 
 <p align="center">
   <strong>High-performance erasure coding storage engine for RustFS distributed object storage</strong>
@@ -17,425 +17,24 @@
 
 ## 📖 Overview
 
-**RustFS ECStore** is the core storage engine of the [RustFS](https://rustfs.com) distributed object storage system. It provides enterprise-grade erasure coding capabilities, data integrity protection, and high-performance object storage operations. This module serves as the foundation for RustFS's distributed storage architecture.
-
-> **Note:** This is a core submodule of RustFS and provides the primary storage capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS ECStore** provides erasure coding storage capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔧 Erasure Coding Storage
-
-- **Reed-Solomon Erasure Coding**: Advanced error correction with configurable redundancy
-- **Data Durability**: Protection against disk failures and bit rot
-- **Automatic Repair**: Self-healing capabilities for corrupted or missing data
-- **Configurable Parity**: Flexible parity configurations (4+2, 8+4, 16+4, etc.)
-
-### 💾 Storage Management
-
-- **Multi-Disk Support**: Intelligent disk management and load balancing
-- **Storage Classes**: Support for different storage tiers and policies
-- **Bucket Management**: Advanced bucket operations and lifecycle management
-- **Object Versioning**: Complete versioning support with metadata tracking
-
-### 🚀 Performance & Scalability
-
-- **High Throughput**: Optimized for large-scale data operations
-- **Parallel Processing**: Concurrent read/write operations across multiple disks
-- **Memory Efficient**: Smart caching and memory management
-- **SIMD Optimization**: Hardware-accelerated erasure coding operations
-
-### 🛡️ Data Integrity
-
-- **Bitrot Detection**: Real-time data corruption detection
-- **Checksum Verification**: Multiple checksum algorithms (MD5, SHA256, XXHash)
-- **Healing System**: Automatic background healing and repair
-- **Data Scrubbing**: Proactive data integrity scanning
-
-### 🔄 Advanced Features
-
-- **Compression**: Built-in compression support for space optimization
-- **Replication**: Cross-region replication capabilities
-- **Notification System**: Real-time event notifications
-- **Metrics & Monitoring**: Comprehensive performance metrics
-
-## 🏗️ Architecture
-
-### Storage Layout
-
-```
-ECStore Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Storage API Layer                        │
-├─────────────────────────────────────────────────────────────┤
-│  Bucket Management  │  Object Operations  │  Metadata Mgmt  │
-├─────────────────────────────────────────────────────────────┤
-│              Erasure Coding Engine                          │
-├─────────────────────────────────────────────────────────────┤
-│    Disk Management    │    Healing System    │    Cache     │
-├─────────────────────────────────────────────────────────────┤
-│              Physical Storage Devices                       │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Erasure Coding Schemes
-
-| Configuration | Data Drives | Parity Drives | Fault Tolerance | Storage Efficiency |
-|---------------|-------------|---------------|-----------------|-------------------|
-| 4+2 | 4 | 2 | 2 disk failures | 66.7% |
-| 8+4 | 8 | 4 | 4 disk failures | 66.7% |
-| 16+4 | 16 | 4 | 4 disk failures | 80% |
-| Custom | N | K | K disk failures | N/(N+K) |
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-ecstore = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Storage Operations
-
-```rust
-use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
-use std::sync::Arc;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Initialize storage layer
-    let storage = new_object_layer_fn("/path/to/storage").await?;
-
-    // Create a bucket
-    storage.make_bucket("my-bucket", None).await?;
-
-    // Put an object
-    let data = b"Hello, RustFS!";
-    storage.put_object("my-bucket", "hello.txt", data.to_vec()).await?;
-
-    // Get an object
-    let retrieved = storage.get_object("my-bucket", "hello.txt", None).await?;
-    println!("Retrieved: {}", String::from_utf8_lossy(&retrieved.data));
-
-    Ok(())
-}
-```
-
-### Advanced Configuration
-
-```rust
-use rustfs_ecstore::{StorageAPI, config::Config};
-
-async fn setup_storage_with_config() -> Result<(), Box<dyn std::error::Error>> {
-    let config = Config {
-        erasure_sets: vec![
-            // 8+4 configuration for high durability
-            ErasureSet::new(8, 4, vec![
-                "/disk1", "/disk2", "/disk3", "/disk4",
-                "/disk5", "/disk6", "/disk7", "/disk8",
-                "/disk9", "/disk10", "/disk11", "/disk12"
-            ])
-        ],
-        healing_enabled: true,
-        compression_enabled: true,
-        ..Default::default()
-    };
-
-    let storage = new_object_layer_fn("/path/to/storage")
-        .with_config(config)
-        .await?;
-
-    Ok(())
-}
-```
-
-### Bucket Management
-
-```rust
-use rustfs_ecstore::{StorageAPI, bucket::BucketInfo};
-
-async fn bucket_operations(storage: Arc<dyn StorageAPI>) -> Result<(), Box<dyn std::error::Error>> {
-    // Create bucket with specific configuration
-    let bucket_info = BucketInfo {
-        name: "enterprise-bucket".to_string(),
-        versioning_enabled: true,
-        lifecycle_config: Some(lifecycle_config()),
-        ..Default::default()
-    };
-
-    storage.make_bucket_with_config(bucket_info).await?;
-
-    // List buckets
-    let buckets = storage.list_buckets().await?;
-    for bucket in buckets {
-        println!("Bucket: {}, Created: {}", bucket.name, bucket.created);
-    }
-
-    // Set bucket policy
-    storage.set_bucket_policy("enterprise-bucket", policy_json).await?;
-
-    Ok(())
-}
-```
-
-### Healing and Maintenance
-
-```rust
-use rustfs_ecstore::{heal::HealingManager, StorageAPI};
-
-async fn healing_operations(storage: Arc<dyn StorageAPI>) -> Result<(), Box<dyn std::error::Error>> {
-    // Check storage health
-    let health = storage.storage_info().await?;
-    println!("Storage Health: {:?}", health);
-
-    // Trigger healing for specific bucket
-    let healing_result = storage.heal_bucket("my-bucket").await?;
-    println!("Healing completed: {:?}", healing_result);
-
-    // Background healing status
-    let healing_status = storage.healing_status().await?;
-    println!("Background healing: {:?}", healing_status);
-
-    Ok(())
-}
-```
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run tests with specific features
-cargo test --features "compression,healing"
-
-# Run benchmarks
-cargo bench
-
-# Run erasure coding benchmarks
-cargo bench --bench erasure_benchmark
-
-# Run comparison benchmarks
-cargo bench --bench comparison_benchmark
-```
-
-## 📊 Performance Benchmarks
-
-ECStore is designed for high-performance storage operations:
-
-### Throughput Performance
-
-- **Sequential Write**: Up to 10GB/s on NVMe storage
-- **Sequential Read**: Up to 12GB/s with parallel reads
-- **Random I/O**: 100K+ IOPS for small objects
-- **Erasure Coding**: 5GB/s encoding/decoding throughput
-
-### Scalability Metrics
-
-- **Storage Capacity**: Exabyte-scale deployments
-- **Concurrent Operations**: 10,000+ concurrent requests
-- **Disk Scaling**: Support for 1000+ disks per node
-- **Fault Tolerance**: Up to 50% disk failure resilience
-
-## 🔧 Configuration
-
-### Storage Configuration
-
-```toml
-[storage]
-# Erasure coding configuration
-erasure_set_size = 12  # Total disks per set
-data_drives = 8        # Data drives per set
-parity_drives = 4      # Parity drives per set
-
-# Performance tuning
-read_quorum = 6        # Minimum disks for read
-write_quorum = 7       # Minimum disks for write
-parallel_reads = true  # Enable parallel reads
-compression = true     # Enable compression
-
-# Healing configuration
-healing_enabled = true
-healing_interval = "24h"
-bitrot_check_interval = "168h"  # Weekly bitrot check
-```
-
-### Advanced Features
-
-```rust
-use rustfs_ecstore::config::StorageConfig;
-
-let config = StorageConfig {
-    // Enable advanced features
-    bitrot_protection: true,
-    automatic_healing: true,
-    compression_level: 6,
-    checksum_algorithm: ChecksumAlgorithm::XXHash64,
-
-    // Performance tuning
-    read_buffer_size: 1024 * 1024,  // 1MB read buffer
-    write_buffer_size: 4 * 1024 * 1024,  // 4MB write buffer
-    concurrent_operations: 1000,
-
-    // Storage optimization
-    small_object_threshold: 128 * 1024,  // 128KB
-    large_object_threshold: 64 * 1024 * 1024,  // 64MB
-
-    ..Default::default()
-};
-```
-
-## 🤝 Integration with RustFS
-
-ECStore integrates seamlessly with other RustFS components:
-
-- **API Server**: Provides S3-compatible storage operations
-- **IAM Module**: Handles authentication and authorization
-- **Policy Engine**: Implements bucket policies and access controls
-- **Notification System**: Publishes storage events
-- **Monitoring**: Provides detailed metrics and health status
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Storage**: Local disks, network storage, cloud storage
-- **Memory**: Minimum 4GB RAM (8GB+ recommended)
-- **Network**: High-speed network for distributed deployments
-
-## 🚀 Performance Tuning
-
-### Optimization Tips
-
-1. **Disk Configuration**:
-   - Use dedicated disks for each erasure set
-   - Prefer NVMe over SATA for better performance
-   - Ensure consistent disk sizes within erasure sets
-
-2. **Memory Settings**:
-   - Allocate sufficient memory for caching
-   - Tune read/write buffer sizes based on workload
-   - Enable memory-mapped files for large objects
-
-3. **Network Optimization**:
-   - Use high-speed network connections
-   - Configure proper MTU sizes
-   - Enable network compression for WAN scenarios
-
-4. **CPU Optimization**:
-   - Utilize SIMD instructions for erasure coding
-   - Balance CPU cores across erasure sets
-   - Enable hardware-accelerated checksums
-
-## 🐛 Troubleshooting
-
-### Common Issues
-
-1. **Disk Failures**:
-   - Check disk health using `storage_info()`
-   - Trigger healing with `heal_bucket()`
-   - Replace failed disks and re-add to cluster
-
-2. **Performance Issues**:
-   - Monitor disk I/O utilization
-   - Check network bandwidth usage
-   - Verify erasure coding configuration
-
-3. **Data Integrity**:
-   - Run bitrot detection scans
-   - Verify checksums for critical data
-   - Check healing system status
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Crypto](../crypto) - Cryptographic operations
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS Policy](../policy) - Policy engine
-- [RustFS FileMeta](../filemeta) - File metadata management
+- Reed-Solomon erasure coding implementation
+- Configurable redundancy levels (N+K schemes)
+- Automatic data healing and reconstruction
+- Multi-drive support with intelligent placement
+- Parallel encoding/decoding for performance
+- Efficient disk space utilization
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Storage API Reference](https://docs.rustfs.com/ecstore/)
-- [Erasure Coding Guide](https://docs.rustfs.com/erasure-coding/)
-- [Performance Tuning](https://docs.rustfs.com/performance/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details on:
-
-- Storage engine architecture and design patterns
-- Erasure coding implementation guidelines
-- Performance optimization techniques
-- Testing procedures for storage operations
-- Documentation standards for storage APIs
-
-### Development Setup
-
-```bash
-# Clone the repository
-git clone https://github.com/rustfs/rustfs.git
-cd rustfs
-
-# Navigate to ECStore module
-cd crates/ecstore
-
-# Install dependencies
-cargo build
-
-# Run tests
-cargo test
-
-# Run benchmarks
-cargo bench
-
-# Format code
-cargo fmt
-
-# Run linter
-cargo clippy
-```
-
-## 💬 Getting Help
-
-- **Documentation**: [docs.rustfs.com](https://docs.rustfs.com)
-- **Issues**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-- **Storage Support**: <storage-support@rustfs.com>
-
-## 📞 Contact
-
-- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Business**: <hello@rustfs.com>
-- **Jobs**: <jobs@rustfs.com>
-- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-
-## 👥 Contributors
-
-This module is maintained by the RustFS storage team and community contributors. Special thanks to all who have contributed to making RustFS storage reliable and efficient.
-
-<a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=rustfs/rustfs" />
-</a>
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
 
 ```
 Copyright 2024 RustFS Team

crates/ecstore/README_cn.md

@@ -1,103 +1,19 @@
 # ECStore - Erasure Coding Storage
 
-ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD
-implementation for optimal performance.
+ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD implementation for optimal performance.
 
-## Reed-Solomon Implementation
+## Features
 
-### SIMD Backend (Only)
+- **Reed-Solomon Implementation**: High-performance SIMD-optimized erasure coding
+- **Cross-Platform Compatibility**: Support for x86_64, aarch64, and other architectures
+- **Performance Optimized**: SIMD instructions for maximum throughput
+- **Thread Safety**: Safe concurrent access with caching optimizations
+- **Scalable**: Excellent performance for high-throughput scenarios
 
-- **Performance**: Uses SIMD optimization for high-performance encoding/decoding
-- **Compatibility**: Works with any shard size through SIMD implementation
-- **Reliability**: High-performance SIMD implementation for large data processing
-- **Use case**: Optimized for maximum performance in large data processing scenarios
+## Documentation
 
-### Usage Example
+For complete documentation, examples, and usage information, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
-```rust
-use rustfs_ecstore::erasure_coding::Erasure;
+## License
 
-// Create erasure coding instance
-// 4 data shards, 2 parity shards, 1KB block size
-let erasure = Erasure::new(4, 2, 1024);
-
-// Encode data
-let data = b"hello world from rustfs erasure coding";
-let shards = erasure.encode_data(data) ?;
-
-// Simulate loss of one shard
-let mut shards_opt: Vec<Option<Vec<u8> > > = shards
-.iter()
-.map( | b| Some(b.to_vec()))
-.collect();
-shards_opt[2] = None; // Lose shard 2
-
-// Reconstruct missing data
-erasure.decode_data( & mut shards_opt) ?;
-
-// Recover original data
-let mut recovered = Vec::new();
-for shard in shards_opt.iter().take(4) { // Only data shards
-recovered.extend_from_slice(shard.as_ref().unwrap());
-}
-recovered.truncate(data.len());
-assert_eq!(&recovered, data);
-```
-
-## Performance Considerations
-
-### SIMD Implementation Benefits
-
-- **High Throughput**: Optimized for large block sizes (>= 1KB recommended)
-- **CPU Optimization**: Leverages modern CPU SIMD instructions
-- **Scalability**: Excellent performance for high-throughput scenarios
-
-### Implementation Details
-
-#### `reed-solomon-simd`
-
-- **Instance Caching**: Encoder/decoder instances are cached and reused for optimal performance
-- **Thread Safety**: Thread-safe with RwLock-based caching
-- **SIMD Optimization**: Leverages CPU SIMD instructions for maximum performance
-- **Reset Capability**: Cached instances are reset for different parameters, avoiding unnecessary allocations
-
-### Performance Tips
-
-1. **Batch Operations**: When possible, batch multiple small operations into larger blocks
-2. **Block Size Optimization**: Use block sizes that are multiples of 64 bytes for optimal SIMD performance
-3. **Memory Allocation**: Pre-allocate buffers when processing multiple blocks
-4. **Cache Warming**: Initial operations may be slower due to cache setup, subsequent operations benefit from caching
-
-## Cross-Platform Compatibility
-
-The SIMD implementation supports:
-
-- x86_64 with advanced SIMD instructions (AVX2, SSE)
-- aarch64 (ARM64) with NEON SIMD optimizations
-- Other architectures with fallback implementations
-
-The implementation automatically selects the best available SIMD instructions for the target platform, providing optimal
-performance across different architectures.
-
-## Testing and Benchmarking
-
-Run performance benchmarks:
-
-```bash
-# Run erasure coding benchmarks
-cargo bench --bench erasure_benchmark
-
-# Run comparison benchmarks
-cargo bench --bench comparison_benchmark
-
-# Generate benchmark reports
-./run_benchmarks.sh
-```
-
-## Error Handling
-
-All operations return `Result` types with comprehensive error information:
-
-- Encoding errors: Invalid parameters, insufficient memory
-- Decoding errors: Too many missing shards, corrupted data
-- Configuration errors: Invalid shard counts, unsupported parameters 
+This project is licensed under the Apache License, Version 2.0.

crates/ecstore/src/disk/local.rs

@@ -145,8 +145,8 @@ impl Debug for LocalDisk {
 impl LocalDisk {
     pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
         debug!("Creating local disk");
-        let root = match fs::canonicalize(ep.get_file_path()).await {
-            Ok(path) => path,
+        let root = match PathBuf::from(ep.get_file_path()).absolutize() {
+            Ok(path) => path.into_owned(),
             Err(e) => {
                 if e.kind() == ErrorKind::NotFound {
                     return Err(DiskError::VolumeNotFound);

crates/ecstore/src/global.rs

@@ -62,7 +62,9 @@ static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
 pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
 pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
 pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
-pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();}
+pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();
+pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
+}
 
 static GLOBAL_ACTIVE_CRED: OnceLock<Credentials> = OnceLock::new();
 
@@ -182,3 +184,11 @@ pub async fn update_erasure_type(setup_type: SetupType) {
 // }
 
 type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
+
+pub fn set_global_region(region: String) {
+    GLOBAL_REGION.set(region).unwrap();
+}
+
+pub fn get_global_region() -> Option<String> {
+    GLOBAL_REGION.get().cloned()
+}

crates/ecstore/src/set_disk.rs

@@ -4099,6 +4099,8 @@ impl ObjectIO for SetDisks {
             }
         }
 
+        drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data
+
         let (online_disks, _, op_old_dir) = Self::rename_data(
             &shuffle_disks,
             RUSTFS_META_TMP_BUCKET,
@@ -5039,6 +5041,8 @@ impl StorageAPI for SetDisks {
 
         let fi_buff = fi.marshal_msg()?;
 
+        drop(writers); // drop writers to close all files
+
         let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
         let _ = Self::rename_part(
             &disks,

crates/ecstore/src/store.rs

@@ -2508,14 +2508,14 @@ fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Result<
 
     #[cfg(target_os = "windows")]
     {
-        if object.contains('\\')
-            || object.contains(':')
+        if object.contains(':')
             || object.contains('*')
             || object.contains('?')
             || object.contains('"')
             || object.contains('|')
             || object.contains('<')
             || object.contains('>')
+        // || object.contains('\\')
         {
             return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
         }
@@ -2549,9 +2549,9 @@ fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
         return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
     }
 
-    if cfg!(target_os = "windows") && object.contains('\\') {
-        return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
-    }
+    // if cfg!(target_os = "windows") && object.contains('\\') {
+    //     return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
+    // }
 
     Ok(())
 }

crates/filemeta/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "File metadata management for RustFS, providing efficient storage and retrieval of file metadata in a distributed system."
+keywords = ["file-metadata", "storage", "retrieval", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "filesystem"]
+documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/"
 
 [dependencies]
 crc32fast = { workspace = true }

crates/filemeta/README.md

@@ -3,7 +3,7 @@
 # RustFS FileMeta - File Metadata Management
 
 <p align="center">
-  <strong>High-performance file metadata management for RustFS distributed object storage</strong>
+  <strong>Advanced file metadata management and indexing module for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,246 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS FileMeta** is the metadata management module for the [RustFS](https://rustfs.com) distributed object storage system. It provides efficient storage, retrieval, and management of file metadata, supporting features like versioning, tagging, and extended attributes with high performance and reliability.
-
-> **Note:** This is a core submodule of RustFS that provides essential metadata management capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS FileMeta** provides advanced file metadata management and indexing capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📝 Metadata Management
-- **File Information**: Complete file metadata including size, timestamps, and checksums
-- **Object Versioning**: Version-aware metadata management
-- **Extended Attributes**: Custom metadata and tagging support
-- **Inline Metadata**: Optimized storage for small metadata
-
-### 🚀 Performance Features
-- **FlatBuffers Serialization**: Zero-copy metadata serialization
-- **Efficient Storage**: Optimized metadata storage layout
-- **Fast Lookups**: High-performance metadata queries
-- **Batch Operations**: Bulk metadata operations
-
-### 🔧 Advanced Capabilities
-- **Schema Evolution**: Forward and backward compatible metadata schemas
-- **Compression**: Metadata compression for space efficiency
-- **Validation**: Metadata integrity verification
-- **Migration**: Seamless metadata format migration
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-filemeta = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Metadata Operations
-
-```rust
-use rustfs_filemeta::{FileInfo, XLMeta};
-use std::collections::HashMap;
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create file metadata
-    let mut file_info = FileInfo::new();
-    file_info.name = "example.txt".to_string();
-    file_info.size = 1024;
-    file_info.mod_time = chrono::Utc::now();
-
-    // Add custom metadata
-    let mut user_defined = HashMap::new();
-    user_defined.insert("author".to_string(), "john@example.com".to_string());
-    user_defined.insert("department".to_string(), "engineering".to_string());
-    file_info.user_defined = user_defined;
-
-    // Create XL metadata
-    let xl_meta = XLMeta::new(file_info);
-
-    // Serialize metadata
-    let serialized = xl_meta.serialize()?;
-
-    // Deserialize metadata
-    let deserialized = XLMeta::deserialize(&serialized)?;
-
-    println!("File: {}, Size: {}", deserialized.file_info.name, deserialized.file_info.size);
-
-    Ok(())
-}
-```
-
-### Advanced Metadata Management
-
-```rust
-use rustfs_filemeta::{XLMeta, FileInfo, VersionInfo};
-
-async fn advanced_metadata_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create versioned metadata
-    let mut xl_meta = XLMeta::new(FileInfo::default());
-
-    // Set version information
-    xl_meta.set_version_info(VersionInfo {
-        version_id: "v1.0.0".to_string(),
-        is_latest: true,
-        delete_marker: false,
-        restore_ongoing: false,
-    });
-
-    // Add checksums
-    xl_meta.add_checksum("md5", "d41d8cd98f00b204e9800998ecf8427e");
-    xl_meta.add_checksum("sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
-
-    // Set object tags
-    xl_meta.set_tags(vec![
-        ("Environment".to_string(), "Production".to_string()),
-        ("Owner".to_string(), "DataTeam".to_string()),
-    ]);
-
-    // Set retention information
-    xl_meta.set_retention_info(
-        chrono::Utc::now() + chrono::Duration::days(365),
-        "GOVERNANCE".to_string(),
-    );
-
-    // Validate metadata
-    xl_meta.validate()?;
-
-    Ok(())
-}
-```
-
-### Inline Metadata Operations
-
-```rust
-use rustfs_filemeta::{InlineMetadata, MetadataSize};
-
-fn inline_metadata_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create inline metadata for small files
-    let mut inline_meta = InlineMetadata::new();
-
-    // Set basic properties
-    inline_meta.set_content_type("text/plain");
-    inline_meta.set_content_encoding("gzip");
-    inline_meta.set_cache_control("max-age=3600");
-
-    // Add custom headers
-    inline_meta.add_header("x-custom-field", "custom-value");
-    inline_meta.add_header("x-app-version", "1.2.3");
-
-    // Check if metadata fits inline storage
-    if inline_meta.size() <= MetadataSize::INLINE_THRESHOLD {
-        println!("Metadata can be stored inline");
-    } else {
-        println!("Metadata requires separate storage");
-    }
-
-    // Serialize for storage
-    let bytes = inline_meta.to_bytes()?;
-
-    // Deserialize from storage
-    let restored = InlineMetadata::from_bytes(&bytes)?;
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Metadata Storage Layout
-
-```
-FileMeta Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Metadata API Layer                       │
-├─────────────────────────────────────────────────────────────┤
-│    XL Metadata    │   Inline Metadata   │   Version Info    │
-├─────────────────────────────────────────────────────────────┤
-│              FlatBuffers Serialization                      │
-├─────────────────────────────────────────────────────────────┤
-│    Compression    │    Validation       │    Migration      │
-├─────────────────────────────────────────────────────────────┤
-│              Storage Backend Integration                     │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Metadata Types
-
-| Type | Use Case | Storage | Performance |
-|------|----------|---------|-------------|
-| XLMeta | Large objects with rich metadata | Separate file | High durability |
-| InlineMeta | Small objects with minimal metadata | Embedded | Fastest access |
-| VersionMeta | Object versioning information | Version-specific | Version-aware |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run serialization benchmarks
-cargo bench
-
-# Test metadata validation
-cargo test validation
-
-# Test schema migration
-cargo test migration
-```
-
-## 🚀 Performance
-
-FileMeta is optimized for high-performance metadata operations:
-
-- **Serialization**: Zero-copy FlatBuffers serialization
-- **Storage**: Compact binary format reduces I/O
-- **Caching**: Intelligent metadata caching
-- **Batch Operations**: Efficient bulk metadata processing
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Memory**: Minimal memory footprint
-- **Storage**: Compatible with RustFS storage backend
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS Utils](../utils) - Utility functions
-- [RustFS Proto](../protos) - Protocol definitions
+- High-performance metadata storage and retrieval
+- Advanced indexing with full-text search capabilities
+- File attribute management and custom metadata
+- Version tracking and history management
+- Distributed metadata replication
+- Real-time metadata synchronization
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-- [RustFS Documentation](https://docs.rustfs.com)
-- [FileMeta API Reference](https://docs.rustfs.com/filemeta/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with ❤️ by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/iam/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Identity and Access Management (IAM) for RustFS, providing user management, roles, and permissions."
+keywords = ["iam", "identity", "access-management", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "authentication"]
+documentation = "https://docs.rs/rustfs-iam/latest/rustfs_iam/"
 
 [lints]
 workspace = true

crates/iam/README.md

@@ -1,9 +1,9 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS IAM - Identity and Access Management
+# RustFS IAM - Identity & Access Management
 
 <p align="center">
-  <strong>Enterprise-grade identity and access management for RustFS distributed object storage</strong>
+  <strong>Identity and access management system for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,592 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS IAM** is the identity and access management module for the [RustFS](https://rustfs.com) distributed object storage system. It provides comprehensive authentication, authorization, and access control capabilities, ensuring secure and compliant access to storage resources.
-
-> **Note:** This is a core submodule of RustFS and provides essential security and access control features for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS IAM** provides identity and access management capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔐 Authentication & Authorization
-
-- **Multi-Factor Authentication**: Support for various authentication methods
-- **Access Key Management**: Secure generation and management of access keys
-- **JWT Token Support**: Stateless authentication with JWT tokens
-- **Session Management**: Secure session handling and token refresh
-
-### 👥 User Management
-
-- **User Accounts**: Complete user lifecycle management
-- **Service Accounts**: Automated service authentication
-- **Temporary Accounts**: Time-limited access credentials
-- **Group Management**: Organize users into groups for easier management
-
-### 🛡️ Access Control
-
-- **Role-Based Access Control (RBAC)**: Flexible role and permission system
-- **Policy-Based Access Control**: Fine-grained access policies
-- **Resource-Level Permissions**: Granular control over storage resources
-- **API-Level Authorization**: Secure API access control
-
-### 🔑 Credential Management
-
-- **Secure Key Generation**: Cryptographically secure key generation
-- **Key Rotation**: Automatic and manual key rotation capabilities
-- **Credential Validation**: Real-time credential verification
-- **Secret Management**: Secure storage and retrieval of secrets
-
-### 🏢 Enterprise Features
-
-- **LDAP Integration**: Enterprise directory service integration
-- **SSO Support**: Single Sign-On capabilities
-- **Audit Logging**: Comprehensive access audit trails
-- **Compliance Features**: Meet regulatory compliance requirements
-
-## 🏗️ Architecture
-
-### IAM System Architecture
-
-```
-IAM Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    IAM API Layer                            │
-├─────────────────────────────────────────────────────────────┤
-│  Authentication  │  Authorization  │  User Management       │
-├─────────────────────────────────────────────────────────────┤
-│              Policy Engine Integration                       │
-├─────────────────────────────────────────────────────────────┤
-│    Credential Store   │    Cache Layer    │   Token Manager │
-├─────────────────────────────────────────────────────────────┤
-│              Storage Backend Integration                     │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Security Model
-
-| Component | Description | Security Level |
-|-----------|-------------|----------------|
-| Access Keys | API authentication credentials | High |
-| JWT Tokens | Stateless authentication tokens | High |
-| Session Management | User session handling | Medium |
-| Policy Enforcement | Access control policies | Critical |
-| Audit Logging | Security event tracking | High |
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-iam = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic IAM Setup
-
-```rust
-use rustfs_iam::{init_iam_sys, get};
-use rustfs_ecstore::ECStore;
-use std::sync::Arc;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Initialize with ECStore backend
-    let ecstore = Arc::new(ECStore::new("/path/to/storage").await?);
-
-    // Initialize IAM system
-    init_iam_sys(ecstore).await?;
-
-    // Get IAM system instance
-    let iam = get()?;
-
-    println!("IAM system initialized successfully");
-    Ok(())
-}
-```
-
-### User Management
-
-```rust
-use rustfs_iam::{get, manager::UserInfo};
-
-async fn user_management_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Create a new user
-    let user_info = UserInfo {
-        access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
-        secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
-        status: "enabled".to_string(),
-        ..Default::default()
-    };
-
-    iam.create_user("john-doe", user_info).await?;
-
-    // List users
-    let users = iam.list_users().await?;
-    for user in users {
-        println!("User: {}, Status: {}", user.name, user.status);
-    }
-
-    // Update user status
-    iam.set_user_status("john-doe", "disabled").await?;
-
-    // Delete user
-    iam.delete_user("john-doe").await?;
-
-    Ok(())
-}
-```
-
-### Group Management
-
-```rust
-use rustfs_iam::{get, manager::GroupInfo};
-
-async fn group_management_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Create a group
-    let group_info = GroupInfo {
-        name: "developers".to_string(),
-        members: vec!["john-doe".to_string(), "jane-smith".to_string()],
-        policies: vec!["read-only-policy".to_string()],
-        ..Default::default()
-    };
-
-    iam.create_group(group_info).await?;
-
-    // Add user to group
-    iam.add_user_to_group("alice", "developers").await?;
-
-    // Remove user from group
-    iam.remove_user_from_group("alice", "developers").await?;
-
-    // List groups
-    let groups = iam.list_groups().await?;
-    for group in groups {
-        println!("Group: {}, Members: {}", group.name, group.members.len());
-    }
-
-    Ok(())
-}
-```
-
-### Policy Management
-
-```rust
-use rustfs_iam::{get, manager::PolicyDocument};
-
-async fn policy_management_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Create a policy
-    let policy_doc = PolicyDocument {
-        version: "2012-10-17".to_string(),
-        statement: vec![
-            Statement {
-                effect: "Allow".to_string(),
-                action: vec!["s3:GetObject".to_string()],
-                resource: vec!["arn:aws:s3:::my-bucket/*".to_string()],
-                ..Default::default()
-            }
-        ],
-        ..Default::default()
-    };
-
-    iam.create_policy("read-only-policy", policy_doc).await?;
-
-    // Attach policy to user
-    iam.attach_user_policy("john-doe", "read-only-policy").await?;
-
-    // Detach policy from user
-    iam.detach_user_policy("john-doe", "read-only-policy").await?;
-
-    // List policies
-    let policies = iam.list_policies().await?;
-    for policy in policies {
-        println!("Policy: {}", policy.name);
-    }
-
-    Ok(())
-}
-```
-
-### Service Account Management
-
-```rust
-use rustfs_iam::{get, manager::ServiceAccountInfo};
-
-async fn service_account_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Create service account
-    let service_account = ServiceAccountInfo {
-        name: "backup-service".to_string(),
-        description: "Automated backup service".to_string(),
-        policies: vec!["backup-policy".to_string()],
-        ..Default::default()
-    };
-
-    iam.create_service_account(service_account).await?;
-
-    // Generate credentials for service account
-    let credentials = iam.generate_service_account_credentials("backup-service").await?;
-    println!("Service Account Credentials: {:?}", credentials);
-
-    // Rotate service account credentials
-    iam.rotate_service_account_credentials("backup-service").await?;
-
-    Ok(())
-}
-```
-
-### Authentication and Authorization
-
-```rust
-use rustfs_iam::{get, auth::Credentials};
-
-async fn auth_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Authenticate user
-    let credentials = Credentials {
-        access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
-        secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
-        session_token: None,
-    };
-
-    let auth_result = iam.authenticate(&credentials).await?;
-    println!("Authentication successful: {}", auth_result.user_name);
-
-    // Check authorization
-    let authorized = iam.is_authorized(
-        &auth_result.user_name,
-        "s3:GetObject",
-        "arn:aws:s3:::my-bucket/file.txt"
-    ).await?;
-
-    if authorized {
-        println!("User is authorized to access the resource");
-    } else {
-        println!("User is not authorized to access the resource");
-    }
-
-    Ok(())
-}
-```
-
-### Temporary Credentials
-
-```rust
-use rustfs_iam::{get, manager::TemporaryCredentials};
-use std::time::Duration;
-
-async fn temp_credentials_example() -> Result<(), Box<dyn std::error::Error>> {
-    let iam = get()?;
-
-    // Create temporary credentials
-    let temp_creds = iam.create_temporary_credentials(
-        "john-doe",
-        Duration::from_secs(3600), // 1 hour
-        Some("read-only-policy".to_string())
-    ).await?;
-
-    println!("Temporary Access Key: {}", temp_creds.access_key);
-    println!("Expires at: {}", temp_creds.expiration);
-
-    // Validate temporary credentials
-    let is_valid = iam.validate_temporary_credentials(&temp_creds.access_key).await?;
-    println!("Temporary credentials valid: {}", is_valid);
-
-    Ok(())
-}
-```
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run tests with specific features
-cargo test --features "ldap,sso"
-
-# Run integration tests
-cargo test --test integration
-
-# Run authentication tests
-cargo test auth
-
-# Run authorization tests
-cargo test authz
-```
-
-## 🔒 Security Best Practices
-
-### Key Management
-
-- Rotate access keys regularly
-- Use strong, randomly generated keys
-- Store keys securely using environment variables or secret management systems
-- Implement key rotation policies
-
-### Access Control
-
-- Follow the principle of least privilege
-- Use groups for easier permission management
-- Regularly audit user permissions
-- Implement resource-based policies
-
-### Monitoring and Auditing
-
-- Enable comprehensive audit logging
-- Monitor failed authentication attempts
-- Set up alerts for suspicious activities
-- Regular security reviews
-
-## 📊 Performance Considerations
-
-### Caching Strategy
-
-- **User Cache**: Cache user information for faster lookups
-- **Policy Cache**: Cache policy documents to reduce latency
-- **Token Cache**: Cache JWT tokens for stateless authentication
-- **Permission Cache**: Cache authorization decisions
-
-### Scalability
-
-- **Distributed Cache**: Use distributed caching for multi-node deployments
-- **Database Optimization**: Optimize database queries for user/group lookups
-- **Connection Pooling**: Use connection pooling for database connections
-- **Async Operations**: Leverage async programming for better throughput
-
-## 🔧 Configuration
-
-### Basic Configuration
-
-```toml
-[iam]
-# Authentication settings
-jwt_secret = "your-jwt-secret-key"
-jwt_expiration = "24h"
-session_timeout = "30m"
-
-# Password policy
-min_password_length = 8
-require_special_chars = true
-require_numbers = true
-require_uppercase = true
-
-# Account lockout
-max_login_attempts = 5
-lockout_duration = "15m"
-
-# Audit settings
-audit_enabled = true
-audit_log_path = "/var/log/rustfs/iam-audit.log"
-```
-
-### Advanced Configuration
-
-```rust
-use rustfs_iam::config::IamConfig;
-
-let config = IamConfig {
-    // Authentication settings
-    jwt_secret: "your-secure-jwt-secret".to_string(),
-    jwt_expiration_hours: 24,
-    session_timeout_minutes: 30,
-
-    // Security settings
-    password_policy: PasswordPolicy {
-        min_length: 8,
-        require_special_chars: true,
-        require_numbers: true,
-        require_uppercase: true,
-        max_age_days: 90,
-    },
-
-    // Rate limiting
-    rate_limit: RateLimit {
-        max_requests_per_minute: 100,
-        burst_size: 10,
-    },
-
-    // Audit settings
-    audit_enabled: true,
-    audit_log_level: "info".to_string(),
-
-    ..Default::default()
-};
-```
-
-## 🤝 Integration with RustFS
-
-IAM integrates seamlessly with other RustFS components:
-
-- **ECStore**: Provides user and policy storage backend
-- **Policy Engine**: Implements fine-grained access control
-- **Crypto Module**: Handles secure key generation and JWT operations
-- **API Server**: Provides authentication and authorization for S3 API
-- **Admin Interface**: Manages users, groups, and policies
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Database**: Compatible with RustFS storage backend
-- **Memory**: Minimum 2GB RAM for caching
-- **Network**: Secure connections for authentication
-
-## 🐛 Troubleshooting
-
-### Common Issues
-
-1. **Authentication Failures**:
-   - Check access key and secret key validity
-   - Verify user account status (enabled/disabled)
-   - Check for account lockout due to failed attempts
-
-2. **Authorization Errors**:
-   - Verify user has required permissions
-   - Check policy attachments (user/group policies)
-   - Validate resource ARN format
-
-3. **Performance Issues**:
-   - Monitor cache hit rates
-   - Check database connection pool utilization
-   - Verify JWT token size and complexity
-
-### Debug Commands
-
-```bash
-# Check IAM system status
-rustfs-cli iam status
-
-# List all users
-rustfs-cli iam list-users
-
-# Validate user credentials
-rustfs-cli iam validate-credentials --access-key <key>
-
-# Test policy evaluation
-rustfs-cli iam test-policy --user <user> --action <action> --resource <resource>
-```
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS Policy](../policy) - Policy engine for access control
-- [RustFS Crypto](../crypto) - Cryptographic operations
-- [RustFS MadAdmin](../madmin) - Administrative interface
+- User and group management with RBAC
+- Service account and API key authentication
+- Policy engine with fine-grained permissions
+- LDAP/Active Directory integration
+- Multi-factor authentication support
+- Session management and token validation
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [IAM API Reference](https://docs.rustfs.com/iam/)
-- [Security Guide](https://docs.rustfs.com/security/)
-- [Authentication Guide](https://docs.rustfs.com/auth/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details on:
-
-- Security-first development practices
-- IAM system architecture guidelines
-- Authentication and authorization patterns
-- Testing procedures for security features
-- Documentation standards for security APIs
-
-### Development Setup
-
-```bash
-# Clone the repository
-git clone https://github.com/rustfs/rustfs.git
-cd rustfs
-
-# Navigate to IAM module
-cd crates/iam
-
-# Install dependencies
-cargo build
-
-# Run tests
-cargo test
-
-# Run security tests
-cargo test security
-
-# Format code
-cargo fmt
-
-# Run linter
-cargo clippy
-```
-
-## 💬 Getting Help
-
-- **Documentation**: [docs.rustfs.com](https://docs.rustfs.com)
-- **Issues**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-- **Security**: Report security issues to <security@rustfs.com>
-
-## 📞 Contact
-
-- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Business**: <hello@rustfs.com>
-- **Jobs**: <jobs@rustfs.com>
-- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-
-## 👥 Contributors
-
-This module is maintained by the RustFS security team and community contributors. Special thanks to all who have contributed to making RustFS secure and compliant.
-
-<a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=rustfs/rustfs" />
-</a>
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
-```
-Copyright 2024 RustFS Team
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-```
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔐 by the RustFS Security Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/lock/Cargo.toml

@@ -19,6 +19,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "Distributed locking mechanism for RustFS, providing synchronization and coordination across distributed systems."
+keywords = ["locking", "asynchronous", "distributed", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "asynchronous"]
+documentation = "https://docs.rs/rustfs-lock/latest/rustfs_lock/"
 
 [lints]
 workspace = true

crates/lock/README.md

@@ -3,7 +3,7 @@
 # RustFS Lock - Distributed Locking
 
 <p align="center">
-  <strong>Distributed locking and synchronization for RustFS object storage</strong>
+  <strong>High-performance distributed locking system for RustFS object storage</strong>
 </p>
 
 <p align="center">
@@ -17,376 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Lock** provides distributed locking and synchronization primitives for the [RustFS](https://rustfs.com) distributed object storage system. It ensures data consistency and prevents race conditions in multi-node environments through various locking mechanisms and coordination protocols.
-
-> **Note:** This is a core submodule of RustFS that provides essential distributed locking capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Lock** provides distributed locking capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔒 Distributed Locking
-
-- **Exclusive Locks**: Mutual exclusion across cluster nodes
-- **Shared Locks**: Reader-writer lock semantics
-- **Timeout Support**: Configurable lock timeouts and expiration
-- **Deadlock Prevention**: Automatic deadlock detection and resolution
-
-### 🔄 Synchronization Primitives
-
-- **Distributed Mutex**: Cross-node mutual exclusion
-- **Distributed Semaphore**: Resource counting across nodes
-- **Distributed Barrier**: Coordination point for multiple nodes
-- **Distributed Condition Variables**: Wait/notify across nodes
-
-### 🛡️ Consistency Guarantees
-
-- **Linearizable Operations**: Strong consistency guarantees
-- **Fault Tolerance**: Automatic recovery from node failures
-- **Network Partition Handling**: CAP theorem aware implementations
-- **Consensus Integration**: Raft-based consensus for critical locks
-
-### 🚀 Performance Features
-
-- **Lock Coalescing**: Efficient batching of lock operations
-- **Adaptive Timeouts**: Dynamic timeout adjustment
-- **Lock Hierarchy**: Hierarchical locking for better scalability
-- **Optimistic Locking**: Reduced contention through optimistic approaches
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-lock = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Distributed Lock
-
-```rust
-use rustfs_lock::{DistributedLock, LockManager, LockOptions};
-use std::time::Duration;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create lock manager
-    let lock_manager = LockManager::new("cluster-endpoint").await?;
-
-    // Acquire distributed lock
-    let lock_options = LockOptions {
-        timeout: Duration::from_secs(30),
-        auto_renew: true,
-        ..Default::default()
-    };
-
-    let lock = lock_manager.acquire_lock("resource-key", lock_options).await?;
-
-    // Critical section
-    {
-        println!("Lock acquired, performing critical operations...");
-        // Your critical code here
-        tokio::time::sleep(Duration::from_secs(2)).await;
-    }
-
-    // Release lock
-    lock.release().await?;
-    println!("Lock released");
-
-    Ok(())
-}
-```
-
-### Distributed Mutex
-
-```rust
-use rustfs_lock::{DistributedMutex, LockManager};
-use std::sync::Arc;
-
-async fn distributed_mutex_example() -> Result<(), Box<dyn std::error::Error>> {
-    let lock_manager = Arc::new(LockManager::new("cluster-endpoint").await?);
-
-    // Create distributed mutex
-    let mutex = DistributedMutex::new(lock_manager.clone(), "shared-resource");
-
-    // Spawn multiple tasks
-    let mut handles = vec![];
-
-    for i in 0..5 {
-        let mutex = mutex.clone();
-        let handle = tokio::spawn(async move {
-            let _guard = mutex.lock().await.unwrap();
-            println!("Task {} acquired mutex", i);
-
-            // Simulate work
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
-            println!("Task {} releasing mutex", i);
-            // Guard is automatically released when dropped
-        });
-
-        handles.push(handle);
-    }
-
-    // Wait for all tasks to complete
-    for handle in handles {
-        handle.await?;
-    }
-
-    Ok(())
-}
-```
-
-### Distributed Semaphore
-
-```rust
-use rustfs_lock::{DistributedSemaphore, LockManager};
-use std::sync::Arc;
-
-async fn distributed_semaphore_example() -> Result<(), Box<dyn std::error::Error>> {
-    let lock_manager = Arc::new(LockManager::new("cluster-endpoint").await?);
-
-    // Create distributed semaphore with 3 permits
-    let semaphore = DistributedSemaphore::new(
-        lock_manager.clone(),
-        "resource-pool",
-        3
-    );
-
-    // Spawn multiple tasks
-    let mut handles = vec![];
-
-    for i in 0..10 {
-        let semaphore = semaphore.clone();
-        let handle = tokio::spawn(async move {
-            let _permit = semaphore.acquire().await.unwrap();
-            println!("Task {} acquired permit", i);
-
-            // Simulate work
-            tokio::time::sleep(Duration::from_secs(2)).await;
-
-            println!("Task {} releasing permit", i);
-            // Permit is automatically released when dropped
-        });
-
-        handles.push(handle);
-    }
-
-    // Wait for all tasks to complete
-    for handle in handles {
-        handle.await?;
-    }
-
-    Ok(())
-}
-```
-
-### Distributed Barrier
-
-```rust
-use rustfs_lock::{DistributedBarrier, LockManager};
-use std::sync::Arc;
-
-async fn distributed_barrier_example() -> Result<(), Box<dyn std::error::Error>> {
-    let lock_manager = Arc::new(LockManager::new("cluster-endpoint").await?);
-
-    // Create distributed barrier for 3 participants
-    let barrier = DistributedBarrier::new(
-        lock_manager.clone(),
-        "sync-point",
-        3
-    );
-
-    // Spawn multiple tasks
-    let mut handles = vec![];
-
-    for i in 0..3 {
-        let barrier = barrier.clone();
-        let handle = tokio::spawn(async move {
-            println!("Task {} doing work...", i);
-
-            // Simulate different work durations
-            tokio::time::sleep(Duration::from_secs(i + 1)).await;
-
-            println!("Task {} waiting at barrier", i);
-            barrier.wait().await.unwrap();
-
-            println!("Task {} passed barrier", i);
-        });
-
-        handles.push(handle);
-    }
-
-    // Wait for all tasks to complete
-    for handle in handles {
-        handle.await?;
-    }
-
-    Ok(())
-}
-```
-
-### Lock with Automatic Renewal
-
-```rust
-use rustfs_lock::{DistributedLock, LockManager, LockOptions};
-use std::time::Duration;
-
-async fn auto_renewal_example() -> Result<(), Box<dyn std::error::Error>> {
-    let lock_manager = LockManager::new("cluster-endpoint").await?;
-
-    let lock_options = LockOptions {
-        timeout: Duration::from_secs(10),
-        auto_renew: true,
-        renew_interval: Duration::from_secs(3),
-        max_renewals: 5,
-        ..Default::default()
-    };
-
-    let lock = lock_manager.acquire_lock("long-running-task", lock_options).await?;
-
-    // Long-running operation
-    for i in 0..20 {
-        println!("Working on step {}", i);
-        tokio::time::sleep(Duration::from_secs(2)).await;
-
-        // Check if lock is still valid
-        if !lock.is_valid().await? {
-            println!("Lock lost, aborting operation");
-            break;
-        }
-    }
-
-    lock.release().await?;
-    Ok(())
-}
-```
-
-### Hierarchical Locking
-
-```rust
-use rustfs_lock::{LockManager, LockHierarchy, LockOptions};
-
-async fn hierarchical_locking_example() -> Result<(), Box<dyn std::error::Error>> {
-    let lock_manager = LockManager::new("cluster-endpoint").await?;
-
-    // Create lock hierarchy
-    let hierarchy = LockHierarchy::new(vec![
-        "global-lock".to_string(),
-        "bucket-lock".to_string(),
-        "object-lock".to_string(),
-    ]);
-
-    // Acquire locks in hierarchy order
-    let locks = lock_manager.acquire_hierarchical_locks(
-        hierarchy,
-        LockOptions::default()
-    ).await?;
-
-    // Critical section with hierarchical locks
-    {
-        println!("All hierarchical locks acquired");
-        // Perform operations that require the full lock hierarchy
-        tokio::time::sleep(Duration::from_secs(1)).await;
-    }
-
-    // Locks are automatically released in reverse order
-    locks.release_all().await?;
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Lock Architecture
-
-```
-Lock Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Lock API Layer                           │
-├─────────────────────────────────────────────────────────────┤
-│   Mutex    │   Semaphore   │   Barrier   │   Condition     │
-├─────────────────────────────────────────────────────────────┤
-│              Lock Manager                                   │
-├─────────────────────────────────────────────────────────────┤
-│   Consensus   │   Heartbeat   │   Timeout   │   Recovery    │
-├─────────────────────────────────────────────────────────────┤
-│              Distributed Coordination                       │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Lock Types
-
-| Type | Use Case | Guarantees |
-|------|----------|------------|
-| Exclusive | Critical sections | Mutual exclusion |
-| Shared | Reader-writer | Multiple readers |
-| Semaphore | Resource pooling | Counting semaphore |
-| Barrier | Synchronization | Coordination point |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test distributed locking
-cargo test distributed_lock
-
-# Test synchronization primitives
-cargo test sync_primitives
-
-# Test fault tolerance
-cargo test fault_tolerance
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Network**: Cluster connectivity required
-- **Consensus**: Raft consensus for critical operations
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Common](../common) - Common types and utilities
-- [RustFS Protos](../protos) - Protocol buffer definitions
+- Distributed lock management across cluster nodes
+- Read-write lock support with concurrent readers
+- Lock timeout and automatic lease renewal
+- Deadlock detection and prevention
+- High-availability with leader election
+- Performance-optimized locking algorithms
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Lock API Reference](https://docs.rustfs.com/lock/)
-- [Distributed Systems Guide](https://docs.rustfs.com/distributed/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔒 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/madmin/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Management and administration tools for RustFS, providing a web interface and API for system management."
+keywords = ["management", "administration", "web-interface", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "config"]
+documentation = "https://docs.rs/rustfs-madmin/latest/rustfs_madmin/"
 
 [lints]
 workspace = true

crates/madmin/README.md

@@ -3,7 +3,7 @@
 # RustFS MadAdmin - Administrative Interface
 
 <p align="center">
-  <strong>Administrative interface and management APIs for RustFS distributed object storage</strong>
+  <strong>Advanced administrative interface and management tools for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,335 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS MadAdmin** provides comprehensive administrative interfaces and management APIs for the [RustFS](https://rustfs.com) distributed object storage system. It enables cluster management, monitoring, configuration, and administrative operations through both programmatic APIs and interactive interfaces.
-
-> **Note:** This is a core submodule of RustFS that provides essential administrative capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS MadAdmin** provides advanced administrative interface and management tools for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🎛️ Cluster Management
-
-- **Node Management**: Add, remove, and monitor cluster nodes
-- **Service Discovery**: Automatic service discovery and registration
-- **Load Balancing**: Distribute load across cluster nodes
-- **Health Monitoring**: Real-time cluster health monitoring
-
-### 📊 System Monitoring
-
-- **Performance Metrics**: CPU, memory, disk, and network metrics
-- **Storage Analytics**: Capacity planning and usage analytics
-- **Alert Management**: Configurable alerts and notifications
-- **Dashboard Interface**: Web-based monitoring dashboard
-
-### ⚙️ Configuration Management
-
-- **Dynamic Configuration**: Runtime configuration updates
-- **Policy Management**: Access control and bucket policies
-- **User Management**: User and group administration
-- **Backup Configuration**: Backup and restore settings
-
-### 🔧 Administrative Operations
-
-- **Data Migration**: Cross-cluster data migration
-- **Healing Operations**: Data integrity repair and healing
-- **Rebalancing**: Storage rebalancing operations
-- **Maintenance Mode**: Graceful maintenance operations
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-madmin = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Admin Client
-
-```rust
-use rustfs_madmin::{AdminClient, AdminConfig, ServerInfo};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create admin client
-    let config = AdminConfig {
-        endpoint: "https://admin.rustfs.local:9001".to_string(),
-        access_key: "admin".to_string(),
-        secret_key: "password".to_string(),
-        region: "us-east-1".to_string(),
-    };
-
-    let client = AdminClient::new(config).await?;
-
-    // Get server information
-    let server_info = client.server_info().await?;
-    println!("Server Version: {}", server_info.version);
-    println!("Uptime: {}", server_info.uptime);
-
-    Ok(())
-}
-```
-
-### Cluster Management
-
-```rust
-use rustfs_madmin::{AdminClient, AddServerRequest, RemoveServerRequest};
-
-async fn cluster_management(client: &AdminClient) -> Result<(), Box<dyn std::error::Error>> {
-    // List cluster nodes
-    let nodes = client.list_servers().await?;
-    for node in nodes {
-        println!("Node: {} - Status: {}", node.endpoint, node.state);
-    }
-
-    // Add new server to cluster
-    let add_request = AddServerRequest {
-        endpoint: "https://new-node.rustfs.local:9000".to_string(),
-        access_key: "node-key".to_string(),
-        secret_key: "node-secret".to_string(),
-    };
-
-    client.add_server(add_request).await?;
-    println!("New server added successfully");
-
-    // Remove server from cluster
-    let remove_request = RemoveServerRequest {
-        endpoint: "https://old-node.rustfs.local:9000".to_string(),
-    };
-
-    client.remove_server(remove_request).await?;
-    println!("Server removed successfully");
-
-    Ok(())
-}
-```
-
-### System Monitoring
-
-```rust
-use rustfs_madmin::{AdminClient, MetricsRequest, AlertConfig};
-
-async fn monitoring_operations(client: &AdminClient) -> Result<(), Box<dyn std::error::Error>> {
-    // Get system metrics
-    let metrics = client.get_metrics(MetricsRequest::default()).await?;
-
-    println!("CPU Usage: {:.2}%", metrics.cpu_usage);
-    println!("Memory Usage: {:.2}%", metrics.memory_usage);
-    println!("Disk Usage: {:.2}%", metrics.disk_usage);
-
-    // Get storage information
-    let storage_info = client.storage_info().await?;
-    println!("Total Capacity: {} GB", storage_info.total_capacity / 1024 / 1024 / 1024);
-    println!("Used Capacity: {} GB", storage_info.used_capacity / 1024 / 1024 / 1024);
-
-    // Configure alerts
-    let alert_config = AlertConfig {
-        name: "high-cpu-usage".to_string(),
-        condition: "cpu_usage > 80".to_string(),
-        notification_endpoint: "https://webhook.example.com/alerts".to_string(),
-        enabled: true,
-    };
-
-    client.set_alert_config(alert_config).await?;
-
-    Ok(())
-}
-```
-
-### User and Policy Management
-
-```rust
-use rustfs_madmin::{AdminClient, UserInfo, PolicyDocument};
-
-async fn user_management(client: &AdminClient) -> Result<(), Box<dyn std::error::Error>> {
-    // Create user
-    let user_info = UserInfo {
-        access_key: "user123".to_string(),
-        secret_key: "user-secret".to_string(),
-        status: "enabled".to_string(),
-        policy: Some("readwrite-policy".to_string()),
-    };
-
-    client.add_user("new-user", user_info).await?;
-
-    // List users
-    let users = client.list_users().await?;
-    for (username, info) in users {
-        println!("User: {} - Status: {}", username, info.status);
-    }
-
-    // Set user policy
-    let policy_doc = PolicyDocument {
-        version: "2012-10-17".to_string(),
-        statement: vec![/* policy statements */],
-    };
-
-    client.set_user_policy("new-user", policy_doc).await?;
-
-    Ok(())
-}
-```
-
-### Data Operations
-
-```rust
-use rustfs_madmin::{AdminClient, HealRequest, RebalanceRequest};
-
-async fn data_operations(client: &AdminClient) -> Result<(), Box<dyn std::error::Error>> {
-    // Start healing operation
-    let heal_request = HealRequest {
-        bucket: Some("important-bucket".to_string()),
-        prefix: Some("documents/".to_string()),
-        recursive: true,
-        dry_run: false,
-    };
-
-    let heal_result = client.heal(heal_request).await?;
-    println!("Healing started: {}", heal_result.heal_sequence);
-
-    // Check healing status
-    let heal_status = client.heal_status(&heal_result.heal_sequence).await?;
-    println!("Healing progress: {:.2}%", heal_status.progress);
-
-    // Start rebalancing
-    let rebalance_request = RebalanceRequest {
-        servers: vec![], // Empty means all servers
-    };
-
-    client.start_rebalance(rebalance_request).await?;
-    println!("Rebalancing started");
-
-    Ok(())
-}
-```
-
-### Configuration Management
-
-```rust
-use rustfs_madmin::{AdminClient, ConfigUpdate, NotificationTarget};
-
-async fn configuration_management(client: &AdminClient) -> Result<(), Box<dyn std::error::Error>> {
-    // Get current configuration
-    let config = client.get_config().await?;
-    println!("Current config version: {}", config.version);
-
-    // Update configuration
-    let config_update = ConfigUpdate {
-        region: Some("us-west-2".to_string()),
-        browser: Some(true),
-        compression: Some(true),
-        // ... other config fields
-    };
-
-    client.set_config(config_update).await?;
-
-    // Configure notification targets
-    let notification_target = NotificationTarget {
-        arn: "arn:aws:sns:us-east-1:123456789012:my-topic".to_string(),
-        target_type: "webhook".to_string(),
-        endpoint: "https://webhook.example.com/notifications".to_string(),
-    };
-
-    client.set_notification_target("bucket1", notification_target).await?;
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### MadAdmin Architecture
-
-```
-MadAdmin Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Admin API Layer                          │
-├─────────────────────────────────────────────────────────────┤
-│   Cluster Mgmt   │   Monitoring    │   User Management      │
-├─────────────────────────────────────────────────────────────┤
-│   Data Ops       │   Config Mgmt   │   Notification         │
-├─────────────────────────────────────────────────────────────┤
-│              HTTP/gRPC Client Layer                         │
-├─────────────────────────────────────────────────────────────┤
-│              Storage System Integration                      │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Administrative Operations
-
-| Category | Operations | Description |
-|----------|------------|-------------|
-| Cluster | Add/Remove nodes, Health checks | Cluster management |
-| Monitoring | Metrics, Alerts, Dashboard | System monitoring |
-| Data | Healing, Rebalancing, Migration | Data operations |
-| Config | Settings, Policies, Notifications | Configuration |
-| Users | Authentication, Authorization | User management |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test admin operations
-cargo test admin_ops
-
-# Test cluster management
-cargo test cluster
-
-# Test monitoring
-cargo test monitoring
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Network**: Administrative access to RustFS cluster
-- **Permissions**: Administrative credentials required
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS Policy](../policy) - Policy engine
-- [RustFS Common](../common) - Common types and utilities
+- Comprehensive cluster management and monitoring
+- Real-time performance metrics and analytics
+- Automated backup and disaster recovery tools
+- User and permission management interface
+- System health monitoring and alerting
+- Configuration management and deployment tools
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [MadAdmin API Reference](https://docs.rustfs.com/madmin/)
-- [Administrative Guide](https://docs.rustfs.com/admin/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🎛️ by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/notify/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "File system notification service for RustFS, providing real-time updates on file changes and events."
+keywords = ["file-system", "notification", "real-time", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "filesystem"]
+documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
 
 [dependencies]
 rustfs-config = { workspace = true, features = ["notify"] }

crates/notify/README.md

@@ -3,7 +3,7 @@
 # RustFS Notify - Event Notification System
 
 <p align="center">
-  <strong>Real-time event notification system for RustFS distributed object storage</strong>
+  <strong>Real-time event notification and messaging system for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,399 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Notify** is the event notification system for the [RustFS](https://rustfs.com) distributed object storage platform. It provides real-time event publishing and delivery to various targets including webhooks, MQTT brokers, and message queues, enabling seamless integration with external systems and workflows.
-
-> **Note:** This is a core submodule of RustFS that provides essential event notification capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Notify** provides real-time event notification and messaging capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📡 Event Publishing
-
-- **Real-time Events**: Instant notification of storage events
-- **Event Filtering**: Advanced filtering based on object patterns and event types
-- **Reliable Delivery**: Guaranteed delivery with retry mechanisms
-- **Batch Processing**: Efficient batch event delivery
-
-### 🎯 Multiple Targets
-
-- **Webhooks**: HTTP/HTTPS webhook notifications
-- **MQTT**: MQTT broker integration for IoT scenarios
-- **Message Queues**: Integration with popular message queue systems
-- **Custom Targets**: Extensible target system for custom integrations
-
-### 🔧 Advanced Features
-
-- **Event Transformation**: Custom event payload transformation
-- **Pattern Matching**: Flexible pattern-based event filtering
-- **Rate Limiting**: Configurable rate limiting for targets
-- **Dead Letter Queue**: Failed event handling and recovery
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-notify = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Event Notification
-
-```rust
-use rustfs_notify::{Event, EventType, NotificationTarget, NotifySystem};
-use rustfs_notify::target::WebhookTarget;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create notification system
-    let notify_system = NotifySystem::new().await?;
-
-    // Create webhook target
-    let webhook = WebhookTarget::new(
-        "webhook-1",
-        "https://api.example.com/webhook",
-        vec![EventType::ObjectCreated, EventType::ObjectRemoved],
-    );
-
-    // Add target to notification system
-    notify_system.add_target(Box::new(webhook)).await?;
-
-    // Create and send event
-    let event = Event::new(
-        EventType::ObjectCreated,
-        "my-bucket",
-        "path/to/object.txt",
-        "user123",
-    );
-
-    notify_system.publish_event(event).await?;
-
-    Ok(())
-}
-```
-
-### Advanced Event Configuration
-
-```rust
-use rustfs_notify::{Event, EventType, NotificationConfig};
-use rustfs_notify::target::{WebhookTarget, MqttTarget};
-use rustfs_notify::filter::{EventFilter, PatternRule};
-
-async fn setup_advanced_notifications() -> Result<(), Box<dyn std::error::Error>> {
-    let notify_system = NotifySystem::new().await?;
-
-    // Create webhook with custom configuration
-    let webhook_config = NotificationConfig {
-        retry_attempts: 3,
-        retry_delay: std::time::Duration::from_secs(5),
-        timeout: std::time::Duration::from_secs(30),
-        rate_limit: Some(100), // 100 events per minute
-        ..Default::default()
-    };
-
-    let webhook = WebhookTarget::builder()
-        .id("production-webhook")
-        .url("https://api.example.com/events")
-        .events(vec![EventType::ObjectCreated, EventType::ObjectRemoved])
-        .config(webhook_config)
-        .headers(vec![
-            ("Authorization".to_string(), "Bearer token123".to_string()),
-            ("Content-Type".to_string(), "application/json".to_string()),
-        ])
-        .build()?;
-
-    // Create MQTT target
-    let mqtt_target = MqttTarget::builder()
-        .id("iot-mqtt")
-        .broker_url("mqtt://broker.example.com:1883")
-        .topic("storage/events")
-        .qos(1)
-        .events(vec![EventType::ObjectCreated])
-        .build()?;
-
-    // Add targets
-    notify_system.add_target(Box::new(webhook)).await?;
-    notify_system.add_target(Box::new(mqtt_target)).await?;
-
-    Ok(())
-}
-```
-
-### Event Filtering and Pattern Matching
-
-```rust
-use rustfs_notify::filter::{EventFilter, PatternRule, ConditionRule};
-
-fn setup_event_filters() -> Result<EventFilter, Box<dyn std::error::Error>> {
-    let filter = EventFilter::builder()
-        // Only images and documents
-        .pattern_rule(PatternRule::new(
-            "suffix",
-            vec!["*.jpg", "*.png", "*.pdf", "*.doc"]
-        ))
-        // Exclude temporary files
-        .pattern_rule(PatternRule::new(
-            "exclude",
-            vec!["*/tmp/*", "*.tmp"]
-        ))
-        // Only files larger than 1MB
-        .condition_rule(ConditionRule::new(
-            "object_size",
-            ">",
-            1024 * 1024
-        ))
-        // Only from specific buckets
-        .bucket_filter(vec!["important-bucket", "backup-bucket"])
-        .build()?;
-
-    Ok(filter)
-}
-```
-
-### Custom Target Implementation
-
-```rust
-use rustfs_notify::{Event, NotificationTarget, TargetResult};
-use async_trait::async_trait;
-
-pub struct SlackTarget {
-    id: String,
-    webhook_url: String,
-    channel: String,
-}
-
-#[async_trait]
-impl NotificationTarget for SlackTarget {
-    fn id(&self) -> &str {
-        &self.id
-    }
-
-    async fn deliver_event(&self, event: &Event) -> TargetResult<()> {
-        let message = format!(
-            "🔔 Storage Event: {} in bucket `{}` - object `{}`",
-            event.event_type,
-            event.bucket_name,
-            event.object_name
-        );
-
-        let payload = serde_json::json!({
-            "text": message,
-            "channel": self.channel,
-        });
-
-        let client = reqwest::Client::new();
-        let response = client
-            .post(&self.webhook_url)
-            .json(&payload)
-            .send()
-            .await?;
-
-        if response.status().is_success() {
-            Ok(())
-        } else {
-            Err(format!("Slack delivery failed: {}", response.status()).into())
-        }
-    }
-
-    fn supports_event_type(&self, event_type: &EventType) -> bool {
-        // Support all event types
-        true
-    }
-}
-```
-
-### Event Transformation
-
-```rust
-use rustfs_notify::{Event, EventTransformer};
-use serde_json::{json, Value};
-
-pub struct CustomEventTransformer;
-
-impl EventTransformer for CustomEventTransformer {
-    fn transform(&self, event: &Event) -> Value {
-        json!({
-            "eventVersion": "2.1",
-            "eventSource": "rustfs:s3",
-            "eventTime": event.timestamp.to_rfc3339(),
-            "eventName": event.event_type.to_string(),
-            "s3": {
-                "bucket": {
-                    "name": event.bucket_name,
-                    "arn": format!("arn:aws:s3:::{}", event.bucket_name)
-                },
-                "object": {
-                    "key": event.object_name,
-                    "size": event.object_size.unwrap_or(0),
-                    "eTag": event.etag.as_ref().unwrap_or(&"".to_string()),
-                }
-            },
-            "userIdentity": {
-                "principalId": event.user_identity
-            }
-        })
-    }
-}
-```
-
-## 🏗️ Architecture
-
-### Notification System Architecture
-
-```
-Notify Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Event Publisher                          │
-├─────────────────────────────────────────────────────────────┤
-│   Event Filter   │   Event Queue   │   Event Transformer   │
-├─────────────────────────────────────────────────────────────┤
-│              Target Manager                                  │
-├─────────────────────────────────────────────────────────────┤
-│  Webhook Target  │   MQTT Target   │   Custom Targets       │
-├─────────────────────────────────────────────────────────────┤
-│              Delivery Engine                                 │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Supported Event Types
-
-| Event Type | Description | Triggers |
-|-----------|-------------|----------|
-| `ObjectCreated` | Object creation events | PUT, POST, COPY |
-| `ObjectRemoved` | Object deletion events | DELETE |
-| `ObjectAccessed` | Object access events | GET, HEAD |
-| `ObjectRestore` | Object restoration events | Restore operations |
-| `BucketCreated` | Bucket creation events | CreateBucket |
-| `BucketRemoved` | Bucket deletion events | DeleteBucket |
-
-### Target Types
-
-| Target | Protocol | Use Case | Reliability |
-|--------|----------|----------|------------|
-| Webhook | HTTP/HTTPS | Web applications, APIs | High |
-| MQTT | MQTT | IoT devices, real-time systems | Medium |
-| Message Queue | AMQP, Redis | Microservices, async processing | High |
-| Custom | Any | Specialized integrations | Configurable |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run integration tests
-cargo test --test integration
-
-# Test webhook delivery
-cargo test webhook
-
-# Test MQTT integration
-cargo test mqtt
-
-# Run with coverage
-cargo test --features test-coverage
-```
-
-## ⚙️ Configuration
-
-### Basic Configuration
-
-```toml
-[notify]
-# Global settings
-enabled = true
-max_concurrent_deliveries = 100
-default_retry_attempts = 3
-default_timeout = "30s"
-
-# Queue settings
-event_queue_size = 10000
-batch_size = 100
-batch_timeout = "5s"
-
-# Dead letter queue
-dlq_enabled = true
-dlq_max_size = 1000
-```
-
-### Target Configuration
-
-```toml
-[[notify.targets]]
-type = "webhook"
-id = "primary-webhook"
-url = "https://api.example.com/webhook"
-events = ["ObjectCreated", "ObjectRemoved"]
-retry_attempts = 5
-timeout = "30s"
-
-[[notify.targets]]
-type = "mqtt"
-id = "iot-broker"
-broker_url = "mqtt://broker.example.com:1883"
-topic = "storage/events"
-qos = 1
-events = ["ObjectCreated"]
-```
-
-## 🚀 Performance
-
-The notification system is designed for high-throughput scenarios:
-
-- **Async Processing**: Non-blocking event delivery
-- **Batch Delivery**: Efficient batch processing for high-volume events
-- **Connection Pooling**: Reused connections for better performance
-- **Rate Limiting**: Configurable rate limiting to prevent overwhelming targets
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Network**: Outbound connectivity for target delivery
-- **Memory**: Scales with event queue size
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS Config](../config) - Configuration management
-- [RustFS Utils](../utils) - Utility functions
+- Real-time event streaming and notifications
+- Multiple notification targets (HTTP, Kafka, Redis, Email)
+- Event filtering and routing based on criteria
+- Message queuing with guaranteed delivery
+- Event replay and auditing capabilities
+- High-throughput messaging with batching support
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Notify API Reference](https://docs.rustfs.com/notify/)
-- [Event Configuration Guide](https://docs.rustfs.com/events/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 📡 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/obs/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Observability and monitoring tools for RustFS, providing metrics, logging, and tracing capabilities."
+keywords = ["observability", "metrics", "logging", "tracing", "RustFS"]
+categories = ["web-programming", "development-tools::profiling", "asynchronous", "api-bindings", "development-tools::debugging"]
+documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
 
 [lints]
 workspace = true

crates/obs/README.md

@@ -3,7 +3,7 @@
 # RustFS Obs - Observability & Monitoring
 
 <p align="center">
-  <strong>Comprehensive observability and monitoring solution for RustFS distributed object storage</strong>
+  <strong>Comprehensive observability and monitoring system for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,457 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Obs** provides comprehensive observability and monitoring capabilities for the [RustFS](https://rustfs.com) distributed object storage system. It includes metrics collection, distributed tracing, logging, alerting, and performance monitoring to ensure optimal system operation and troubleshooting.
-
-> **Note:** This is a critical operational submodule of RustFS that provides essential observability capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Obs** provides comprehensive observability and monitoring capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📊 Metrics Collection
-
-- **Prometheus Integration**: Native Prometheus metrics export
-- **Custom Metrics**: Application-specific performance metrics
-- **System Metrics**: CPU, memory, disk, and network monitoring
-- **Business Metrics**: Storage usage, request rates, and error tracking
-
-### 🔍 Distributed Tracing
-
-- **OpenTelemetry Support**: Standard distributed tracing
-- **Request Tracking**: End-to-end request lifecycle tracking
-- **Performance Analysis**: Latency and bottleneck identification
-- **Cross-Service Correlation**: Trace requests across microservices
-
-### 📝 Structured Logging
-
-- **JSON Logging**: Machine-readable structured logs
-- **Log Levels**: Configurable log levels and filtering
-- **Context Propagation**: Request context in all logs
-- **Log Aggregation**: Centralized log collection support
-
-### 🚨 Alerting & Notifications
-
-- **Rule-Based Alerts**: Configurable alerting rules
-- **Multiple Channels**: Email, Slack, webhook notifications
-- **Alert Escalation**: Tiered alerting and escalation policies
-- **Alert Correlation**: Group related alerts together
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-obs = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Observability Setup
-
-```rust
-use rustfs_obs::{ObservabilityConfig, MetricsCollector, TracingProvider};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Configure observability
-    let config = ObservabilityConfig {
-        service_name: "rustfs-storage".to_string(),
-        metrics_endpoint: "http://prometheus:9090".to_string(),
-        tracing_endpoint: "http://jaeger:14268/api/traces".to_string(),
-        log_level: "info".to_string(),
-        enable_metrics: true,
-        enable_tracing: true,
-    };
-
-    // Initialize observability
-    let obs = rustfs_obs::init(config).await?;
-
-    // Your application code here
-    run_application().await?;
-
-    // Shutdown observability
-    obs.shutdown().await?;
-
-    Ok(())
-}
-```
-
-### Metrics Collection
-
-```rust
-use rustfs_obs::metrics::{Counter, Histogram, Gauge, register_counter};
-
-// Define metrics
-lazy_static! {
-    static ref REQUESTS_TOTAL: Counter = register_counter!(
-        "rustfs_requests_total",
-        "Total number of requests",
-        &["method", "status"]
-    ).unwrap();
-
-    static ref REQUEST_DURATION: Histogram = register_histogram!(
-        "rustfs_request_duration_seconds",
-        "Request duration in seconds",
-        &["method"]
-    ).unwrap();
-
-    static ref ACTIVE_CONNECTIONS: Gauge = register_gauge!(
-        "rustfs_active_connections",
-        "Number of active connections"
-    ).unwrap();
-}
-
-async fn handle_request(method: &str) -> Result<(), Box<dyn std::error::Error>> {
-    let _timer = REQUEST_DURATION.with_label_values(&[method]).start_timer();
-
-    // Increment active connections
-    ACTIVE_CONNECTIONS.inc();
-
-    // Simulate request processing
-    tokio::time::sleep(Duration::from_millis(100)).await;
-
-    // Record request completion
-    REQUESTS_TOTAL.with_label_values(&[method, "success"]).inc();
-
-    // Decrement active connections
-    ACTIVE_CONNECTIONS.dec();
-
-    Ok(())
-}
-```
-
-### Distributed Tracing
-
-```rust
-use rustfs_obs::tracing::{trace_fn, Span, SpanContext};
-use tracing::{info, instrument};
-
-#[instrument(skip(data))]
-async fn process_upload(bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn std::error::Error>> {
-    let span = Span::current();
-    span.set_attribute("bucket", bucket);
-    span.set_attribute("key", key);
-    span.set_attribute("size", data.len() as i64);
-
-    info!("Starting upload process");
-
-    // Validate data
-    let validation_result = validate_data(data).await?;
-    span.add_event("data_validated", &[("result", &validation_result)]);
-
-    // Store data
-    let storage_result = store_data(bucket, key, data).await?;
-    span.add_event("data_stored", &[("etag", &storage_result.etag)]);
-
-    // Update metadata
-    update_metadata(bucket, key, &storage_result).await?;
-    span.add_event("metadata_updated", &[]);
-
-    info!("Upload completed successfully");
-    Ok(storage_result.etag)
-}
-
-#[instrument]
-async fn validate_data(data: &[u8]) -> Result<String, Box<dyn std::error::Error>> {
-    // Validation logic
-    tokio::time::sleep(Duration::from_millis(50)).await;
-    Ok("valid".to_string())
-}
-
-#[instrument]
-async fn store_data(bucket: &str, key: &str, data: &[u8]) -> Result<StorageResult, Box<dyn std::error::Error>> {
-    // Storage logic
-    tokio::time::sleep(Duration::from_millis(200)).await;
-    Ok(StorageResult {
-        etag: "d41d8cd98f00b204e9800998ecf8427e".to_string(),
-    })
-}
-```
-
-### Structured Logging
-
-```rust
-use rustfs_obs::logging::{LogEvent, LogLevel, StructuredLogger};
-use serde_json::json;
-
-async fn logging_example() -> Result<(), Box<dyn std::error::Error>> {
-    let logger = StructuredLogger::new();
-
-    // Basic logging
-    logger.info("Application started").await;
-
-    // Structured logging with context
-    logger.log(LogEvent {
-        level: LogLevel::Info,
-        message: "Processing upload request".to_string(),
-        context: json!({
-            "bucket": "example-bucket",
-            "key": "example-object",
-            "size": 1024,
-            "user_id": "user123",
-            "request_id": "req-456"
-        }),
-        timestamp: chrono::Utc::now(),
-    }).await;
-
-    // Error logging with details
-    logger.error_with_context(
-        "Failed to process upload",
-        json!({
-            "error_code": "STORAGE_FULL",
-            "bucket": "example-bucket",
-            "available_space": 0,
-            "required_space": 1024
-        })
-    ).await;
-
-    Ok(())
-}
-```
-
-### Alerting Configuration
-
-```rust
-use rustfs_obs::alerting::{AlertManager, AlertRule, NotificationChannel};
-
-async fn setup_alerting() -> Result<(), Box<dyn std::error::Error>> {
-    let alert_manager = AlertManager::new().await?;
-
-    // Configure notification channels
-    let slack_channel = NotificationChannel::Slack {
-        webhook_url: "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK".to_string(),
-        channel: "#rustfs-alerts".to_string(),
-    };
-
-    let email_channel = NotificationChannel::Email {
-        smtp_server: "smtp.example.com".to_string(),
-        recipients: vec!["admin@example.com".to_string()],
-    };
-
-    alert_manager.add_notification_channel("slack", slack_channel).await?;
-    alert_manager.add_notification_channel("email", email_channel).await?;
-
-    // Define alert rules
-    let high_error_rate = AlertRule {
-        name: "high_error_rate".to_string(),
-        description: "High error rate detected".to_string(),
-        condition: "rate(rustfs_requests_total{status!=\"success\"}[5m]) > 0.1".to_string(),
-        severity: "critical".to_string(),
-        notifications: vec!["slack".to_string(), "email".to_string()],
-        cooldown: Duration::from_minutes(15),
-    };
-
-    let low_disk_space = AlertRule {
-        name: "low_disk_space".to_string(),
-        description: "Disk space running low".to_string(),
-        condition: "rustfs_disk_usage_percent > 85".to_string(),
-        severity: "warning".to_string(),
-        notifications: vec!["slack".to_string()],
-        cooldown: Duration::from_minutes(30),
-    };
-
-    alert_manager.add_rule(high_error_rate).await?;
-    alert_manager.add_rule(low_disk_space).await?;
-
-    // Start alert monitoring
-    alert_manager.start().await?;
-
-    Ok(())
-}
-```
-
-### Performance Monitoring
-
-```rust
-use rustfs_obs::monitoring::{PerformanceMonitor, SystemMetrics, ApplicationMetrics};
-
-async fn performance_monitoring() -> Result<(), Box<dyn std::error::Error>> {
-    let monitor = PerformanceMonitor::new().await?;
-
-    // Start system monitoring
-    monitor.start_system_monitoring(Duration::from_secs(10)).await?;
-
-    // Custom application metrics
-    let app_metrics = ApplicationMetrics::new();
-
-    // Monitor specific operations
-    let upload_metrics = app_metrics.create_operation_monitor("upload");
-    let download_metrics = app_metrics.create_operation_monitor("download");
-
-    // Simulate operations with monitoring
-    tokio::spawn(async move {
-        loop {
-            // Monitor upload operation
-            let upload_timer = upload_metrics.start_timer();
-            simulate_upload().await;
-            upload_timer.record_success();
-
-            // Monitor download operation
-            let download_timer = download_metrics.start_timer();
-            match simulate_download().await {
-                Ok(_) => download_timer.record_success(),
-                Err(_) => download_timer.record_error(),
-            }
-
-            tokio::time::sleep(Duration::from_secs(1)).await;
-        }
-    });
-
-    // Periodic metrics reporting
-    tokio::spawn(async move {
-        let mut interval = tokio::time::interval(Duration::from_secs(60));
-
-        loop {
-            interval.tick().await;
-
-            let system_metrics = monitor.get_system_metrics().await;
-            let app_metrics = monitor.get_application_metrics().await;
-
-            println!("=== System Metrics ===");
-            println!("CPU Usage: {:.2}%", system_metrics.cpu_usage);
-            println!("Memory Usage: {:.2}%", system_metrics.memory_usage);
-            println!("Disk Usage: {:.2}%", system_metrics.disk_usage);
-
-            println!("=== Application Metrics ===");
-            println!("Upload Throughput: {:.2} ops/sec", app_metrics.upload_throughput);
-            println!("Download Throughput: {:.2} ops/sec", app_metrics.download_throughput);
-            println!("Error Rate: {:.2}%", app_metrics.error_rate);
-        }
-    });
-
-    Ok(())
-}
-```
-
-### Health Checks
-
-```rust
-use rustfs_obs::health::{HealthChecker, HealthStatus, HealthCheck};
-
-async fn setup_health_checks() -> Result<(), Box<dyn std::error::Error>> {
-    let health_checker = HealthChecker::new();
-
-    // Add component health checks
-    health_checker.add_check("database", Box::new(DatabaseHealthCheck)).await;
-    health_checker.add_check("storage", Box::new(StorageHealthCheck)).await;
-    health_checker.add_check("cache", Box::new(CacheHealthCheck)).await;
-
-    // Start health monitoring
-    health_checker.start_monitoring(Duration::from_secs(30)).await?;
-
-    // Expose health endpoint
-    health_checker.expose_http_endpoint("0.0.0.0:8080").await?;
-
-    Ok(())
-}
-
-struct DatabaseHealthCheck;
-
-#[async_trait::async_trait]
-impl HealthCheck for DatabaseHealthCheck {
-    async fn check(&self) -> HealthStatus {
-        // Perform database health check
-        match check_database_connection().await {
-            Ok(_) => HealthStatus::Healthy,
-            Err(e) => HealthStatus::Unhealthy(e.to_string()),
-        }
-    }
-}
-```
-
-## 🏗️ Architecture
-
-### Observability Architecture
-
-```
-Observability Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Observability API                        │
-├─────────────────────────────────────────────────────────────┤
-│   Metrics     │   Tracing     │   Logging    │   Alerting   │
-├─────────────────────────────────────────────────────────────┤
-│              Data Collection & Processing                    │
-├─────────────────────────────────────────────────────────────┤
-│  Prometheus   │  OpenTelemetry │  Structured  │  Alert Mgr  │
-├─────────────────────────────────────────────────────────────┤
-│              External Integrations                          │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Monitoring Stack
-
-| Component | Purpose | Integration |
-|-----------|---------|-------------|
-| Prometheus | Metrics storage | Pull-based metrics collection |
-| Jaeger | Distributed tracing | OpenTelemetry traces |
-| Grafana | Visualization | Dashboards and alerts |
-| ELK Stack | Log aggregation | Structured log processing |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test metrics collection
-cargo test metrics
-
-# Test tracing functionality
-cargo test tracing
-
-# Test alerting
-cargo test alerting
-
-# Integration tests
-cargo test --test integration
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **External Services**: Prometheus, Jaeger (optional)
-- **Network**: HTTP endpoint exposure capability
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Common](../common) - Common types and utilities
-- [RustFS Config](../config) - Configuration management
+- OpenTelemetry integration for distributed tracing
+- Prometheus metrics collection and exposition
+- Structured logging with configurable levels
+- Performance profiling and analytics
+- Real-time health checks and status monitoring
+- Custom dashboards and alerting integration
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Obs API Reference](https://docs.rustfs.com/obs/)
-- [Monitoring Guide](https://docs.rustfs.com/monitoring/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 📊 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/obs/src/telemetry.rs

@@ -419,7 +419,7 @@ fn format_with_color(w: &mut dyn std::io::Write, now: &mut DeferredNow, record:
 
     writeln!(
         w,
-        "{} {} [{}] [{}:{}] [{}:{}] {}",
+        "[{}] {} [{}] [{}:{}] [{}:{}] {}",
         now.now().format("%Y-%m-%d %H:%M:%S%.6f"),
         level_style.paint(level.to_string()),
         Color::Magenta.paint(record.target()),
@@ -443,7 +443,7 @@ fn format_for_file(w: &mut dyn std::io::Write, now: &mut DeferredNow, record: &R
 
     writeln!(
         w,
-        "{} {} [{}] [{}:{}] [{}:{}] {}",
+        "[{}] {} [{}] [{}:{}] [{}:{}] {}",
         now.now().format("%Y-%m-%d %H:%M:%S%.6f"),
         level,
         record.target(),

crates/policy/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Policy management for RustFS, providing a framework for defining and enforcing policies across the system."
+keywords = ["policy", "management", "enforcement", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "accessibility"]
+documentation = "https://docs.rs/rustfs-policy/latest/rustfs_policy/"
 
 [lints]
 workspace = true

crates/policy/README.md

@@ -1,9 +1,9 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS Policy Engine
+# RustFS Policy - Policy Engine
 
 <p align="center">
-  <strong>Advanced policy-based access control engine for RustFS distributed object storage</strong>
+  <strong>Advanced policy engine and access control system for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,574 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Policy Engine** is a sophisticated access control system for the [RustFS](https://rustfs.com) distributed object storage platform. It provides fine-grained, attribute-based access control (ABAC) with support for complex policy expressions, dynamic evaluation, and AWS IAM-compatible policy syntax.
-
-> **Note:** This is a core submodule of RustFS that provides essential access control and authorization capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Policy** provides advanced policy engine and access control capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔐 Access Control
-
-- **AWS IAM Compatible**: Full support for AWS IAM policy syntax
-- **Fine-Grained Permissions**: Resource-level and action-level access control
-- **Dynamic Policy Evaluation**: Real-time policy evaluation with context
-- **Conditional Access**: Support for complex conditional expressions
-
-### 📜 Policy Management
-
-- **Policy Documents**: Structured policy definition and management
-- **Policy Versioning**: Version control for policy documents
-- **Policy Validation**: Syntax and semantic validation
-- **Policy Templates**: Pre-built policy templates for common use cases
-
-### 🎯 Advanced Features
-
-- **Attribute-Based Access Control (ABAC)**: Context-aware access decisions
-- **Function-Based Conditions**: Rich set of condition functions
-- **Principal-Based Policies**: User, group, and service account policies
-- **Resource-Based Policies**: Bucket and object-level policies
-
-### 🛠️ Integration Features
-
-- **ARN Support**: AWS-style Amazon Resource Names
-- **Multi-Tenant Support**: Isolated policy evaluation per tenant
-- **Real-Time Evaluation**: High-performance policy evaluation engine
-- **Audit Trail**: Comprehensive policy evaluation logging
-
-## 🏗️ Architecture
-
-### Policy Engine Architecture
-
-```
-Policy Engine Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Policy API Layer                         │
-├─────────────────────────────────────────────────────────────┤
-│   Policy Parser   │   Policy Validator   │   Policy Store   │
-├─────────────────────────────────────────────────────────────┤
-│              Policy Evaluation Engine                        │
-├─────────────────────────────────────────────────────────────┤
-│  Condition Functions  │  Principal Resolver  │  Resource Mgr │
-├─────────────────────────────────────────────────────────────┤
-│              Authentication Integration                      │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Policy Decision Flow
-
-```
-Policy Decision Flow:
-┌─────────────┐    ┌─────────────┐    ┌─────────────┐
-│   Request   │───▶│   Policy    │───▶│   Decision  │
-│  (Subject,  │    │  Evaluation │    │  (Allow/    │
-│   Action,   │    │   Engine    │    │   Deny/     │
-│  Resource)  │    │             │    │  Not Found) │
-└─────────────┘    └─────────────┘    └─────────────┘
-       │                   │                   │
-       ▼                   ▼                   ▼
-┌─────────────┐    ┌─────────────┐    ┌─────────────┐
-│  Context    │    │  Condition  │    │   Audit     │
-│ Information │    │ Functions   │    │    Log      │
-└─────────────┘    └─────────────┘    └─────────────┘
-```
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-policy = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Policy Creation
-
-```rust
-use rustfs_policy::policy::{Policy, Statement, Effect, Action, Resource};
-use rustfs_policy::arn::ARN;
-use serde_json::json;
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create a simple policy
-    let policy = Policy::new(
-        "2012-10-17".to_string(),
-        vec![
-            Statement::new(
-                Effect::Allow,
-                vec![Action::from_str("s3:GetObject")?],
-                vec![Resource::from_str("arn:aws:s3:::my-bucket/*")?],
-                None, // No conditions
-            ),
-        ],
-    );
-
-    // Serialize to JSON
-    let policy_json = serde_json::to_string_pretty(&policy)?;
-    println!("Policy JSON:\n{}", policy_json);
-
-    Ok(())
-}
-```
-
-### Advanced Policy with Conditions
-
-```rust
-use rustfs_policy::policy::{Policy, Statement, Effect, Action, Resource};
-use rustfs_policy::policy::function::{Function, FunctionName};
-use serde_json::json;
-
-fn create_conditional_policy() -> Result<Policy, Box<dyn std::error::Error>> {
-    // Create a policy with IP address restrictions
-    let policy = Policy::new(
-        "2012-10-17".to_string(),
-        vec![
-            Statement::builder()
-                .effect(Effect::Allow)
-                .action(Action::from_str("s3:GetObject")?)
-                .resource(Resource::from_str("arn:aws:s3:::secure-bucket/*")?)
-                .condition(
-                    "IpAddress",
-                    "aws:SourceIp",
-                    json!(["192.168.1.0/24", "10.0.0.0/8"])
-                )
-                .build(),
-            Statement::builder()
-                .effect(Effect::Deny)
-                .action(Action::from_str("s3:*")?)
-                .resource(Resource::from_str("arn:aws:s3:::secure-bucket/*")?)
-                .condition(
-                    "DateGreaterThan",
-                    "aws:CurrentTime",
-                    json!("2024-12-31T23:59:59Z")
-                )
-                .build(),
-        ],
-    );
-
-    Ok(policy)
-}
-```
-
-### Policy Evaluation
-
-```rust
-use rustfs_policy::policy::{Policy, PolicyDoc};
-use rustfs_policy::auth::{Identity, Request};
-use rustfs_policy::arn::ARN;
-
-async fn evaluate_policy_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Load policy from storage
-    let policy_doc = PolicyDoc::try_from(policy_bytes)?;
-
-    // Create evaluation context
-    let identity = Identity::new(
-        "user123".to_string(),
-        vec!["group1".to_string(), "group2".to_string()],
-    );
-
-    let request = Request::new(
-        "s3:GetObject".to_string(),
-        ARN::from_str("arn:aws:s3:::my-bucket/file.txt")?,
-        Some(json!({
-            "aws:SourceIp": "192.168.1.100",
-            "aws:CurrentTime": "2024-01-15T10:30:00Z"
-        })),
-    );
-
-    // Evaluate policy
-    let result = policy_doc.policy.evaluate(&identity, &request).await?;
-
-    match result {
-        Effect::Allow => println!("Access allowed"),
-        Effect::Deny => println!("Access denied"),
-        Effect::NotEvaluated => println!("No applicable policy found"),
-    }
-
-    Ok(())
-}
-```
-
-### Policy Templates
-
-```rust
-use rustfs_policy::policy::{Policy, Statement, Effect};
-use rustfs_policy::templates::PolicyTemplate;
-
-fn create_common_policies() -> Result<(), Box<dyn std::error::Error>> {
-    // Read-only policy template
-    let read_only_policy = PolicyTemplate::read_only_bucket("my-bucket")?;
-
-    // Full access policy template
-    let full_access_policy = PolicyTemplate::full_access_bucket("my-bucket")?;
-
-    // Admin policy template
-    let admin_policy = PolicyTemplate::admin_all_resources()?;
-
-    // Custom policy with multiple permissions
-    let custom_policy = Policy::builder()
-        .version("2012-10-17")
-        .statement(
-            Statement::builder()
-                .effect(Effect::Allow)
-                .action("s3:GetObject")
-                .action("s3:PutObject")
-                .resource("arn:aws:s3:::uploads/*")
-                .condition("StringEquals", "s3:x-amz-acl", "bucket-owner-full-control")
-                .build()
-        )
-        .statement(
-            Statement::builder()
-                .effect(Effect::Allow)
-                .action("s3:ListBucket")
-                .resource("arn:aws:s3:::uploads")
-                .condition("StringLike", "s3:prefix", "user/${aws:username}/*")
-                .build()
-        )
-        .build();
-
-    Ok(())
-}
-```
-
-### Resource-Based Policies
-
-```rust
-use rustfs_policy::policy::{Policy, Statement, Effect, Principal};
-use rustfs_policy::arn::ARN;
-
-fn create_resource_policy() -> Result<Policy, Box<dyn std::error::Error>> {
-    // Create a bucket policy allowing cross-account access
-    let bucket_policy = Policy::builder()
-        .version("2012-10-17")
-        .statement(
-            Statement::builder()
-                .effect(Effect::Allow)
-                .principal(Principal::AWS("arn:aws:iam::123456789012:root".to_string()))
-                .action("s3:GetObject")
-                .resource("arn:aws:s3:::shared-bucket/*")
-                .condition("StringEquals", "s3:ExistingObjectTag/Department", "Finance")
-                .build()
-        )
-        .statement(
-            Statement::builder()
-                .effect(Effect::Allow)
-                .principal(Principal::AWS("arn:aws:iam::123456789012:user/john".to_string()))
-                .action("s3:PutObject")
-                .resource("arn:aws:s3:::shared-bucket/uploads/*")
-                .condition("StringEquals", "s3:x-amz-acl", "bucket-owner-full-control")
-                .build()
-        )
-        .build();
-
-    Ok(bucket_policy)
-}
-```
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run policy evaluation tests
-cargo test policy_evaluation
-
-# Run condition function tests
-cargo test condition_functions
-
-# Run ARN parsing tests
-cargo test arn_parsing
-
-# Run policy validation tests
-cargo test policy_validation
-
-# Run with test coverage
-cargo test --features test-coverage
-```
-
-## 📋 Policy Syntax
-
-### Basic Policy Structure
-
-```json
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "s3:GetObject",
-        "s3:PutObject"
-      ],
-      "Resource": [
-        "arn:aws:s3:::my-bucket/*"
-      ],
-      "Condition": {
-        "StringEquals": {
-          "s3:x-amz-acl": "bucket-owner-full-control"
-        }
-      }
-    }
-  ]
-}
-```
-
-### Supported Actions
-
-| Action Category | Actions | Description |
-|----------------|---------|-------------|
-| Object Operations | `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject` | Object-level operations |
-| Bucket Operations | `s3:ListBucket`, `s3:CreateBucket`, `s3:DeleteBucket` | Bucket-level operations |
-| Access Control | `s3:GetBucketAcl`, `s3:PutBucketAcl` | Access control operations |
-| Versioning | `s3:GetObjectVersion`, `s3:DeleteObjectVersion` | Object versioning operations |
-| Multipart Upload | `s3:ListMultipartUploads`, `s3:AbortMultipartUpload` | Multipart upload operations |
-
-### Condition Functions
-
-| Function | Description | Example |
-|----------|-------------|---------|
-| `StringEquals` | Exact string matching | `"StringEquals": {"s3:x-amz-acl": "private"}` |
-| `StringLike` | Wildcard string matching | `"StringLike": {"s3:prefix": "photos/*"}` |
-| `IpAddress` | IP address/CIDR matching | `"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}` |
-| `DateGreaterThan` | Date comparison | `"DateGreaterThan": {"aws:CurrentTime": "2024-01-01T00:00:00Z"}` |
-| `NumericEquals` | Numeric comparison | `"NumericEquals": {"s3:max-keys": "100"}` |
-| `Bool` | Boolean comparison | `"Bool": {"aws:SecureTransport": "true"}` |
-
-## 🔧 Configuration
-
-### Policy Engine Configuration
-
-```toml
-[policy]
-# Policy evaluation settings
-max_policy_size = 2048  # Maximum policy size in KB
-max_conditions_per_statement = 10
-max_statements_per_policy = 100
-
-# Performance settings
-cache_policy_documents = true
-cache_ttl_seconds = 300
-max_cached_policies = 1000
-
-# Security settings
-require_secure_transport = true
-allow_anonymous_access = false
-default_effect = "deny"
-
-# Audit settings
-audit_policy_evaluation = true
-audit_log_path = "/var/log/rustfs/policy-audit.log"
-```
-
-### Advanced Configuration
-
-```rust
-use rustfs_policy::config::PolicyConfig;
-
-let config = PolicyConfig {
-    // Policy parsing settings
-    max_policy_size_kb: 2048,
-    max_conditions_per_statement: 10,
-    max_statements_per_policy: 100,
-
-    // Evaluation settings
-    default_effect: Effect::Deny,
-    require_explicit_deny: false,
-    cache_policy_documents: true,
-    cache_ttl_seconds: 300,
-
-    // Security settings
-    require_secure_transport: true,
-    allow_anonymous_access: false,
-    validate_resource_arns: true,
-
-    // Performance settings
-    max_cached_policies: 1000,
-    evaluation_timeout_ms: 100,
-
-    ..Default::default()
-};
-```
-
-## 🚀 Performance Optimization
-
-### Caching Strategy
-
-- **Policy Document Cache**: Cache parsed policy documents
-- **Evaluation Result Cache**: Cache evaluation results for identical requests
-- **Condition Cache**: Cache condition function results
-- **Principal Cache**: Cache principal resolution results
-
-### Best Practices
-
-1. **Minimize Policy Size**: Keep policies as small as possible
-2. **Use Specific Actions**: Avoid overly broad action patterns
-3. **Optimize Conditions**: Use efficient condition functions
-4. **Cache Frequently Used Policies**: Enable policy caching for better performance
-
-## 🤝 Integration with RustFS
-
-The Policy Engine integrates seamlessly with other RustFS components:
-
-- **IAM Module**: Provides policy storage and user/group management
-- **ECStore**: Implements resource-based access control
-- **API Server**: Enforces policies on S3 API operations
-- **Audit System**: Logs policy evaluation decisions
-- **Admin Interface**: Manages policy documents and templates
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Memory**: Minimum 1GB RAM for policy caching
-- **Storage**: Compatible with RustFS storage backend
-
-## 🐛 Troubleshooting
-
-### Common Issues
-
-1. **Policy Parse Errors**:
-   - Check JSON syntax validity
-   - Verify action and resource ARN formats
-   - Validate condition function syntax
-
-2. **Policy Evaluation Failures**:
-   - Check principal resolution
-   - Verify resource ARN matching
-   - Debug condition function evaluation
-
-3. **Performance Issues**:
-   - Monitor policy cache hit rates
-   - Check policy document sizes
-   - Optimize condition functions
-
-### Debug Commands
-
-```bash
-# Validate policy syntax
-rustfs-cli policy validate --file policy.json
-
-# Test policy evaluation
-rustfs-cli policy test --policy policy.json --user john --action s3:GetObject --resource arn:aws:s3:::bucket/key
-
-# Show policy evaluation trace
-rustfs-cli policy trace --policy policy.json --user john --action s3:GetObject --resource arn:aws:s3:::bucket/key
-```
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS Crypto](../crypto) - Cryptographic operations
-- [RustFS Utils](../utils) - Utility functions
+- AWS-compatible bucket policy engine
+- Fine-grained resource-based access control
+- Condition-based policy evaluation
+- Policy validation and syntax checking
+- Role-based access control integration
+- Dynamic policy evaluation with context
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Policy Engine API Reference](https://docs.rustfs.com/policy/)
-- [Policy Language Guide](https://docs.rustfs.com/policy-language/)
-- [Access Control Guide](https://docs.rustfs.com/access-control/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details on:
-
-- Policy engine architecture and design patterns
-- Policy language syntax and semantics
-- Condition function implementation
-- Performance optimization techniques
-- Security considerations for access control
-
-### Development Setup
-
-```bash
-# Clone the repository
-git clone https://github.com/rustfs/rustfs.git
-cd rustfs
-
-# Navigate to Policy module
-cd crates/policy
-
-# Install dependencies
-cargo build
-
-# Run tests
-cargo test
-
-# Run policy validation tests
-cargo test policy_validation
-
-# Format code
-cargo fmt
-
-# Run linter
-cargo clippy
-```
-
-## 💬 Getting Help
-
-- **Documentation**: [docs.rustfs.com](https://docs.rustfs.com)
-- **Issues**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-- **Security**: Report security issues to <security@rustfs.com>
-
-## 📞 Contact
-
-- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
-- **Business**: <hello@rustfs.com>
-- **Jobs**: <jobs@rustfs.com>
-- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-
-## 👥 Contributors
-
-This module is maintained by the RustFS security team and community contributors. Special thanks to all who have contributed to making RustFS access control robust and flexible.
-
-<a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=rustfs/rustfs" />
-</a>
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
-```
-Copyright 2024 RustFS Team
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-```
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🛡️ by the RustFS Security Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/protos/Cargo.toml

@@ -16,6 +16,14 @@
 name = "rustfs-protos"
 version.workspace = true
 edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "Protocol definitions for RustFS, providing gRPC and FlatBuffers interfaces for communication between components."
+keywords = ["protocols", "gRPC", "FlatBuffers", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "data-structures", "asynchronous"]
+documentation = "https://docs.rs/rustfs-protos/latest/rustfs_protos/"
 
 [lints]
 workspace = true

crates/protos/README.md

@@ -3,7 +3,7 @@
 # RustFS Protos - Protocol Buffer Definitions
 
 <p align="center">
-  <strong>Protocol buffer definitions and gRPC interfaces for RustFS distributed object storage</strong>
+  <strong>Protocol buffer definitions and gRPC services for RustFS distributed object storage</strong>
 </p>
 
 <p align="center">
@@ -17,410 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Protos** provides protocol buffer definitions and gRPC service interfaces for the [RustFS](https://rustfs.com) distributed object storage system. It defines the communication protocols, message formats, and service contracts used across all RustFS components.
-
-> **Note:** This is a foundational submodule of RustFS that provides essential communication protocols for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Protos** provides protocol buffer definitions and gRPC services for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📡 gRPC Services
-
-- **Storage Service**: Core storage operations (get, put, delete)
-- **Admin Service**: Administrative and management operations
-- **Metadata Service**: Metadata management and queries
-- **Lock Service**: Distributed locking and coordination
-
-### 📦 Message Types
-
-- **Storage Messages**: Object and bucket operation messages
-- **Administrative Messages**: Cluster management messages
-- **Metadata Messages**: File and object metadata structures
-- **Error Messages**: Standardized error reporting
-
-### 🔧 Protocol Features
-
-- **Versioning**: Protocol version compatibility management
-- **Extensions**: Custom field extensions for future expansion
-- **Streaming**: Support for streaming large data transfers
-- **Compression**: Built-in message compression support
-
-### 🛠️ Code Generation
-
-- **Rust Bindings**: Automatic Rust code generation
-- **Type Safety**: Strong typing for all protocol messages
-- **Documentation**: Generated API documentation
-- **Validation**: Message validation and constraints
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-protos = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic gRPC Client
-
-```rust
-use rustfs_protos::storage::{StorageServiceClient, GetObjectRequest, PutObjectRequest};
-use tonic::transport::Channel;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Connect to storage service
-    let channel = Channel::from_static("http://storage.rustfs.local:9000")
-        .connect()
-        .await?;
-
-    let mut client = StorageServiceClient::new(channel);
-
-    // Get object
-    let request = GetObjectRequest {
-        bucket: "example-bucket".to_string(),
-        key: "example-object".to_string(),
-        version_id: None,
-        range: None,
-    };
-
-    let response = client.get_object(request).await?;
-    let object_data = response.into_inner();
-
-    println!("Retrieved object: {} bytes", object_data.content.len());
-    println!("Content type: {}", object_data.content_type);
-    println!("ETag: {}", object_data.etag);
-
-    Ok(())
-}
-```
-
-### Storage Operations
-
-```rust
-use rustfs_protos::storage::{
-    StorageServiceClient, PutObjectRequest, DeleteObjectRequest,
-    ListObjectsRequest, CreateBucketRequest
-};
-
-async fn storage_operations_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut client = StorageServiceClient::connect("http://storage.rustfs.local:9000").await?;
-
-    // Create bucket
-    let create_bucket_request = CreateBucketRequest {
-        bucket: "new-bucket".to_string(),
-        region: "us-east-1".to_string(),
-        acl: None,
-        storage_class: "STANDARD".to_string(),
-    };
-
-    client.create_bucket(create_bucket_request).await?;
-    println!("Bucket created successfully");
-
-    // Put object
-    let put_request = PutObjectRequest {
-        bucket: "new-bucket".to_string(),
-        key: "test-file.txt".to_string(),
-        content: b"Hello, RustFS!".to_vec(),
-        content_type: "text/plain".to_string(),
-        metadata: std::collections::HashMap::new(),
-        storage_class: None,
-    };
-
-    let put_response = client.put_object(put_request).await?;
-    println!("Object uploaded, ETag: {}", put_response.into_inner().etag);
-
-    // List objects
-    let list_request = ListObjectsRequest {
-        bucket: "new-bucket".to_string(),
-        prefix: None,
-        marker: None,
-        max_keys: Some(100),
-        delimiter: None,
-    };
-
-    let list_response = client.list_objects(list_request).await?;
-    let objects = list_response.into_inner();
-
-    for object in objects.contents {
-        println!("Object: {} (size: {} bytes)", object.key, object.size);
-    }
-
-    Ok(())
-}
-```
-
-### Administrative Operations
-
-```rust
-use rustfs_protos::admin::{
-    AdminServiceClient, GetServerInfoRequest, AddServerRequest,
-    ListServersRequest, ConfigUpdateRequest
-};
-
-async fn admin_operations_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut client = AdminServiceClient::connect("http://admin.rustfs.local:9001").await?;
-
-    // Get server information
-    let info_request = GetServerInfoRequest {};
-    let info_response = client.get_server_info(info_request).await?;
-    let server_info = info_response.into_inner();
-
-    println!("Server version: {}", server_info.version);
-    println!("Uptime: {} seconds", server_info.uptime_seconds);
-    println!("Memory usage: {} MB", server_info.memory_usage_mb);
-
-    // List cluster servers
-    let list_request = ListServersRequest {};
-    let list_response = client.list_servers(list_request).await?;
-    let servers = list_response.into_inner();
-
-    for server in servers.servers {
-        println!("Server: {} - Status: {}", server.endpoint, server.status);
-    }
-
-    // Add new server
-    let add_request = AddServerRequest {
-        endpoint: "https://new-node.rustfs.local:9000".to_string(),
-        access_key: "node-access-key".to_string(),
-        secret_key: "node-secret-key".to_string(),
-    };
-
-    client.add_server(add_request).await?;
-    println!("New server added to cluster");
-
-    Ok(())
-}
-```
-
-### Metadata Operations
-
-```rust
-use rustfs_protos::metadata::{
-    MetadataServiceClient, SearchObjectsRequest, GetObjectMetadataRequest,
-    UpdateObjectMetadataRequest
-};
-
-async fn metadata_operations_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut client = MetadataServiceClient::connect("http://metadata.rustfs.local:9002").await?;
-
-    // Search objects
-    let search_request = SearchObjectsRequest {
-        query: "content_type:image/*".to_string(),
-        bucket: Some("photos-bucket".to_string()),
-        limit: Some(50),
-        offset: Some(0),
-    };
-
-    let search_response = client.search_objects(search_request).await?;
-    let results = search_response.into_inner();
-
-    for object in results.objects {
-        println!("Found: {} ({})", object.key, object.content_type);
-    }
-
-    // Get object metadata
-    let metadata_request = GetObjectMetadataRequest {
-        bucket: "photos-bucket".to_string(),
-        key: "vacation-photo.jpg".to_string(),
-    };
-
-    let metadata_response = client.get_object_metadata(metadata_request).await?;
-    let metadata = metadata_response.into_inner();
-
-    println!("Object metadata:");
-    for (key, value) in metadata.metadata {
-        println!("  {}: {}", key, value);
-    }
-
-    Ok(())
-}
-```
-
-### Lock Service Operations
-
-```rust
-use rustfs_protos::lock::{
-    LockServiceClient, AcquireLockRequest, ReleaseLockRequest,
-    LockType, LockMode
-};
-use std::time::Duration;
-
-async fn lock_operations_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut client = LockServiceClient::connect("http://lock.rustfs.local:9003").await?;
-
-    // Acquire distributed lock
-    let acquire_request = AcquireLockRequest {
-        resource_id: "bucket/important-data".to_string(),
-        lock_type: LockType::Exclusive as i32,
-        timeout_seconds: 30,
-        auto_renew: true,
-    };
-
-    let acquire_response = client.acquire_lock(acquire_request).await?;
-    let lock_token = acquire_response.into_inner().lock_token;
-
-    println!("Lock acquired: {}", lock_token);
-
-    // Perform critical operations
-    tokio::time::sleep(Duration::from_secs(5)).await;
-
-    // Release lock
-    let release_request = ReleaseLockRequest {
-        lock_token: lock_token.clone(),
-    };
-
-    client.release_lock(release_request).await?;
-    println!("Lock released: {}", lock_token);
-
-    Ok(())
-}
-```
-
-### Streaming Operations
-
-```rust
-use rustfs_protos::storage::{StorageServiceClient, StreamUploadRequest, StreamDownloadRequest};
-use tokio_stream::wrappers::ReceiverStream;
-
-async fn streaming_operations_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut client = StorageServiceClient::connect("http://storage.rustfs.local:9000").await?;
-
-    // Streaming upload
-    let (tx, rx) = tokio::sync::mpsc::channel(100);
-    let request_stream = ReceiverStream::new(rx);
-
-    // Send upload metadata
-    tx.send(StreamUploadRequest {
-        bucket: "large-files".to_string(),
-        key: "big-video.mp4".to_string(),
-        content_type: "video/mp4".to_string(),
-        chunk: vec![], // Empty chunk for metadata
-    }).await?;
-
-    // Send file chunks
-    let mut file = tokio::fs::File::open("big-video.mp4").await?;
-    let mut buffer = vec![0u8; 64 * 1024]; // 64KB chunks
-
-    tokio::spawn(async move {
-        loop {
-            match file.read(&mut buffer).await {
-                Ok(0) => break, // EOF
-                Ok(n) => {
-                    let chunk_request = StreamUploadRequest {
-                        bucket: String::new(),
-                        key: String::new(),
-                        content_type: String::new(),
-                        chunk: buffer[..n].to_vec(),
-                    };
-
-                    if tx.send(chunk_request).await.is_err() {
-                        break;
-                    }
-                }
-                Err(_) => break,
-            }
-        }
-    });
-
-    let upload_response = client.stream_upload(request_stream).await?;
-    println!("Upload completed: {}", upload_response.into_inner().etag);
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Protocol Architecture
-
-```
-Protocol Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    gRPC Services                            │
-├─────────────────────────────────────────────────────────────┤
-│  Storage API  │  Admin API   │  Metadata API │  Lock API   │
-├─────────────────────────────────────────────────────────────┤
-│              Protocol Buffer Messages                       │
-├─────────────────────────────────────────────────────────────┤
-│   Requests    │   Responses   │   Streaming   │   Errors    │
-├─────────────────────────────────────────────────────────────┤
-│              Transport Layer (HTTP/2)                       │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Service Definitions
-
-| Service | Purpose | Key Operations |
-|---------|---------|----------------|
-| Storage | Object operations | Get, Put, Delete, List |
-| Admin | Cluster management | Add/Remove nodes, Config |
-| Metadata | Metadata queries | Search, Index, Update |
-| Lock | Distributed locking | Acquire, Release, Renew |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test protocol buffer compilation
-cargo test proto_compilation
-
-# Test service interfaces
-cargo test service_interfaces
-
-# Test message serialization
-cargo test serialization
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Protocol Buffers compiler (protoc)
-- **Network**: gRPC transport support
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Common](../common) - Common types and utilities
-- [RustFS Lock](../lock) - Distributed locking
+- Comprehensive gRPC service definitions
+- Cross-language compatibility with Protocol Buffers
+- Efficient binary serialization for network communication
+- Versioned API schemas with backward compatibility
+- Type-safe message definitions
+- Code generation for multiple programming languages
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Protos API Reference](https://docs.rustfs.com/protos/)
-- [gRPC Guide](https://docs.rustfs.com/grpc/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 📡 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/rio/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Rio is a RustFS component that provides a high-performance, asynchronous I/O framework for building scalable and efficient applications."
+keywords = ["asynchronous", "IO", "framework", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "asynchronous"]
+documentation = "https://docs.rs/rustfs-rio/latest/rustfs_rio/"
 
 [lints]
 workspace = true

crates/rio/README.md

@@ -17,398 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Rio** provides high-performance asynchronous I/O operations for the [RustFS](https://rustfs.com) distributed object storage system. It implements efficient data streaming, encryption, compression, and integrity checking with zero-copy operations and optimized buffering strategies.
-
-> **Note:** This is a performance-critical submodule of RustFS that provides essential I/O capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Rio** provides high-performance asynchronous I/O operations for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🚀 High-Performance I/O
-
-- **Zero-Copy Operations**: Efficient data movement without unnecessary copying
-- **Async Streaming**: Non-blocking streaming I/O with backpressure handling
-- **Vectored I/O**: Scatter-gather operations for improved throughput
-- **Buffer Management**: Intelligent buffer pooling and reuse
-
-### 🔐 Cryptographic Operations
-
-- **AES-GCM Encryption**: Hardware-accelerated encryption/decryption
-- **Streaming Encryption**: Encrypt data on-the-fly without buffering
-- **Key Management**: Secure key derivation and rotation
-- **Digital Signatures**: Data integrity verification
-
-### 📦 Compression Support
-
-- **Multi-Algorithm**: Support for various compression algorithms
-- **Streaming Compression**: Real-time compression during transfer
-- **Adaptive Compression**: Dynamic algorithm selection based on data
-- **Compression Levels**: Configurable compression vs. speed tradeoffs
-
-### 🔧 Data Integrity
-
-- **CRC32 Checksums**: Fast integrity checking
-- **MD5 Hashing**: Legacy compatibility and verification
-- **Merkle Trees**: Hierarchical integrity verification
-- **Error Correction**: Automatic error detection and correction
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-rio = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Streaming I/O
-
-```rust
-use rustfs_rio::{StreamReader, StreamWriter, BufferPool};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create buffer pool for efficient memory management
-    let buffer_pool = BufferPool::new(64 * 1024, 100); // 64KB buffers, 100 in pool
-
-    // Create streaming reader
-    let mut reader = StreamReader::new(input_source, buffer_pool.clone());
-
-    // Create streaming writer
-    let mut writer = StreamWriter::new(output_destination, buffer_pool.clone());
-
-    // High-performance streaming copy
-    let mut buffer = vec![0u8; 8192];
-    loop {
-        let n = reader.read(&mut buffer).await?;
-        if n == 0 {
-            break;
-        }
-        writer.write_all(&buffer[..n]).await?;
-    }
-
-    writer.flush().await?;
-    Ok(())
-}
-```
-
-### Encrypted Streaming
-
-```rust
-use rustfs_rio::{EncryptedWriter, EncryptedReader, EncryptionKey};
-use aes_gcm::{Aes256Gcm, Key, Nonce};
-
-async fn encrypted_streaming_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Generate encryption key
-    let key = EncryptionKey::generate()?;
-
-    // Create encrypted writer
-    let mut encrypted_writer = EncryptedWriter::new(
-        output_stream,
-        key.clone(),
-        Aes256Gcm::new(&key.into())
-    )?;
-
-    // Write encrypted data
-    encrypted_writer.write_all(b"Hello, encrypted world!").await?;
-    encrypted_writer.finalize().await?;
-
-    // Create encrypted reader
-    let mut encrypted_reader = EncryptedReader::new(
-        input_stream,
-        key.clone(),
-        Aes256Gcm::new(&key.into())
-    )?;
-
-    // Read decrypted data
-    let mut decrypted_data = Vec::new();
-    encrypted_reader.read_to_end(&mut decrypted_data).await?;
-
-    println!("Decrypted: {}", String::from_utf8(decrypted_data)?);
-    Ok(())
-}
-```
-
-### Compressed Streaming
-
-```rust
-use rustfs_rio::{CompressedWriter, CompressedReader, CompressionAlgorithm};
-
-async fn compressed_streaming_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create compressed writer
-    let mut compressed_writer = CompressedWriter::new(
-        output_stream,
-        CompressionAlgorithm::Zstd,
-        6 // compression level
-    )?;
-
-    // Write compressed data
-    compressed_writer.write_all(b"This data will be compressed").await?;
-    compressed_writer.write_all(b"and streamed efficiently").await?;
-    compressed_writer.finish().await?;
-
-    // Create compressed reader
-    let mut compressed_reader = CompressedReader::new(
-        input_stream,
-        CompressionAlgorithm::Zstd
-    )?;
-
-    // Read decompressed data
-    let mut decompressed_data = Vec::new();
-    compressed_reader.read_to_end(&mut decompressed_data).await?;
-
-    println!("Decompressed: {}", String::from_utf8(decompressed_data)?);
-    Ok(())
-}
-```
-
-### Integrity Checking
-
-```rust
-use rustfs_rio::{ChecksumWriter, ChecksumReader, ChecksumAlgorithm};
-
-async fn integrity_checking_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create checksum writer
-    let mut checksum_writer = ChecksumWriter::new(
-        output_stream,
-        ChecksumAlgorithm::Crc32
-    );
-
-    // Write data with checksum calculation
-    checksum_writer.write_all(b"Data with integrity checking").await?;
-    let write_checksum = checksum_writer.finalize().await?;
-
-    println!("Write checksum: {:08x}", write_checksum);
-
-    // Create checksum reader
-    let mut checksum_reader = ChecksumReader::new(
-        input_stream,
-        ChecksumAlgorithm::Crc32
-    );
-
-    // Read data with checksum verification
-    let mut data = Vec::new();
-    checksum_reader.read_to_end(&mut data).await?;
-    let read_checksum = checksum_reader.checksum();
-
-    println!("Read checksum: {:08x}", read_checksum);
-
-    // Verify integrity
-    if write_checksum == read_checksum {
-        println!("Data integrity verified!");
-    } else {
-        println!("Data corruption detected!");
-    }
-
-    Ok(())
-}
-```
-
-### Multi-Layer Streaming
-
-```rust
-use rustfs_rio::{MultiLayerWriter, MultiLayerReader, Layer};
-
-async fn multi_layer_streaming_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create multi-layer writer (compression + encryption + checksum)
-    let mut writer = MultiLayerWriter::new(output_stream)
-        .add_layer(Layer::Compression(CompressionAlgorithm::Zstd, 6))
-        .add_layer(Layer::Encryption(encryption_key.clone()))
-        .add_layer(Layer::Checksum(ChecksumAlgorithm::Crc32))
-        .build()?;
-
-    // Write data through all layers
-    writer.write_all(b"This data will be compressed, encrypted, and checksummed").await?;
-    let final_checksum = writer.finalize().await?;
-
-    // Create multi-layer reader (reverse order)
-    let mut reader = MultiLayerReader::new(input_stream)
-        .add_layer(Layer::Checksum(ChecksumAlgorithm::Crc32))
-        .add_layer(Layer::Decryption(encryption_key.clone()))
-        .add_layer(Layer::Decompression(CompressionAlgorithm::Zstd))
-        .build()?;
-
-    // Read data through all layers
-    let mut data = Vec::new();
-    reader.read_to_end(&mut data).await?;
-
-    // Verify final checksum
-    if reader.verify_checksum(final_checksum)? {
-        println!("All layers verified successfully!");
-    }
-
-    Ok(())
-}
-```
-
-### Vectored I/O Operations
-
-```rust
-use rustfs_rio::{VectoredWriter, VectoredReader, IoVec};
-
-async fn vectored_io_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create vectored writer
-    let mut vectored_writer = VectoredWriter::new(output_stream);
-
-    // Prepare multiple buffers
-    let header = b"HEADER";
-    let data = b"Important data content";
-    let footer = b"FOOTER";
-
-    // Write multiple buffers in one operation
-    let io_vecs = vec![
-        IoVec::new(header),
-        IoVec::new(data),
-        IoVec::new(footer),
-    ];
-
-    let bytes_written = vectored_writer.write_vectored(&io_vecs).await?;
-    println!("Wrote {} bytes in vectored operation", bytes_written);
-
-    // Create vectored reader
-    let mut vectored_reader = VectoredReader::new(input_stream);
-
-    // Read into multiple buffers
-    let mut header_buf = vec![0u8; 6];
-    let mut data_buf = vec![0u8; 22];
-    let mut footer_buf = vec![0u8; 6];
-
-    let mut read_vecs = vec![
-        IoVec::new_mut(&mut header_buf),
-        IoVec::new_mut(&mut data_buf),
-        IoVec::new_mut(&mut footer_buf),
-    ];
-
-    let bytes_read = vectored_reader.read_vectored(&mut read_vecs).await?;
-    println!("Read {} bytes in vectored operation", bytes_read);
-
-    Ok(())
-}
-```
-
-### Async Stream Processing
-
-```rust
-use rustfs_rio::{AsyncStreamProcessor, ProcessorChain};
-use futures::StreamExt;
-
-async fn stream_processing_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create processor chain
-    let processor = ProcessorChain::new()
-        .add_processor(Box::new(CompressionProcessor::new(CompressionAlgorithm::Zstd)))
-        .add_processor(Box::new(EncryptionProcessor::new(encryption_key)))
-        .add_processor(Box::new(ChecksumProcessor::new(ChecksumAlgorithm::Crc32)));
-
-    // Create async stream processor
-    let mut stream_processor = AsyncStreamProcessor::new(input_stream, processor);
-
-    // Process stream chunks
-    while let Some(chunk) = stream_processor.next().await {
-        let processed_chunk = chunk?;
-
-        // Handle processed chunk
-        output_stream.write_all(&processed_chunk).await?;
-    }
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Rio Architecture
-
-```
-Rio I/O Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Stream API Layer                         │
-├─────────────────────────────────────────────────────────────┤
-│ Encryption  │ Compression │ Checksum    │ Vectored I/O     │
-├─────────────────────────────────────────────────────────────┤
-│              Buffer Management                               │
-├─────────────────────────────────────────────────────────────┤
-│   Zero-Copy   │   Async I/O   │   Backpressure Control     │
-├─────────────────────────────────────────────────────────────┤
-│              Tokio Runtime Integration                       │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Performance Features
-
-| Feature | Benefit | Implementation |
-|---------|---------|----------------|
-| Zero-Copy | Reduced memory usage | Direct buffer operations |
-| Async I/O | High concurrency | Tokio-based operations |
-| Vectored I/O | Reduced syscalls | Scatter-gather operations |
-| Buffer Pooling | Memory efficiency | Reusable buffer management |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test streaming operations
-cargo test streaming
-
-# Test encryption
-cargo test encryption
-
-# Test compression
-cargo test compression
-
-# Run benchmarks
-cargo bench
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Tokio async runtime
-- **Hardware**: AES-NI support recommended for encryption
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Utils](../utils) - Utility functions
-- [RustFS Crypto](../crypto) - Cryptographic operations
+- Zero-copy streaming I/O operations
+- Hardware-accelerated encryption/decryption
+- Multi-algorithm compression support
+- Efficient buffer management and pooling
+- Vectored I/O for improved throughput
+- Real-time data integrity verification
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Rio API Reference](https://docs.rustfs.com/rio/)
-- [Performance Guide](https://docs.rustfs.com/performance/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🚀 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/s3select-api/Cargo.toml

@@ -19,6 +19,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "S3 Select API implementation for RustFS, enabling efficient data retrieval from S3-compatible object stores."
+keywords = ["s3-select", "api", "rustfs", "Minio", "object-store"]
+categories = ["web-programming", "development-tools", "asynchronous"]
+documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/"
 
 [dependencies]
 async-trait.workspace = true

crates/s3select-api/README.md

@@ -17,575 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS S3Select API** provides AWS S3 Select compatible SQL query capabilities for the [RustFS](https://rustfs.com) distributed object storage system. It enables clients to retrieve subsets of data from objects using SQL expressions, reducing data transfer and improving query performance through server-side filtering.
-
-> **Note:** This is a high-performance submodule of RustFS that provides essential SQL query capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS S3Select API** provides AWS S3 Select compatible SQL query capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📊 SQL Query Support
-
-- **Standard SQL**: Support for SELECT, WHERE, GROUP BY, ORDER BY clauses
-- **Data Types**: Support for strings, numbers, booleans, timestamps
-- **Functions**: Built-in SQL functions (aggregation, string, date functions)
-- **Complex Expressions**: Nested queries and complex conditional logic
-
-### 📁 Format Support
-
-- **CSV Files**: Comma-separated values with customizable delimiters
-- **JSON Documents**: JSON objects and arrays with path expressions
-- **Parquet Files**: Columnar format with schema evolution
-- **Apache Arrow**: High-performance columnar data format
-
-### 🚀 Performance Features
-
-- **Streaming Processing**: Process large files without loading into memory
-- **Parallel Execution**: Multi-threaded query execution
-- **Predicate Pushdown**: Push filters down to storage layer
-- **Columnar Processing**: Efficient columnar data processing with Apache DataFusion
-
-### 🔧 S3 Compatibility
-
-- **S3 Select API**: Full compatibility with AWS S3 Select API
-- **Request Formats**: Support for JSON and XML request formats
-- **Response Streaming**: Streaming query results back to clients
-- **Error Handling**: AWS-compatible error responses
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-s3select-api = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic S3 Select Query
-
-```rust
-use rustfs_s3select_api::{S3SelectService, SelectRequest, InputSerialization, OutputSerialization};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create S3 Select service
-    let s3select = S3SelectService::new().await?;
-
-    // Configure input format (CSV)
-    let input_serialization = InputSerialization::CSV {
-        file_header_info: "USE".to_string(),
-        record_delimiter: "\n".to_string(),
-        field_delimiter: ",".to_string(),
-        quote_character: "\"".to_string(),
-        quote_escape_character: "\"".to_string(),
-        comments: "#".to_string(),
-    };
-
-    // Configure output format
-    let output_serialization = OutputSerialization::JSON {
-        record_delimiter: "\n".to_string(),
-    };
-
-    // Create select request
-    let select_request = SelectRequest {
-        bucket: "sales-data".to_string(),
-        key: "2024/sales.csv".to_string(),
-        expression: "SELECT name, revenue FROM S3Object WHERE revenue > 10000".to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization,
-        output_serialization,
-        request_progress: false,
-    };
-
-    // Execute query
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    // Process streaming results
-    while let Some(event) = result_stream.next().await {
-        match event? {
-            SelectEvent::Records(data) => {
-                println!("Query result: {}", String::from_utf8(data)?);
-            }
-            SelectEvent::Stats(stats) => {
-                println!("Bytes scanned: {}", stats.bytes_scanned);
-                println!("Bytes processed: {}", stats.bytes_processed);
-                println!("Bytes returned: {}", stats.bytes_returned);
-            }
-            SelectEvent::Progress(progress) => {
-                println!("Progress: {}%", progress.details.bytes_processed_percent);
-            }
-            SelectEvent::End => {
-                println!("Query completed");
-                break;
-            }
-        }
-    }
-
-    Ok(())
-}
-```
-
-### CSV Data Processing
-
-```rust
-use rustfs_s3select_api::{S3SelectService, CSVInputSerialization};
-
-async fn csv_processing_example() -> Result<(), Box<dyn std::error::Error>> {
-    let s3select = S3SelectService::new().await?;
-
-    // Configure CSV input with custom settings
-    let csv_input = CSVInputSerialization {
-        file_header_info: "USE".to_string(),
-        record_delimiter: "\r\n".to_string(),
-        field_delimiter: "|".to_string(),
-        quote_character: "'".to_string(),
-        quote_escape_character: "\\".to_string(),
-        comments: "//".to_string(),
-        allow_quoted_record_delimiter: false,
-    };
-
-    // Query with aggregation
-    let select_request = SelectRequest {
-        bucket: "analytics".to_string(),
-        key: "user-events.csv".to_string(),
-        expression: r#"
-            SELECT
-                event_type,
-                COUNT(*) as event_count,
-                AVG(CAST(duration as DECIMAL)) as avg_duration
-            FROM S3Object
-            WHERE timestamp >= '2024-01-01'
-            GROUP BY event_type
-            ORDER BY event_count DESC
-        "#.to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization: InputSerialization::CSV(csv_input),
-        output_serialization: OutputSerialization::JSON {
-            record_delimiter: "\n".to_string(),
-        },
-        request_progress: true,
-    };
-
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    let mut total_events = 0;
-    while let Some(event) = result_stream.next().await {
-        match event? {
-            SelectEvent::Records(data) => {
-                let result: serde_json::Value = serde_json::from_slice(&data)?;
-                println!("Event type: {}, Count: {}, Avg duration: {}",
-                    result["event_type"], result["event_count"], result["avg_duration"]);
-                total_events += result["event_count"].as_u64().unwrap_or(0);
-            }
-            SelectEvent::Progress(progress) => {
-                println!("Processing: {}%", progress.details.bytes_processed_percent);
-            }
-            _ => {}
-        }
-    }
-
-    println!("Total events processed: {}", total_events);
-    Ok(())
-}
-```
-
-### JSON Data Querying
-
-```rust
-use rustfs_s3select_api::{JSONInputSerialization, JSONType};
-
-async fn json_querying_example() -> Result<(), Box<dyn std::error::Error>> {
-    let s3select = S3SelectService::new().await?;
-
-    // Configure JSON input
-    let json_input = JSONInputSerialization {
-        json_type: JSONType::Lines, // JSON Lines format
-    };
-
-    // Query nested JSON data
-    let select_request = SelectRequest {
-        bucket: "logs".to_string(),
-        key: "application.jsonl".to_string(),
-        expression: r#"
-            SELECT
-                s.timestamp,
-                s.level,
-                s.message,
-                s.metadata.user_id,
-                s.metadata.request_id
-            FROM S3Object[*] s
-            WHERE s.level = 'ERROR'
-            AND s.metadata.user_id IS NOT NULL
-            ORDER BY s.timestamp DESC
-        "#.to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization: InputSerialization::JSON(json_input),
-        output_serialization: OutputSerialization::JSON {
-            record_delimiter: "\n".to_string(),
-        },
-        request_progress: false,
-    };
-
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    while let Some(event) = result_stream.next().await {
-        if let SelectEvent::Records(data) = event? {
-            let log_entry: serde_json::Value = serde_json::from_slice(&data)?;
-            println!("Error at {}: {} (User: {}, Request: {})",
-                log_entry["timestamp"],
-                log_entry["message"],
-                log_entry["user_id"],
-                log_entry["request_id"]
-            );
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Parquet File Analysis
-
-```rust
-use rustfs_s3select_api::{ParquetInputSerialization};
-
-async fn parquet_analysis_example() -> Result<(), Box<dyn std::error::Error>> {
-    let s3select = S3SelectService::new().await?;
-
-    // Parquet files don't need serialization configuration
-    let parquet_input = ParquetInputSerialization {};
-
-    // Complex analytical query
-    let select_request = SelectRequest {
-        bucket: "data-warehouse".to_string(),
-        key: "sales/2024/q1/sales_data.parquet".to_string(),
-        expression: r#"
-            SELECT
-                region,
-                product_category,
-                SUM(amount) as total_sales,
-                COUNT(*) as transaction_count,
-                AVG(amount) as avg_transaction,
-                MIN(amount) as min_sale,
-                MAX(amount) as max_sale
-            FROM S3Object
-            WHERE sale_date >= '2024-01-01'
-            AND sale_date < '2024-04-01'
-            AND amount > 0
-            GROUP BY region, product_category
-            HAVING SUM(amount) > 50000
-            ORDER BY total_sales DESC
-            LIMIT 20
-        "#.to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization: InputSerialization::Parquet(parquet_input),
-        output_serialization: OutputSerialization::JSON {
-            record_delimiter: "\n".to_string(),
-        },
-        request_progress: true,
-    };
-
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    while let Some(event) = result_stream.next().await {
-        match event? {
-            SelectEvent::Records(data) => {
-                let sales_data: serde_json::Value = serde_json::from_slice(&data)?;
-                println!("Region: {}, Category: {}, Total Sales: ${:.2}",
-                    sales_data["region"],
-                    sales_data["product_category"],
-                    sales_data["total_sales"]
-                );
-            }
-            SelectEvent::Stats(stats) => {
-                println!("Query statistics:");
-                println!("  Bytes scanned: {}", stats.bytes_scanned);
-                println!("  Bytes processed: {}", stats.bytes_processed);
-                println!("  Bytes returned: {}", stats.bytes_returned);
-            }
-            _ => {}
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Advanced SQL Functions
-
-```rust
-async fn advanced_sql_functions_example() -> Result<(), Box<dyn std::error::Error>> {
-    let s3select = S3SelectService::new().await?;
-
-    // Query with various SQL functions
-    let select_request = SelectRequest {
-        bucket: "analytics".to_string(),
-        key: "user_data.csv".to_string(),
-        expression: r#"
-            SELECT
-                -- String functions
-                UPPER(name) as name_upper,
-                SUBSTRING(email, 1, POSITION('@' IN email) - 1) as username,
-                LENGTH(description) as desc_length,
-
-                -- Date functions
-                EXTRACT(YEAR FROM registration_date) as reg_year,
-                DATE_DIFF('day', registration_date, last_login) as days_since_reg,
-
-                -- Numeric functions
-                ROUND(score, 2) as rounded_score,
-                CASE
-                    WHEN score >= 90 THEN 'Excellent'
-                    WHEN score >= 70 THEN 'Good'
-                    WHEN score >= 50 THEN 'Average'
-                    ELSE 'Poor'
-                END as score_category,
-
-                -- Conditional logic
-                COALESCE(nickname, SUBSTRING(name, 1, POSITION(' ' IN name) - 1)) as display_name
-
-            FROM S3Object
-            WHERE registration_date IS NOT NULL
-            AND score IS NOT NULL
-            ORDER BY score DESC
-        "#.to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization: InputSerialization::CSV {
-            file_header_info: "USE".to_string(),
-            record_delimiter: "\n".to_string(),
-            field_delimiter: ",".to_string(),
-            quote_character: "\"".to_string(),
-            quote_escape_character: "\"".to_string(),
-            comments: "#".to_string(),
-        },
-        output_serialization: OutputSerialization::JSON {
-            record_delimiter: "\n".to_string(),
-        },
-        request_progress: false,
-    };
-
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    while let Some(event) = result_stream.next().await {
-        if let SelectEvent::Records(data) = event? {
-            let user: serde_json::Value = serde_json::from_slice(&data)?;
-            println!("User: {} ({}) - Score: {} ({})",
-                user["display_name"],
-                user["username"],
-                user["rounded_score"],
-                user["score_category"]
-            );
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Streaming Large Datasets
-
-```rust
-use rustfs_s3select_api::{SelectObjectContentStream, ProgressDetails};
-
-async fn streaming_large_datasets() -> Result<(), Box<dyn std::error::Error>> {
-    let s3select = S3SelectService::new().await?;
-
-    let select_request = SelectRequest {
-        bucket: "big-data".to_string(),
-        key: "large_dataset.csv".to_string(),
-        expression: "SELECT * FROM S3Object WHERE status = 'active'".to_string(),
-        expression_type: "SQL".to_string(),
-        input_serialization: InputSerialization::CSV {
-            file_header_info: "USE".to_string(),
-            record_delimiter: "\n".to_string(),
-            field_delimiter: ",".to_string(),
-            quote_character: "\"".to_string(),
-            quote_escape_character: "\"".to_string(),
-            comments: "".to_string(),
-        },
-        output_serialization: OutputSerialization::JSON {
-            record_delimiter: "\n".to_string(),
-        },
-        request_progress: true,
-    };
-
-    let mut result_stream = s3select.select_object_content(select_request).await?;
-
-    let mut processed_count = 0;
-    let mut output_file = tokio::fs::File::create("filtered_results.jsonl").await?;
-
-    while let Some(event) = result_stream.next().await {
-        match event? {
-            SelectEvent::Records(data) => {
-                // Write results to file
-                output_file.write_all(&data).await?;
-                processed_count += 1;
-
-                if processed_count % 1000 == 0 {
-                    println!("Processed {} records", processed_count);
-                }
-            }
-            SelectEvent::Progress(progress) => {
-                println!("Progress: {:.1}% ({} bytes processed)",
-                    progress.details.bytes_processed_percent,
-                    progress.details.bytes_processed
-                );
-            }
-            SelectEvent::Stats(stats) => {
-                println!("Final statistics:");
-                println!("  Total bytes scanned: {}", stats.bytes_scanned);
-                println!("  Total bytes processed: {}", stats.bytes_processed);
-                println!("  Total bytes returned: {}", stats.bytes_returned);
-                println!("  Processing efficiency: {:.2}%",
-                    (stats.bytes_returned as f64 / stats.bytes_scanned as f64) * 100.0
-                );
-            }
-            SelectEvent::End => {
-                println!("Streaming completed. Total records: {}", processed_count);
-                break;
-            }
-        }
-    }
-
-    output_file.flush().await?;
-    Ok(())
-}
-```
-
-### HTTP API Integration
-
-```rust
-use rustfs_s3select_api::{S3SelectHandler, SelectRequestXML};
-use axum::{Router, Json, extract::{Path, Query}};
-
-async fn setup_s3select_http_api() -> Router {
-    let s3select_handler = S3SelectHandler::new().await.unwrap();
-
-    Router::new()
-        .route("/buckets/:bucket/objects/:key/select",
-               axum::routing::post(handle_select_object_content))
-        .layer(Extension(s3select_handler))
-}
-
-async fn handle_select_object_content(
-    Path((bucket, key)): Path<(String, String)>,
-    Extension(handler): Extension<S3SelectHandler>,
-    body: String,
-) -> Result<impl axum::response::IntoResponse, Box<dyn std::error::Error>> {
-    // Parse S3 Select request (XML or JSON)
-    let select_request = handler.parse_request(&body, &bucket, &key).await?;
-
-    // Execute query
-    let result_stream = handler.execute_select(select_request).await?;
-
-    // Return streaming response
-    let response = axum::response::Response::builder()
-        .header("content-type", "application/xml")
-        .header("x-amz-request-id", "12345")
-        .body(axum::body::Body::from_stream(result_stream))?;
-
-    Ok(response)
-}
-```
-
-## 🏗️ Architecture
-
-### S3Select API Architecture
-
-```
-S3Select API Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    S3 Select HTTP API                       │
-├─────────────────────────────────────────────────────────────┤
-│   Request      │   Response     │   Streaming  │   Error    │
-│   Parsing      │   Formatting   │   Results    │   Handling │
-├─────────────────────────────────────────────────────────────┤
-│              Query Engine (DataFusion)                      │
-├─────────────────────────────────────────────────────────────┤
-│   SQL Parser   │   Optimizer    │   Execution  │   Streaming│
-├─────────────────────────────────────────────────────────────┤
-│              Storage Integration                            │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Supported Data Formats
-
-| Format | Features | Use Cases |
-|--------|----------|-----------|
-| CSV | Custom delimiters, headers, quotes | Log files, exports |
-| JSON | Objects, arrays, nested data | APIs, documents |
-| JSON Lines | Streaming JSON records | Event logs, analytics |
-| Parquet | Columnar, schema evolution | Data warehousing |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test SQL parsing
-cargo test sql_parsing
-
-# Test format support
-cargo test format_support
-
-# Test streaming
-cargo test streaming
-
-# Integration tests
-cargo test --test integration
-
-# Performance tests
-cargo test --test performance --release
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Apache DataFusion, Arrow
-- **Memory**: Sufficient RAM for query processing
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS S3Select Query](../s3select-query) - Query engine implementation
-- [RustFS ECStore](../ecstore) - Storage backend
+- Standard SQL query support (SELECT, WHERE, GROUP BY, ORDER BY)
+- Multiple data format support (CSV, JSON, Parquet, Arrow)
+- Streaming processing for large files
+- AWS S3 Select API compatibility
+- Parallel query execution
+- Predicate pushdown optimization
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [S3Select API Reference](https://docs.rustfs.com/s3select-api/)
-- [SQL Reference](https://docs.rustfs.com/sql/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 📊 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/s3select-query/Cargo.toml

@@ -19,6 +19,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "S3 Select query engine for RustFS, enabling efficient data retrieval from S3-compatible storage using SQL-like queries."
+keywords = ["s3-select", "query-engine", "rustfs", "Minio", "data-retrieval"]
+categories = ["web-programming", "development-tools", "data-structures"]
+documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_query/"
 
 [dependencies]
 rustfs-s3select-api = { workspace = true }

crates/s3select-query/README.md

@@ -1,6 +1,6 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS S3Select Query - High-Performance Query Engine
+# RustFS S3Select Query - SQL Query Engine
 
 <p align="center">
   <strong>Apache DataFusion-powered SQL query engine for RustFS S3 Select implementation</strong>
@@ -17,641 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS S3Select Query** is the high-performance query engine that powers SQL processing for the [RustFS](https://rustfs.com) S3 Select API. Built on Apache DataFusion, it provides blazing-fast SQL execution with advanced optimization techniques, streaming processing, and support for multiple data formats.
-
-> **Note:** This is a core performance-critical submodule of RustFS that provides the SQL query execution engine for the S3 Select API. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS S3Select Query** provides Apache DataFusion-powered SQL query engine capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🚀 High-Performance Query Engine
-
-- **Apache DataFusion**: Built on the fastest SQL engine in Rust
-- **Vectorized Processing**: SIMD-accelerated columnar processing
-- **Parallel Execution**: Multi-threaded query execution
-- **Memory Efficient**: Streaming processing with minimal memory footprint
-
-### 📊 Advanced SQL Support
-
-- **Standard SQL**: Full support for SQL:2016 standard
-- **Complex Queries**: Joins, subqueries, window functions, CTEs
-- **Aggregations**: Group by, having, order by with optimizations
-- **Built-in Functions**: 200+ SQL functions including UDFs
-
-### 🔧 Query Optimization
-
-- **Cost-Based Optimizer**: Intelligent query planning
-- **Predicate Pushdown**: Push filters to data sources
-- **Projection Pushdown**: Only read required columns
-- **Join Optimization**: Hash joins, sort-merge joins
-
-### 📁 Data Format Support
-
-- **Parquet**: Native columnar format with predicate pushdown
-- **CSV**: Efficient CSV parsing with schema inference
-- **JSON**: Nested JSON processing with path expressions
-- **Arrow**: Zero-copy Arrow format processing
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-s3select-query = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Query Engine Setup
-
-```rust
-use rustfs_s3select_query::{QueryEngine, DataSource, QueryResult};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create query engine
-    let query_engine = QueryEngine::new().await?;
-
-    // Register data source
-    let data_source = DataSource::from_csv("s3://bucket/data.csv").await?;
-    query_engine.register_table("sales", data_source).await?;
-
-    // Execute SQL query
-    let sql = "SELECT region, SUM(amount) as total FROM sales GROUP BY region";
-    let result = query_engine.execute_query(sql).await?;
-
-    // Process results
-    while let Some(batch) = result.next().await {
-        let batch = batch?;
-        println!("Batch with {} rows", batch.num_rows());
-
-        // Convert to JSON for display
-        let json_rows = batch.to_json()?;
-        for row in json_rows {
-            println!("{}", row);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Advanced Query Execution
-
-```rust
-use rustfs_s3select_query::{
-    QueryEngine, QueryPlan, ExecutionConfig,
-    DataSource, SchemaRef, RecordBatch
-};
-
-async fn advanced_query_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Configure execution settings
-    let config = ExecutionConfig::new()
-        .with_target_partitions(8)
-        .with_batch_size(8192)
-        .with_max_memory(1024 * 1024 * 1024); // 1GB memory limit
-
-    let query_engine = QueryEngine::with_config(config).await?;
-
-    // Register multiple data sources
-    let customers = DataSource::from_parquet("s3://warehouse/customers.parquet").await?;
-    let orders = DataSource::from_csv("s3://logs/orders.csv").await?;
-    let products = DataSource::from_json("s3://catalog/products.json").await?;
-
-    query_engine.register_table("customers", customers).await?;
-    query_engine.register_table("orders", orders).await?;
-    query_engine.register_table("products", products).await?;
-
-    // Complex analytical query
-    let sql = r#"
-        SELECT
-            c.customer_segment,
-            p.category,
-            COUNT(*) as order_count,
-            SUM(o.amount) as total_revenue,
-            AVG(o.amount) as avg_order_value,
-            STDDEV(o.amount) as revenue_stddev
-        FROM customers c
-        JOIN orders o ON c.customer_id = o.customer_id
-        JOIN products p ON o.product_id = p.product_id
-        WHERE o.order_date >= '2024-01-01'
-        AND o.status = 'completed'
-        GROUP BY c.customer_segment, p.category
-        HAVING SUM(o.amount) > 10000
-        ORDER BY total_revenue DESC
-        LIMIT 50
-    "#;
-
-    // Get query plan for optimization analysis
-    let plan = query_engine.create_logical_plan(sql).await?;
-    println!("Query plan:\n{}", plan.display_indent());
-
-    // Execute with streaming results
-    let mut result_stream = query_engine.execute_stream(sql).await?;
-
-    let mut total_rows = 0;
-    while let Some(batch) = result_stream.next().await {
-        let batch = batch?;
-        total_rows += batch.num_rows();
-
-        // Process batch
-        for row_idx in 0..batch.num_rows() {
-            let segment = batch.column_by_name("customer_segment")?
-                .as_any().downcast_ref::<StringArray>()
-                .unwrap().value(row_idx);
-            let category = batch.column_by_name("category")?
-                .as_any().downcast_ref::<StringArray>()
-                .unwrap().value(row_idx);
-            let revenue = batch.column_by_name("total_revenue")?
-                .as_any().downcast_ref::<Float64Array>()
-                .unwrap().value(row_idx);
-
-            println!("Segment: {}, Category: {}, Revenue: ${:.2}",
-                segment, category, revenue);
-        }
-    }
-
-    println!("Total rows processed: {}", total_rows);
-    Ok(())
-}
-```
-
-### Custom Data Sources
-
-```rust
-use rustfs_s3select_query::{DataSource, TableProvider, SchemaRef};
-use datafusion::arrow::datatypes::{Schema, Field, DataType};
-use datafusion::arrow::record_batch::RecordBatch;
-
-struct CustomS3DataSource {
-    bucket: String,
-    key: String,
-    schema: SchemaRef,
-}
-
-impl CustomS3DataSource {
-    async fn new(bucket: &str, key: &str) -> Result<Self, Box<dyn std::error::Error>> {
-        // Infer schema from S3 object
-        let schema = Self::infer_schema(bucket, key).await?;
-
-        Ok(Self {
-            bucket: bucket.to_string(),
-            key: key.to_string(),
-            schema: Arc::new(schema),
-        })
-    }
-
-    async fn infer_schema(bucket: &str, key: &str) -> Result<Schema, Box<dyn std::error::Error>> {
-        // Read sample data to infer schema
-        let sample_data = read_s3_sample(bucket, key).await?;
-
-        // Create schema based on data format
-        let schema = Schema::new(vec![
-            Field::new("id", DataType::Int64, false),
-            Field::new("name", DataType::Utf8, false),
-            Field::new("value", DataType::Float64, true),
-            Field::new("timestamp", DataType::Timestamp(TimeUnit::Millisecond, None), false),
-        ]);
-
-        Ok(schema)
-    }
-}
-
-#[async_trait::async_trait]
-impl TableProvider for CustomS3DataSource {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    async fn scan(
-        &self,
-        projection: Option<&Vec<usize>>,
-        filters: &[Expr],
-        limit: Option<usize>,
-    ) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> {
-        // Create execution plan for scanning S3 data
-        let scan_plan = S3ScanExec::new(
-            self.bucket.clone(),
-            self.key.clone(),
-            self.schema.clone(),
-            projection.cloned(),
-            filters.to_vec(),
-            limit,
-        );
-
-        Ok(Arc::new(scan_plan))
-    }
-}
-
-async fn custom_data_source_example() -> Result<(), Box<dyn std::error::Error>> {
-    let query_engine = QueryEngine::new().await?;
-
-    // Register custom data source
-    let custom_source = CustomS3DataSource::new("analytics", "events.parquet").await?;
-    query_engine.register_table("events", Arc::new(custom_source)).await?;
-
-    // Query custom data source
-    let sql = "SELECT * FROM events WHERE timestamp > NOW() - INTERVAL '1 day'";
-    let result = query_engine.execute_query(sql).await?;
-
-    // Process results
-    while let Some(batch) = result.next().await {
-        let batch = batch?;
-        println!("Custom source batch: {} rows", batch.num_rows());
-    }
-
-    Ok(())
-}
-```
-
-### Query Optimization and Analysis
-
-```rust
-use rustfs_s3select_query::{QueryEngine, QueryOptimizer, QueryMetrics};
-
-async fn query_optimization_example() -> Result<(), Box<dyn std::error::Error>> {
-    let query_engine = QueryEngine::new().await?;
-
-    // Register data source
-    let data_source = DataSource::from_parquet("s3://warehouse/sales.parquet").await?;
-    query_engine.register_table("sales", data_source).await?;
-
-    let sql = r#"
-        SELECT
-            region,
-            product_category,
-            SUM(amount) as total_sales,
-            COUNT(*) as transaction_count
-        FROM sales
-        WHERE sale_date >= '2024-01-01'
-        AND amount > 100
-        GROUP BY region, product_category
-        ORDER BY total_sales DESC
-    "#;
-
-    // Analyze query plan
-    let logical_plan = query_engine.create_logical_plan(sql).await?;
-    println!("Logical Plan:\n{}", logical_plan.display_indent());
-
-    let physical_plan = query_engine.create_physical_plan(&logical_plan).await?;
-    println!("Physical Plan:\n{}", physical_plan.display_indent());
-
-    // Execute with metrics
-    let start_time = std::time::Instant::now();
-    let mut result_stream = query_engine.execute_stream(sql).await?;
-
-    let mut total_rows = 0;
-    let mut total_batches = 0;
-
-    while let Some(batch) = result_stream.next().await {
-        let batch = batch?;
-        total_rows += batch.num_rows();
-        total_batches += 1;
-    }
-
-    let execution_time = start_time.elapsed();
-
-    // Get execution metrics
-    let metrics = query_engine.get_execution_metrics().await?;
-
-    println!("Query Performance:");
-    println!("  Execution time: {:?}", execution_time);
-    println!("  Total rows: {}", total_rows);
-    println!("  Total batches: {}", total_batches);
-    println!("  Rows per second: {:.2}", total_rows as f64 / execution_time.as_secs_f64());
-    println!("  Memory used: {} bytes", metrics.memory_used);
-    println!("  Bytes scanned: {}", metrics.bytes_scanned);
-
-    Ok(())
-}
-```
-
-### Streaming Query Processing
-
-```rust
-use rustfs_s3select_query::{StreamingQueryEngine, StreamingResult};
-use futures::StreamExt;
-
-async fn streaming_processing_example() -> Result<(), Box<dyn std::error::Error>> {
-    let streaming_engine = StreamingQueryEngine::new().await?;
-
-    // Register streaming data source
-    let stream_source = DataSource::from_streaming_csv("s3://logs/stream.csv").await?;
-    streaming_engine.register_table("log_stream", stream_source).await?;
-
-    // Continuous query with windowing
-    let sql = r#"
-        SELECT
-            TUMBLE_START(timestamp, INTERVAL '5' MINUTE) as window_start,
-            COUNT(*) as event_count,
-            AVG(response_time) as avg_response_time,
-            MAX(response_time) as max_response_time
-        FROM log_stream
-        WHERE status_code >= 400
-        GROUP BY TUMBLE(timestamp, INTERVAL '5' MINUTE)
-    "#;
-
-    let mut result_stream = streaming_engine.execute_streaming_query(sql).await?;
-
-    // Process streaming results
-    while let Some(window_result) = result_stream.next().await {
-        let batch = window_result?;
-
-        for row_idx in 0..batch.num_rows() {
-            let window_start = batch.column_by_name("window_start")?
-                .as_any().downcast_ref::<TimestampArray>()
-                .unwrap().value(row_idx);
-            let event_count = batch.column_by_name("event_count")?
-                .as_any().downcast_ref::<Int64Array>()
-                .unwrap().value(row_idx);
-            let avg_response = batch.column_by_name("avg_response_time")?
-                .as_any().downcast_ref::<Float64Array>()
-                .unwrap().value(row_idx);
-
-            println!("Window {}: {} errors, avg response time: {:.2}ms",
-                window_start, event_count, avg_response);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### User-Defined Functions (UDFs)
-
-```rust
-use rustfs_s3select_query::{QueryEngine, ScalarUDF, Volatility};
-use datafusion::arrow::datatypes::{DataType, Field};
-
-async fn custom_functions_example() -> Result<(), Box<dyn std::error::Error>> {
-    let query_engine = QueryEngine::new().await?;
-
-    // Register custom scalar function
-    let extract_domain_udf = ScalarUDF::new(
-        "extract_domain",
-        vec![DataType::Utf8],
-        DataType::Utf8,
-        Volatility::Immutable,
-        Arc::new(|args: &[ArrayRef]| {
-            let emails = args[0].as_any().downcast_ref::<StringArray>().unwrap();
-            let mut domains = Vec::new();
-
-            for i in 0..emails.len() {
-                if let Some(email) = emails.value_opt(i) {
-                    if let Some(domain) = email.split('@').nth(1) {
-                        domains.push(Some(domain.to_string()));
-                    } else {
-                        domains.push(None);
-                    }
-                } else {
-                    domains.push(None);
-                }
-            }
-
-            Ok(Arc::new(StringArray::from(domains)))
-        }),
-    );
-
-    query_engine.register_udf(extract_domain_udf).await?;
-
-    // Register aggregate function
-    let percentile_udf = AggregateUDF::new(
-        "percentile_90",
-        vec![DataType::Float64],
-        DataType::Float64,
-        Volatility::Immutable,
-        Arc::new(|| Box::new(PercentileAccumulator::new(0.9))),
-    );
-
-    query_engine.register_udaf(percentile_udf).await?;
-
-    // Use custom functions in query
-    let data_source = DataSource::from_csv("s3://users/profiles.csv").await?;
-    query_engine.register_table("users", data_source).await?;
-
-    let sql = r#"
-        SELECT
-            extract_domain(email) as domain,
-            COUNT(*) as user_count,
-            percentile_90(score) as p90_score
-        FROM users
-        GROUP BY extract_domain(email)
-        ORDER BY user_count DESC
-    "#;
-
-    let result = query_engine.execute_query(sql).await?;
-
-    while let Some(batch) = result.next().await {
-        let batch = batch?;
-
-        for row_idx in 0..batch.num_rows() {
-            let domain = batch.column_by_name("domain")?
-                .as_any().downcast_ref::<StringArray>()
-                .unwrap().value(row_idx);
-            let user_count = batch.column_by_name("user_count")?
-                .as_any().downcast_ref::<Int64Array>()
-                .unwrap().value(row_idx);
-            let p90_score = batch.column_by_name("p90_score")?
-                .as_any().downcast_ref::<Float64Array>()
-                .unwrap().value(row_idx);
-
-            println!("Domain: {}, Users: {}, P90 Score: {:.2}",
-                domain, user_count, p90_score);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Query Caching and Materialization
-
-```rust
-use rustfs_s3select_query::{QueryEngine, QueryCache, MaterializedView};
-
-async fn query_caching_example() -> Result<(), Box<dyn std::error::Error>> {
-    let mut query_engine = QueryEngine::new().await?;
-
-    // Enable query result caching
-    let cache_config = QueryCache::new()
-        .with_max_size(1024 * 1024 * 1024) // 1GB cache
-        .with_ttl(Duration::from_secs(300)); // 5 minutes TTL
-
-    query_engine.enable_caching(cache_config).await?;
-
-    // Register data source
-    let data_source = DataSource::from_parquet("s3://warehouse/transactions.parquet").await?;
-    query_engine.register_table("transactions", data_source).await?;
-
-    // Create materialized view for common queries
-    let materialized_view = MaterializedView::new(
-        "daily_sales",
-        r#"
-            SELECT
-                DATE(transaction_date) as date,
-                SUM(amount) as total_sales,
-                COUNT(*) as transaction_count
-            FROM transactions
-            GROUP BY DATE(transaction_date)
-        "#.to_string(),
-        Duration::from_secs(3600), // Refresh every hour
-    );
-
-    query_engine.register_materialized_view(materialized_view).await?;
-
-    // Query using materialized view
-    let sql = r#"
-        SELECT
-            date,
-            total_sales,
-            LAG(total_sales, 1) OVER (ORDER BY date) as prev_day_sales,
-            (total_sales - LAG(total_sales, 1) OVER (ORDER BY date)) /
-             LAG(total_sales, 1) OVER (ORDER BY date) * 100 as growth_rate
-        FROM daily_sales
-        WHERE date >= CURRENT_DATE - INTERVAL '30' DAY
-        ORDER BY date DESC
-    "#;
-
-    // First execution - cache miss
-    let start_time = std::time::Instant::now();
-    let result1 = query_engine.execute_query(sql).await?;
-    let mut rows1 = 0;
-    while let Some(batch) = result1.next().await {
-        rows1 += batch?.num_rows();
-    }
-    let first_execution_time = start_time.elapsed();
-
-    // Second execution - cache hit
-    let start_time = std::time::Instant::now();
-    let result2 = query_engine.execute_query(sql).await?;
-    let mut rows2 = 0;
-    while let Some(batch) = result2.next().await {
-        rows2 += batch?.num_rows();
-    }
-    let second_execution_time = start_time.elapsed();
-
-    println!("First execution: {:?} ({} rows)", first_execution_time, rows1);
-    println!("Second execution: {:?} ({} rows)", second_execution_time, rows2);
-    println!("Cache speedup: {:.2}x",
-        first_execution_time.as_secs_f64() / second_execution_time.as_secs_f64());
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Query Engine Architecture
-
-```
-Query Engine Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    SQL Query Interface                      │
-├─────────────────────────────────────────────────────────────┤
-│   Parser       │   Planner     │   Optimizer  │   Executor  │
-├─────────────────────────────────────────────────────────────┤
-│              Apache DataFusion Core                         │
-├─────────────────────────────────────────────────────────────┤
-│   Vectorized   │   Parallel    │   Streaming  │   Memory    │
-│   Processing   │   Execution   │   Engine     │   Management│
-├─────────────────────────────────────────────────────────────┤
-│              Data Source Integration                        │
-├─────────────────────────────────────────────────────────────┤
-│   Parquet      │   CSV         │   JSON       │   Arrow     │
-│   Reader       │   Parser      │   Parser     │   Format    │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Execution Flow
-
-1. **SQL Parsing**: Convert SQL string to logical plan
-2. **Logical Optimization**: Apply rule-based optimizations
-3. **Physical Planning**: Create physical execution plan
-4. **Execution**: Execute plan with streaming results
-5. **Result Streaming**: Return results as Arrow batches
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test query execution
-cargo test query_execution
-
-# Test optimization
-cargo test optimization
-
-# Test data formats
-cargo test data_formats
-
-# Benchmark tests
-cargo test --test benchmarks --release
-
-# Integration tests
-cargo test --test integration
-```
-
-## 📊 Performance Benchmarks
-
-| Operation | Throughput | Latency | Memory |
-|-----------|------------|---------|---------|
-| CSV Scan | 2.5 GB/s | 10ms | 50MB |
-| Parquet Scan | 5.0 GB/s | 5ms | 30MB |
-| JSON Parse | 1.2 GB/s | 15ms | 80MB |
-| Aggregation | 1.8 GB/s | 20ms | 100MB |
-| Join | 800 MB/s | 50ms | 200MB |
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **CPU**: Multi-core recommended for parallel processing
-- **Memory**: Variable based on query complexity
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS S3Select API](../s3select-api) - S3 Select API implementation
-- [RustFS ECStore](../ecstore) - Storage backend
+- Apache DataFusion integration for high-performance queries
+- Vectorized processing with SIMD acceleration
+- Parallel query execution across multiple threads
+- Cost-based query optimization
+- Support for complex SQL operations (joins, subqueries, window functions)
+- Multiple data format support (Parquet, CSV, JSON, Arrow)
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [S3Select Query Reference](https://docs.rustfs.com/s3select-query/)
-- [DataFusion Integration Guide](https://docs.rustfs.com/datafusion/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with ⚡ by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/signer/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Digital signature generation and verification for RustFS, ensuring data integrity and authenticity."
+keywords = ["digital-signature", "verification", "integrity", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "cryptography"]
+documentation = "https://docs.rs/rustfs-signer/latest/rustfs_signer/"
 
 [dependencies]
 tracing.workspace = true

crates/signer/README.md

@@ -17,390 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Signer** provides AWS-compatible request signing and authentication for the [RustFS](https://rustfs.com) distributed object storage system. It implements AWS Signature Version 4 (SigV4) signing algorithm, pre-signed URLs, and various authentication methods to ensure secure API access.
-
-> **Note:** This is a security-critical submodule of RustFS that provides essential authentication capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Signer** provides AWS-compatible request signing and authentication capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔐 AWS-Compatible Signing
-
-- **SigV4 Implementation**: Full AWS Signature Version 4 support
-- **Pre-signed URLs**: Temporary access URLs with expiration
-- **Chunked Upload**: Streaming upload with signature validation
-- **Multi-part Upload**: Signature validation for large files
-
-### 🛡️ Authentication Methods
-
-- **Access Key/Secret**: Traditional AWS-style authentication
-- **STS Token**: Temporary security token support
-- **IAM Role**: Role-based authentication
-- **Anonymous Access**: Public read access support
-
-### 🚀 Performance Features
-
-- **Signature Caching**: Avoid repeated signature calculations
-- **Batch Signing**: Sign multiple requests efficiently
-- **Streaming Support**: Sign data streams without buffering
-- **Hardware Acceleration**: Use hardware crypto when available
-
-### 🔧 Advanced Features
-
-- **Custom Headers**: Support for custom and vendor headers
-- **Regional Signing**: Multi-region signature support
-- **Clock Skew Handling**: Automatic time synchronization
-- **Signature Validation**: Server-side signature verification
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-signer = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Request Signing
-
-```rust
-use rustfs_signer::{Signer, SigningConfig, Credentials};
-use http::{Request, Method};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create credentials
-    let credentials = Credentials::new(
-        "AKIAIOSFODNN7EXAMPLE".to_string(),
-        "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
-        None, // No session token
-    );
-
-    // Create signing configuration
-    let config = SigningConfig {
-        region: "us-east-1".to_string(),
-        service: "s3".to_string(),
-        credentials,
-        ..Default::default()
-    };
-
-    // Create signer
-    let signer = Signer::new(config);
-
-    // Create request
-    let request = Request::builder()
-        .method(Method::GET)
-        .uri("https://example-bucket.s3.amazonaws.com/example-object")
-        .body(Vec::new())?;
-
-    // Sign request
-    let signed_request = signer.sign_request(request).await?;
-
-    println!("Authorization header: {:?}", signed_request.headers().get("authorization"));
-
-    Ok(())
-}
-```
-
-### Pre-signed URLs
-
-```rust
-use rustfs_signer::{Signer, PresignedUrlRequest};
-use std::time::Duration;
-
-async fn presigned_url_example() -> Result<(), Box<dyn std::error::Error>> {
-    let signer = Signer::new(signing_config);
-
-    // Create pre-signed URL for GET request
-    let presigned_request = PresignedUrlRequest {
-        method: Method::GET,
-        uri: "https://example-bucket.s3.amazonaws.com/example-object".parse()?,
-        headers: Default::default(),
-        expires_in: Duration::from_secs(3600), // 1 hour
-    };
-
-    let presigned_url = signer.presign_url(presigned_request).await?;
-    println!("Pre-signed URL: {}", presigned_url);
-
-    // Create pre-signed URL for PUT request
-    let put_request = PresignedUrlRequest {
-        method: Method::PUT,
-        uri: "https://example-bucket.s3.amazonaws.com/upload-object".parse()?,
-        headers: {
-            let mut headers = HeaderMap::new();
-            headers.insert("content-type", "text/plain".parse()?);
-            headers
-        },
-        expires_in: Duration::from_secs(1800), // 30 minutes
-    };
-
-    let upload_url = signer.presign_url(put_request).await?;
-    println!("Pre-signed upload URL: {}", upload_url);
-
-    Ok(())
-}
-```
-
-### Streaming Upload Signing
-
-```rust
-use rustfs_signer::{StreamingSigner, ChunkedUploadSigner};
-use tokio::io::AsyncReadExt;
-
-async fn streaming_upload_example() -> Result<(), Box<dyn std::error::Error>> {
-    let signer = Signer::new(signing_config);
-
-    // Create streaming signer
-    let streaming_signer = StreamingSigner::new(signer, "s3".to_string());
-
-    // Create chunked upload signer
-    let mut chunked_signer = ChunkedUploadSigner::new(
-        streaming_signer,
-        "example-bucket".to_string(),
-        "large-file.dat".to_string(),
-    );
-
-    // Initialize multipart upload
-    let upload_id = chunked_signer.initiate_multipart_upload().await?;
-    println!("Upload ID: {}", upload_id);
-
-    // Upload chunks
-    let mut file = tokio::fs::File::open("large-file.dat").await?;
-    let mut chunk_buffer = vec![0u8; 5 * 1024 * 1024]; // 5MB chunks
-    let mut part_number = 1;
-    let mut etags = Vec::new();
-
-    loop {
-        let bytes_read = file.read(&mut chunk_buffer).await?;
-        if bytes_read == 0 {
-            break;
-        }
-
-        let chunk = &chunk_buffer[..bytes_read];
-        let etag = chunked_signer.upload_part(part_number, chunk).await?;
-        etags.push((part_number, etag));
-
-        part_number += 1;
-    }
-
-    // Complete multipart upload
-    chunked_signer.complete_multipart_upload(upload_id, etags).await?;
-    println!("Upload completed successfully");
-
-    Ok(())
-}
-```
-
-### Signature Validation
-
-```rust
-use rustfs_signer::{SignatureValidator, ValidationResult};
-use http::HeaderMap;
-
-async fn signature_validation_example() -> Result<(), Box<dyn std::error::Error>> {
-    let validator = SignatureValidator::new(signing_config);
-
-    // Extract signature from request headers
-    let headers = HeaderMap::new(); // Headers from incoming request
-    let method = "GET";
-    let uri = "/example-bucket/example-object";
-    let body = b""; // Request body
-
-    // Validate signature
-    let validation_result = validator.validate_signature(
-        method,
-        uri,
-        &headers,
-        body,
-    ).await?;
-
-    match validation_result {
-        ValidationResult::Valid { credentials, .. } => {
-            println!("Signature valid for user: {}", credentials.access_key);
-        }
-        ValidationResult::Invalid { reason } => {
-            println!("Signature invalid: {}", reason);
-        }
-        ValidationResult::Expired { expired_at } => {
-            println!("Signature expired at: {}", expired_at);
-        }
-    }
-
-    Ok(())
-}
-```
-
-### Batch Signing
-
-```rust
-use rustfs_signer::{BatchSigner, BatchSigningRequest};
-
-async fn batch_signing_example() -> Result<(), Box<dyn std::error::Error>> {
-    let signer = Signer::new(signing_config);
-    let batch_signer = BatchSigner::new(signer);
-
-    // Create multiple requests
-    let requests = vec![
-        BatchSigningRequest {
-            method: Method::GET,
-            uri: "https://bucket1.s3.amazonaws.com/object1".parse()?,
-            headers: HeaderMap::new(),
-            body: Vec::new(),
-        },
-        BatchSigningRequest {
-            method: Method::PUT,
-            uri: "https://bucket2.s3.amazonaws.com/object2".parse()?,
-            headers: HeaderMap::new(),
-            body: b"Hello, World!".to_vec(),
-        },
-    ];
-
-    // Sign all requests in batch
-    let signed_requests = batch_signer.sign_batch(requests).await?;
-
-    for (i, signed_request) in signed_requests.iter().enumerate() {
-        println!("Request {}: {:?}", i + 1, signed_request.headers().get("authorization"));
-    }
-
-    Ok(())
-}
-```
-
-### Custom Authentication
-
-```rust
-use rustfs_signer::{CustomAuthenticator, AuthenticationResult};
-use async_trait::async_trait;
-
-struct CustomAuth {
-    // Custom authentication logic
-}
-
-#[async_trait]
-impl CustomAuthenticator for CustomAuth {
-    async fn authenticate(&self, request: &Request<Vec<u8>>) -> Result<AuthenticationResult, Box<dyn std::error::Error>> {
-        // Custom authentication logic
-        let auth_header = request.headers().get("authorization");
-
-        if let Some(auth) = auth_header {
-            // Parse and validate custom authentication
-            let auth_str = auth.to_str()?;
-
-            if auth_str.starts_with("Custom ") {
-                // Validate custom token
-                let token = &auth_str[7..];
-                if self.validate_token(token).await? {
-                    return Ok(AuthenticationResult::Authenticated {
-                        user_id: "custom-user".to_string(),
-                        permissions: vec!["read".to_string(), "write".to_string()],
-                    });
-                }
-            }
-        }
-
-        Ok(AuthenticationResult::Unauthenticated)
-    }
-}
-
-impl CustomAuth {
-    async fn validate_token(&self, token: &str) -> Result<bool, Box<dyn std::error::Error>> {
-        // Implement token validation logic
-        Ok(token.len() > 10) // Simple example
-    }
-}
-```
-
-## 🏗️ Architecture
-
-### Signer Architecture
-
-```
-Signer Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Signing API Layer                        │
-├─────────────────────────────────────────────────────────────┤
-│   SigV4      │   Pre-signed  │   Streaming  │   Batch      │
-├─────────────────────────────────────────────────────────────┤
-│              Signature Calculation                          │
-├─────────────────────────────────────────────────────────────┤
-│   HMAC-SHA256  │   Canonicalization │   String to Sign     │
-├─────────────────────────────────────────────────────────────┤
-│              Cryptographic Primitives                       │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Signing Process
-
-| Step | Description | Purpose |
-|------|-------------|---------|
-| 1. Canonicalize | Format request components | Consistent representation |
-| 2. Create String to Sign | Combine canonicalized data | Prepare for signing |
-| 3. Calculate Signature | HMAC-SHA256 computation | Generate signature |
-| 4. Add Headers | Add signature to request | Complete authentication |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test signature generation
-cargo test signature
-
-# Test pre-signed URLs
-cargo test presigned
-
-# Test validation
-cargo test validation
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Cryptographic libraries (ring, rustls)
-- **Compatibility**: AWS S3 API compatible
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS IAM](../iam) - Identity and access management
-- [RustFS Crypto](../crypto) - Cryptographic operations
+- AWS Signature Version 4 (SigV4) implementation
+- Pre-signed URL generation and validation
+- Multiple authentication methods (access key, STS token, IAM role)
+- Streaming upload signature support
+- Hardware-accelerated cryptographic operations
+- Multi-region signature support
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Signer API Reference](https://docs.rustfs.com/signer/)
-- [AWS S3 Compatibility](https://docs.rustfs.com/s3-compatibility/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔐 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/utils/Cargo.toml

@@ -19,6 +19,10 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Utility functions and data structures for RustFS, providing essential features like hashing, compression, and network utilities."
+keywords = ["utilities", "hashing", "compression", "network", "rustfs"]
+categories = ["web-programming", "development-tools", "cryptography"]
 
 [dependencies]
 base64-simd = { workspace = true, optional = true }

crates/utils/README.md

@@ -1,6 +1,6 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS Utils - Utility Library
+# RustFS Utils - Utility Functions
 
 <p align="center">
   <strong>Essential utility functions and common tools for RustFS distributed object storage</strong>
@@ -17,378 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Utils** is the utility library for the [RustFS](https://rustfs.com) distributed object storage system. It provides a comprehensive collection of utility functions, helper tools, and common functionality used across all RustFS modules, including system operations, cryptographic utilities, compression, and cross-platform compatibility tools.
-
-> **Note:** This is a foundational submodule of RustFS that provides essential utility functions for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Utils** provides essential utility functions and common tools for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔧 System Utilities
-
-- **Cross-Platform Operations**: Unified system operations across platforms
-- **Process Management**: Process spawning and management utilities
-- **Resource Monitoring**: CPU, memory, and disk usage monitoring
-- **Network Utilities**: Network interface and connectivity tools
-
-### 📁 File System Utilities
-
-- **Path Manipulation**: Advanced path handling and normalization
-- **File Operations**: Safe file operations with atomic writes
-- **Directory Management**: Recursive directory operations
-- **Symbolic Link Handling**: Cross-platform symlink management
-
-### 🗜️ Compression & Encoding
-
-- **Multiple Algorithms**: Support for gzip, zstd, lz4, and more
-- **Streaming Compression**: Memory-efficient streaming compression
-- **Base64 Encoding**: High-performance base64 operations
-- **URL Encoding**: Safe URL encoding and decoding
-
-### 🔐 Cryptographic Utilities
-
-- **Hash Functions**: MD5, SHA1, SHA256, XXHash implementations
-- **Random Generation**: Cryptographically secure random utilities
-- **Certificate Handling**: X.509 certificate parsing and validation
-- **Key Generation**: Secure key generation utilities
-
-### 🌐 Network Utilities
-
-- **HTTP Helpers**: HTTP client and server utilities
-- **DNS Resolution**: DNS lookup and resolution tools
-- **Network Interface**: Interface detection and configuration
-- **Protocol Utilities**: Various network protocol helpers
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-utils = "0.1.0"
-
-# Or with specific features
-rustfs-utils = { version = "0.1.0", features = ["compression", "crypto", "network"] }
-```
-
-### Feature Flags
-
-```toml
-[dependencies]
-rustfs-utils = { version = "0.1.0", features = ["full"] }
-```
-
-Available features:
-
-- `compression` - Compression and decompression utilities
-- `crypto` - Cryptographic functions and utilities
-- `network` - Network-related utilities
-- `path` - Advanced path manipulation tools
-- `system` - System monitoring and management
-- `full` - All features enabled
-
-## 🔧 Usage
-
-### File System Utilities
-
-```rust
-use rustfs_utils::fs::{ensure_dir, atomic_write, safe_remove};
-use rustfs_utils::path::{normalize_path, is_subdirectory};
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Ensure directory exists
-    ensure_dir("/path/to/directory")?;
-
-    // Atomic file write
-    atomic_write("/path/to/file.txt", b"Hello, World!")?;
-
-    // Path normalization
-    let normalized = normalize_path("./some/../path/./file.txt");
-    println!("Normalized: {}", normalized.display());
-
-    // Check if path is subdirectory
-    if is_subdirectory("/safe/path", "/safe/path/subdir") {
-        println!("Path is safe");
-    }
-
-    Ok(())
-}
-```
-
-### Compression Utilities
-
-```rust
-use rustfs_utils::compress::{compress_data, decompress_data, Algorithm};
-
-fn compression_example() -> Result<(), Box<dyn std::error::Error>> {
-    let data = b"This is some test data to compress";
-
-    // Compress with different algorithms
-    let gzip_compressed = compress_data(data, Algorithm::Gzip)?;
-    let zstd_compressed = compress_data(data, Algorithm::Zstd)?;
-    let lz4_compressed = compress_data(data, Algorithm::Lz4)?;
-
-    // Decompress
-    let decompressed = decompress_data(&gzip_compressed, Algorithm::Gzip)?;
-    assert_eq!(data, decompressed.as_slice());
-
-    println!("Original size: {}", data.len());
-    println!("Gzip compressed: {}", gzip_compressed.len());
-    println!("Zstd compressed: {}", zstd_compressed.len());
-    println!("LZ4 compressed: {}", lz4_compressed.len());
-
-    Ok(())
-}
-```
-
-### Cryptographic Utilities
-
-```rust
-use rustfs_utils::crypto::{hash_data, random_bytes, generate_key};
-use rustfs_utils::crypto::HashAlgorithm;
-
-fn crypto_example() -> Result<(), Box<dyn std::error::Error>> {
-    let data = b"Important data to hash";
-
-    // Generate hashes
-    let md5_hash = hash_data(data, HashAlgorithm::MD5)?;
-    let sha256_hash = hash_data(data, HashAlgorithm::SHA256)?;
-    let xxhash = hash_data(data, HashAlgorithm::XXHash64)?;
-
-    println!("MD5: {}", hex::encode(md5_hash));
-    println!("SHA256: {}", hex::encode(sha256_hash));
-    println!("XXHash64: {}", hex::encode(xxhash));
-
-    // Generate secure random data
-    let random_data = random_bytes(32)?;
-    println!("Random data: {}", hex::encode(random_data));
-
-    // Generate cryptographic key
-    let key = generate_key(256)?; // 256-bit key
-    println!("Generated key: {}", hex::encode(key));
-
-    Ok(())
-}
-```
-
-### System Monitoring
-
-```rust
-use rustfs_utils::sys::{get_system_info, monitor_resources, DiskUsage};
-
-async fn system_monitoring_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Get system information
-    let sys_info = get_system_info().await?;
-    println!("OS: {} {}", sys_info.os_name, sys_info.os_version);
-    println!("CPU: {} cores", sys_info.cpu_cores);
-    println!("Total Memory: {} GB", sys_info.total_memory / 1024 / 1024 / 1024);
-
-    // Monitor disk usage
-    let disk_usage = DiskUsage::for_path("/var/data")?;
-    println!("Disk space: {} / {} bytes", disk_usage.used, disk_usage.total);
-    println!("Available: {} bytes", disk_usage.available);
-
-    // Monitor resources
-    let resources = monitor_resources().await?;
-    println!("CPU Usage: {:.2}%", resources.cpu_percent);
-    println!("Memory Usage: {:.2}%", resources.memory_percent);
-
-    Ok(())
-}
-```
-
-### Network Utilities
-
-```rust
-use rustfs_utils::net::{resolve_hostname, get_local_ip, is_port_available};
-
-async fn network_example() -> Result<(), Box<dyn std::error::Error>> {
-    // DNS resolution
-    let addresses = resolve_hostname("example.com").await?;
-    for addr in addresses {
-        println!("Resolved address: {}", addr);
-    }
-
-    // Get local IP
-    let local_ip = get_local_ip().await?;
-    println!("Local IP: {}", local_ip);
-
-    // Check port availability
-    if is_port_available(8080).await? {
-        println!("Port 8080 is available");
-    } else {
-        println!("Port 8080 is in use");
-    }
-
-    Ok(())
-}
-```
-
-### Certificate Utilities
-
-```rust
-use rustfs_utils::certs::{parse_certificate, validate_certificate_chain, CertificateInfo};
-
-fn certificate_example() -> Result<(), Box<dyn std::error::Error>> {
-    let cert_pem = include_str!("../test_data/certificate.pem");
-
-    // Parse certificate
-    let cert_info = parse_certificate(cert_pem)?;
-    println!("Subject: {}", cert_info.subject);
-    println!("Issuer: {}", cert_info.issuer);
-    println!("Valid from: {}", cert_info.not_before);
-    println!("Valid until: {}", cert_info.not_after);
-
-    // Validate certificate chain
-    let ca_certs = vec![/* CA certificates */];
-    let is_valid = validate_certificate_chain(&cert_info, &ca_certs)?;
-
-    if is_valid {
-        println!("Certificate chain is valid");
-    } else {
-        println!("Certificate chain is invalid");
-    }
-
-    Ok(())
-}
-```
-
-### Encoding Utilities
-
-```rust
-use rustfs_utils::encoding::{base64_encode, base64_decode, url_encode, url_decode};
-
-fn encoding_example() -> Result<(), Box<dyn std::error::Error>> {
-    let data = b"Hello, World!";
-
-    // Base64 encoding
-    let encoded = base64_encode(data);
-    let decoded = base64_decode(&encoded)?;
-    assert_eq!(data, decoded.as_slice());
-
-    // URL encoding
-    let url = "https://example.com/path with spaces?param=value&other=data";
-    let encoded_url = url_encode(url);
-    let decoded_url = url_decode(&encoded_url)?;
-    assert_eq!(url, decoded_url);
-
-    println!("Base64 encoded: {}", encoded);
-    println!("URL encoded: {}", encoded_url);
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Utils Module Structure
-
-```
-Utils Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Public API Layer                         │
-├─────────────────────────────────────────────────────────────┤
-│  File System  │  Compression  │  Crypto     │  Network      │
-├─────────────────────────────────────────────────────────────┤
-│  System Info  │  Encoding     │  Certs      │  Path Utils   │
-├─────────────────────────────────────────────────────────────┤
-│              Platform Abstraction Layer                     │
-├─────────────────────────────────────────────────────────────┤
-│              Operating System Integration                    │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Feature Overview
-
-| Category | Features | Platform Support |
-|----------|----------|------------------|
-| File System | Atomic operations, path manipulation | All platforms |
-| Compression | Gzip, Zstd, LZ4, Brotli | All platforms |
-| Cryptography | Hashing, random generation, keys | All platforms |
-| System | Resource monitoring, process management | Linux, macOS, Windows |
-| Network | DNS, connectivity, interface detection | All platforms |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Run tests for specific features
-cargo test --features compression
-cargo test --features crypto
-cargo test --features network
-
-# Run tests with all features
-cargo test --features full
-
-# Run benchmarks
-cargo bench
-```
-
-## 📊 Performance
-
-The utils library is optimized for performance:
-
-- **Zero-Copy Operations**: Minimize memory allocations where possible
-- **Lazy Evaluation**: Defer expensive operations until needed
-- **Platform Optimization**: Use platform-specific optimizations
-- **Efficient Algorithms**: Choose the most efficient algorithms for each task
-
-### Benchmarks
-
-| Operation | Performance | Notes |
-|-----------|-------------|-------|
-| Path Normalization | ~50 ns | Uses efficient string operations |
-| Base64 Encoding | ~1.2 GB/s | SIMD-optimized implementation |
-| XXHash64 | ~15 GB/s | Hardware-accelerated when available |
-| File Copy | ~2 GB/s | Platform-optimized copy operations |
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Architecture**: x86_64, aarch64, and others
-- **Dependencies**: Minimal external dependencies
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS ECStore](../ecstore) - Erasure coding storage engine
-- [RustFS Crypto](../crypto) - Cryptographic operations
-- [RustFS Config](../config) - Configuration management
+- Cross-platform system operations and monitoring
+- File system utilities with atomic operations
+- Multi-algorithm compression and encoding support
+- Cryptographic utilities and secure key generation
+- Network utilities and protocol helpers
+- Certificate handling and validation tools
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Utils API Reference](https://docs.rustfs.com/utils/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔧 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/utils/src/net.rs

@@ -18,7 +18,7 @@ use futures::{Stream, StreamExt};
 use hyper::client::conn::http2::Builder;
 use hyper_util::rt::TokioExecutor;
 use lazy_static::lazy_static;
-use std::net::Ipv4Addr;
+use std::net::Ipv6Addr;
 use std::{
     collections::HashSet,
     fmt::Display,
@@ -202,7 +202,7 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr>
         } else {
             port
         };
-        SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), final_port)
+        SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), final_port)
     } else {
         let mut addr = check_local_server_addr(addr_str)?; // assume check_local_server_addr is available here
         if addr.port() == 0 {
@@ -478,12 +478,12 @@ mod test {
     fn test_parse_and_resolve_address() {
         // Test port-only format
         let result = parse_and_resolve_address(":8080").unwrap();
-        assert_eq!(result.ip(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
+        assert_eq!(result.ip(), IpAddr::V6(Ipv6Addr::UNSPECIFIED));
         assert_eq!(result.port(), 8080);
 
         // Test port-only format with port 0 (should get available port)
         let result = parse_and_resolve_address(":0").unwrap();
-        assert_eq!(result.ip(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
+        assert_eq!(result.ip(), IpAddr::V6(Ipv6Addr::UNSPECIFIED));
         assert!(result.port() > 0);
 
         // Test localhost with port

crates/workers/Cargo.toml

@@ -19,10 +19,15 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "Workers for RustFS, providing background processing capabilities for tasks such as data synchronization, indexing, and more."
+keywords = ["workers", "tasks", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools"]
+documentation = "https://docs.rs/rustfs-workers/latest/rustfs_workers/"
 
 [lints]
 workspace = true
 
 [dependencies]
-tokio.workspace = true
+tokio = { workspace = true, features = ["sync"] }
 tracing.workspace = true

crates/workers/README.md

@@ -1,6 +1,6 @@
 [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-# RustFS Workers - Background Job Processing
+# RustFS Workers - Background Job System
 
 <p align="center">
   <strong>Distributed background job processing system for RustFS object storage</strong>
@@ -17,446 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Workers** provides a distributed background job processing system for the [RustFS](https://rustfs.com) distributed object storage system. It handles asynchronous tasks such as data replication, cleanup, healing, indexing, and other maintenance operations across the cluster.
-
-> **Note:** This is a core submodule of RustFS that provides essential background processing capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Workers** provides distributed background job processing capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 🔄 Job Processing
-
-- **Distributed Execution**: Jobs run across multiple cluster nodes
-- **Priority Queues**: Multiple priority levels for job scheduling
-- **Retry Logic**: Automatic retry with exponential backoff
-- **Dead Letter Queue**: Failed job isolation and analysis
-
-### 🛠️ Built-in Workers
-
-- **Replication Worker**: Data replication across nodes
-- **Cleanup Worker**: Garbage collection and cleanup
-- **Healing Worker**: Data integrity repair
-- **Indexing Worker**: Metadata indexing and search
-- **Metrics Worker**: Performance metrics collection
-
-### 🚀 Scalability Features
-
-- **Horizontal Scaling**: Add worker nodes dynamically
-- **Load Balancing**: Intelligent job distribution
-- **Circuit Breaker**: Prevent cascading failures
-- **Rate Limiting**: Control resource consumption
-
-### 🔧 Management & Monitoring
-
-- **Job Tracking**: Real-time job status monitoring
-- **Health Checks**: Worker health and availability
-- **Metrics Collection**: Performance and throughput metrics
-- **Administrative Interface**: Job management and control
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-workers = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Worker Setup
-
-```rust
-use rustfs_workers::{WorkerManager, WorkerConfig, JobQueue};
-use std::time::Duration;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create worker configuration
-    let config = WorkerConfig {
-        worker_id: "worker-1".to_string(),
-        max_concurrent_jobs: 10,
-        job_timeout: Duration::from_secs(300),
-        retry_limit: 3,
-        cleanup_interval: Duration::from_secs(60),
-    };
-
-    // Create worker manager
-    let worker_manager = WorkerManager::new(config).await?;
-
-    // Start worker processing
-    worker_manager.start().await?;
-
-    // Keep running
-    tokio::signal::ctrl_c().await?;
-    worker_manager.shutdown().await?;
-
-    Ok(())
-}
-```
-
-### Job Definition and Scheduling
-
-```rust
-use rustfs_workers::{Job, JobBuilder, JobPriority, JobQueue};
-use serde::{Deserialize, Serialize};
-use async_trait::async_trait;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ReplicationJob {
-    pub source_path: String,
-    pub target_nodes: Vec<String>,
-    pub replication_factor: u32,
-}
-
-#[async_trait]
-impl Job for ReplicationJob {
-    async fn execute(&self) -> Result<(), Box<dyn std::error::Error>> {
-        println!("Starting replication job for: {}", self.source_path);
-
-        // Perform replication logic
-        for node in &self.target_nodes {
-            self.replicate_to_node(node).await?;
-        }
-
-        println!("Replication job completed successfully");
-        Ok(())
-    }
-
-    fn job_type(&self) -> &str {
-        "replication"
-    }
-
-    fn max_retries(&self) -> u32 {
-        3
-    }
-}
-
-impl ReplicationJob {
-    async fn replicate_to_node(&self, node: &str) -> Result<(), Box<dyn std::error::Error>> {
-        // Implementation for replicating data to a specific node
-        println!("Replicating {} to node: {}", self.source_path, node);
-        tokio::time::sleep(Duration::from_secs(1)).await; // Simulate work
-        Ok(())
-    }
-}
-
-async fn schedule_replication_job() -> Result<(), Box<dyn std::error::Error>> {
-    let job_queue = JobQueue::new().await?;
-
-    // Create replication job
-    let job = ReplicationJob {
-        source_path: "/bucket/important-file.txt".to_string(),
-        target_nodes: vec!["node-2".to_string(), "node-3".to_string()],
-        replication_factor: 2,
-    };
-
-    // Schedule job with high priority
-    let job_id = job_queue.schedule_job(
-        Box::new(job),
-        JobPriority::High,
-        None, // Execute immediately
-    ).await?;
-
-    println!("Scheduled replication job with ID: {}", job_id);
-    Ok(())
-}
-```
-
-### Custom Worker Implementation
-
-```rust
-use rustfs_workers::{Worker, WorkerContext, JobResult};
-use async_trait::async_trait;
-
-pub struct CleanupWorker {
-    storage_path: String,
-    max_file_age: Duration,
-}
-
-#[async_trait]
-impl Worker for CleanupWorker {
-    async fn process_job(&self, job: Box<dyn Job>, context: &WorkerContext) -> JobResult {
-        match job.job_type() {
-            "cleanup" => {
-                if let Some(cleanup_job) = job.as_any().downcast_ref::<CleanupJob>() {
-                    self.execute_cleanup(cleanup_job, context).await
-                } else {
-                    JobResult::Failed("Invalid job type for cleanup worker".to_string())
-                }
-            }
-            _ => JobResult::Skipped,
-        }
-    }
-
-    async fn health_check(&self) -> bool {
-        // Check if storage is accessible
-        tokio::fs::metadata(&self.storage_path).await.is_ok()
-    }
-
-    fn worker_type(&self) -> &str {
-        "cleanup"
-    }
-}
-
-impl CleanupWorker {
-    pub fn new(storage_path: String, max_file_age: Duration) -> Self {
-        Self {
-            storage_path,
-            max_file_age,
-        }
-    }
-
-    async fn execute_cleanup(&self, job: &CleanupJob, context: &WorkerContext) -> JobResult {
-        println!("Starting cleanup job for: {}", job.target_path);
-
-        match self.cleanup_old_files(&job.target_path).await {
-            Ok(cleaned_count) => {
-                context.update_metrics("files_cleaned", cleaned_count).await;
-                JobResult::Success
-            }
-            Err(e) => JobResult::Failed(e.to_string()),
-        }
-    }
-
-    async fn cleanup_old_files(&self, path: &str) -> Result<u64, Box<dyn std::error::Error>> {
-        let mut cleaned_count = 0;
-        // Implementation for cleaning up old files
-        // ... cleanup logic ...
-        Ok(cleaned_count)
-    }
-}
-```
-
-### Job Queue Management
-
-```rust
-use rustfs_workers::{JobQueue, JobFilter, JobStatus};
-
-async fn job_queue_management() -> Result<(), Box<dyn std::error::Error>> {
-    let job_queue = JobQueue::new().await?;
-
-    // List pending jobs
-    let pending_jobs = job_queue.list_jobs(JobFilter {
-        status: Some(JobStatus::Pending),
-        job_type: None,
-        priority: None,
-        limit: Some(100),
-    }).await?;
-
-    println!("Pending jobs: {}", pending_jobs.len());
-
-    // Cancel a job
-    let job_id = "job-123";
-    job_queue.cancel_job(job_id).await?;
-
-    // Retry failed jobs
-    let failed_jobs = job_queue.list_jobs(JobFilter {
-        status: Some(JobStatus::Failed),
-        job_type: None,
-        priority: None,
-        limit: Some(50),
-    }).await?;
-
-    for job in failed_jobs {
-        job_queue.retry_job(&job.id).await?;
-    }
-
-    // Get job statistics
-    let stats = job_queue.get_statistics().await?;
-    println!("Job statistics: {:?}", stats);
-
-    Ok(())
-}
-```
-
-### Distributed Worker Coordination
-
-```rust
-use rustfs_workers::{WorkerCluster, WorkerNode, ClusterConfig};
-
-async fn distributed_worker_setup() -> Result<(), Box<dyn std::error::Error>> {
-    let cluster_config = ClusterConfig {
-        node_id: "worker-node-1".to_string(),
-        cluster_endpoint: "https://cluster.rustfs.local".to_string(),
-        heartbeat_interval: Duration::from_secs(30),
-        leader_election_timeout: Duration::from_secs(60),
-    };
-
-    // Create worker cluster
-    let cluster = WorkerCluster::new(cluster_config).await?;
-
-    // Register worker types
-    cluster.register_worker_type("replication", Box::new(ReplicationWorkerFactory)).await?;
-    cluster.register_worker_type("cleanup", Box::new(CleanupWorkerFactory)).await?;
-    cluster.register_worker_type("healing", Box::new(HealingWorkerFactory)).await?;
-
-    // Start cluster participation
-    cluster.join().await?;
-
-    // Handle cluster events
-    let mut event_receiver = cluster.event_receiver();
-
-    tokio::spawn(async move {
-        while let Some(event) = event_receiver.recv().await {
-            match event {
-                ClusterEvent::NodeJoined(node) => {
-                    println!("Worker node joined: {}", node.id);
-                }
-                ClusterEvent::NodeLeft(node) => {
-                    println!("Worker node left: {}", node.id);
-                }
-                ClusterEvent::LeadershipChanged(new_leader) => {
-                    println!("New cluster leader: {}", new_leader);
-                }
-            }
-        }
-    });
-
-    Ok(())
-}
-```
-
-### Job Monitoring and Metrics
-
-```rust
-use rustfs_workers::{JobMonitor, WorkerMetrics, AlertConfig};
-
-async fn job_monitoring_setup() -> Result<(), Box<dyn std::error::Error>> {
-    let monitor = JobMonitor::new().await?;
-
-    // Set up alerting
-    let alert_config = AlertConfig {
-        failed_job_threshold: 10,
-        worker_down_threshold: Duration::from_secs(300),
-        queue_size_threshold: 1000,
-        notification_endpoint: "https://alerts.example.com/webhook".to_string(),
-    };
-
-    monitor.configure_alerts(alert_config).await?;
-
-    // Start monitoring
-    monitor.start_monitoring().await?;
-
-    // Get real-time metrics
-    let metrics = monitor.get_metrics().await?;
-    println!("Worker metrics: {:?}", metrics);
-
-    // Set up periodic reporting
-    tokio::spawn(async move {
-        let mut interval = tokio::time::interval(Duration::from_secs(60));
-
-        loop {
-            interval.tick().await;
-
-            if let Ok(metrics) = monitor.get_metrics().await {
-                println!("=== Worker Metrics ===");
-                println!("Active jobs: {}", metrics.active_jobs);
-                println!("Completed jobs: {}", metrics.completed_jobs);
-                println!("Failed jobs: {}", metrics.failed_jobs);
-                println!("Queue size: {}", metrics.queue_size);
-                println!("Worker count: {}", metrics.worker_count);
-            }
-        }
-    });
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Workers Architecture
-
-```
-Workers Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Job Management API                       │
-├─────────────────────────────────────────────────────────────┤
-│   Scheduling  │   Monitoring   │   Queue Mgmt │   Metrics   │
-├─────────────────────────────────────────────────────────────┤
-│              Worker Coordination                            │
-├─────────────────────────────────────────────────────────────┤
-│   Job Queue   │   Load Balancer │   Health Check │   Retry  │
-├─────────────────────────────────────────────────────────────┤
-│              Distributed Execution                          │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Worker Types
-
-| Worker Type | Purpose | Characteristics |
-|-------------|---------|----------------|
-| Replication | Data replication | I/O intensive, network bound |
-| Cleanup | Garbage collection | CPU intensive, periodic |
-| Healing | Data repair | I/O intensive, high priority |
-| Indexing | Metadata indexing | CPU intensive, background |
-| Metrics | Performance monitoring | Low resource, continuous |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test job processing
-cargo test job_processing
-
-# Test worker coordination
-cargo test worker_coordination
-
-# Test distributed scenarios
-cargo test distributed
-
-# Integration tests
-cargo test --test integration
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Network**: Cluster connectivity required
-- **Storage**: Persistent queue storage recommended
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Common](../common) - Common types and utilities
-- [RustFS Lock](../lock) - Distributed locking
+- Distributed job execution across cluster nodes
+- Priority-based job scheduling and queue management
+- Built-in workers for replication, cleanup, healing, and indexing
+- Automatic retry logic with exponential backoff
+- Horizontal scaling with load balancing
+- Real-time job monitoring and administrative interface
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Workers API Reference](https://docs.rustfs.com/workers/)
-- [Job Processing Guide](https://docs.rustfs.com/jobs/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 🔄 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/zip/Cargo.toml

@@ -19,6 +19,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
+homepage.workspace = true
+description = "ZIP file handling for RustFS, providing support for reading and writing ZIP archives."
+keywords = ["zip", "compression", "rustfs", "Minio"]
+categories = ["web-programming", "development-tools", "compression"]
+documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/"
 
 
 [dependencies]

crates/zip/README.md

@@ -17,391 +17,21 @@
 
 ## 📖 Overview
 
-**RustFS Zip** provides high-performance compression and archiving capabilities for the [RustFS](https://rustfs.com) distributed object storage system. It supports multiple compression algorithms, streaming compression, and efficient archiving operations optimized for storage systems.
-
-> **Note:** This is a performance-critical submodule of RustFS that provides essential compression capabilities for the distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
+**RustFS Zip** provides high-performance compression and archiving capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
 ## ✨ Features
 
-### 📦 Compression Algorithms
-
-- **Zstandard (Zstd)**: Fast compression with excellent ratios
-- **LZ4**: Ultra-fast compression for real-time applications
-- **Gzip**: Industry-standard compression for compatibility
-- **Brotli**: Web-optimized compression for text content
-
-### 🚀 Performance Features
-
-- **Streaming Compression**: Compress data on-the-fly without buffering
-- **Parallel Processing**: Multi-threaded compression for large files
-- **Adaptive Compression**: Automatic algorithm selection based on data
-- **Hardware Acceleration**: Leverage CPU-specific optimizations
-
-### 🔧 Archive Management
-
-- **ZIP Format**: Standard ZIP archive creation and extraction
-- **TAR Format**: UNIX-style tar archive support
-- **Custom Formats**: RustFS-optimized archive formats
-- **Metadata Preservation**: Maintain file attributes and timestamps
-
-### 📊 Compression Analytics
-
-- **Ratio Analysis**: Detailed compression statistics
-- **Performance Metrics**: Compression speed and efficiency
-- **Content-Type Detection**: Automatic compression algorithm selection
-- **Deduplication**: Identify and handle duplicate content
-
-## 📦 Installation
-
-Add this to your `Cargo.toml`:
-
-```toml
-[dependencies]
-rustfs-zip = "0.1.0"
-```
-
-## 🔧 Usage
-
-### Basic Compression
-
-```rust
-use rustfs_zip::{Compressor, CompressionLevel, CompressionAlgorithm};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create compressor
-    let compressor = Compressor::new(CompressionAlgorithm::Zstd, CompressionLevel::Default);
-
-    // Compress data
-    let input_data = b"Hello, World! This is some test data to compress.";
-    let compressed = compressor.compress(input_data).await?;
-
-    println!("Original size: {} bytes", input_data.len());
-    println!("Compressed size: {} bytes", compressed.len());
-    println!("Compression ratio: {:.2}%",
-        (1.0 - compressed.len() as f64 / input_data.len() as f64) * 100.0);
-
-    // Decompress data
-    let decompressed = compressor.decompress(&compressed).await?;
-    assert_eq!(input_data, decompressed.as_slice());
-
-    Ok(())
-}
-```
-
-### Streaming Compression
-
-```rust
-use rustfs_zip::{StreamingCompressor, StreamingDecompressor};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-async fn streaming_compression_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create streaming compressor
-    let mut compressor = StreamingCompressor::new(
-        CompressionAlgorithm::Zstd,
-        CompressionLevel::Fast,
-    )?;
-
-    // Compress streaming data
-    let input = tokio::fs::File::open("large_file.txt").await?;
-    let output = tokio::fs::File::create("compressed_file.zst").await?;
-
-    let mut reader = tokio::io::BufReader::new(input);
-    let mut writer = tokio::io::BufWriter::new(output);
-
-    // Stream compression
-    let mut buffer = vec![0u8; 8192];
-    loop {
-        let bytes_read = reader.read(&mut buffer).await?;
-        if bytes_read == 0 {
-            break;
-        }
-
-        let compressed_chunk = compressor.compress_chunk(&buffer[..bytes_read]).await?;
-        writer.write_all(&compressed_chunk).await?;
-    }
-
-    // Finalize compression
-    let final_chunk = compressor.finalize().await?;
-    writer.write_all(&final_chunk).await?;
-    writer.flush().await?;
-
-    Ok(())
-}
-```
-
-### Archive Creation
-
-```rust
-use rustfs_zip::{ZipArchive, ArchiveBuilder, CompressionMethod};
-
-async fn create_archive_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create archive builder
-    let mut builder = ArchiveBuilder::new("backup.zip".to_string());
-
-    // Add files to archive
-    builder.add_file("config.json", "config/app.json", CompressionMethod::Deflate).await?;
-    builder.add_file("data.txt", "data/sample.txt", CompressionMethod::Store).await?;
-
-    // Add directory
-    builder.add_directory("logs/", "application_logs/").await?;
-
-    // Create archive
-    let archive = builder.build().await?;
-
-    println!("Archive created: {}", archive.path());
-    println!("Total files: {}", archive.file_count());
-    println!("Compressed size: {} bytes", archive.compressed_size());
-    println!("Uncompressed size: {} bytes", archive.uncompressed_size());
-
-    Ok(())
-}
-```
-
-### Archive Extraction
-
-```rust
-use rustfs_zip::{ZipExtractor, ExtractionOptions};
-
-async fn extract_archive_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create extractor
-    let extractor = ZipExtractor::new("backup.zip".to_string());
-
-    // List archive contents
-    let entries = extractor.list_entries().await?;
-    for entry in &entries {
-        println!("File: {} ({} bytes)", entry.name, entry.size);
-    }
-
-    // Extract specific file
-    let file_data = extractor.extract_file("config.json").await?;
-    println!("Extracted config.json: {} bytes", file_data.len());
-
-    // Extract all files
-    let extraction_options = ExtractionOptions {
-        output_directory: "extracted/".to_string(),
-        preserve_paths: true,
-        overwrite_existing: false,
-    };
-
-    extractor.extract_all(extraction_options).await?;
-    println!("Archive extracted successfully");
-
-    Ok(())
-}
-```
-
-### Adaptive Compression
-
-```rust
-use rustfs_zip::{AdaptiveCompressor, ContentAnalyzer, CompressionProfile};
-
-async fn adaptive_compression_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create adaptive compressor
-    let mut compressor = AdaptiveCompressor::new();
-
-    // Configure compression profiles
-    compressor.add_profile(CompressionProfile {
-        content_type: "text/*".to_string(),
-        algorithm: CompressionAlgorithm::Brotli,
-        level: CompressionLevel::High,
-        min_size: 1024,
-    });
-
-    compressor.add_profile(CompressionProfile {
-        content_type: "image/*".to_string(),
-        algorithm: CompressionAlgorithm::Lz4,
-        level: CompressionLevel::Fast,
-        min_size: 10240,
-    });
-
-    // Compress different types of content
-    let text_content = std::fs::read("document.txt")?;
-    let image_content = std::fs::read("photo.jpg")?;
-
-    // Analyze and compress
-    let text_result = compressor.compress_adaptive(&text_content, Some("text/plain")).await?;
-    let image_result = compressor.compress_adaptive(&image_content, Some("image/jpeg")).await?;
-
-    println!("Text compression: {} -> {} bytes ({})",
-        text_content.len(), text_result.compressed_size, text_result.algorithm);
-    println!("Image compression: {} -> {} bytes ({})",
-        image_content.len(), image_result.compressed_size, image_result.algorithm);
-
-    Ok(())
-}
-```
-
-### Parallel Compression
-
-```rust
-use rustfs_zip::{ParallelCompressor, CompressionJob};
-
-async fn parallel_compression_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create parallel compressor
-    let compressor = ParallelCompressor::new(4); // 4 worker threads
-
-    // Prepare compression jobs
-    let jobs = vec![
-        CompressionJob {
-            id: "file1".to_string(),
-            data: std::fs::read("file1.txt")?,
-            algorithm: CompressionAlgorithm::Zstd,
-            level: CompressionLevel::Default,
-        },
-        CompressionJob {
-            id: "file2".to_string(),
-            data: std::fs::read("file2.txt")?,
-            algorithm: CompressionAlgorithm::Lz4,
-            level: CompressionLevel::Fast,
-        },
-        CompressionJob {
-            id: "file3".to_string(),
-            data: std::fs::read("file3.txt")?,
-            algorithm: CompressionAlgorithm::Gzip,
-            level: CompressionLevel::High,
-        },
-    ];
-
-    // Execute parallel compression
-    let results = compressor.compress_batch(jobs).await?;
-
-    for result in results {
-        println!("Job {}: {} -> {} bytes",
-            result.job_id, result.original_size, result.compressed_size);
-    }
-
-    Ok(())
-}
-```
-
-### Content Deduplication
-
-```rust
-use rustfs_zip::{DeduplicationCompressor, ContentHash};
-
-async fn deduplication_example() -> Result<(), Box<dyn std::error::Error>> {
-    // Create deduplication compressor
-    let mut compressor = DeduplicationCompressor::new();
-
-    // Add files for compression
-    let file1 = std::fs::read("document1.txt")?;
-    let file2 = std::fs::read("document2.txt")?;
-    let file3 = std::fs::read("document1.txt")?; // Duplicate of file1
-
-    // Compress with deduplication
-    let result1 = compressor.compress_with_dedup("doc1", &file1).await?;
-    let result2 = compressor.compress_with_dedup("doc2", &file2).await?;
-    let result3 = compressor.compress_with_dedup("doc3", &file3).await?;
-
-    println!("File 1: {} bytes -> {} bytes", file1.len(), result1.compressed_size);
-    println!("File 2: {} bytes -> {} bytes", file2.len(), result2.compressed_size);
-    println!("File 3: {} bytes -> {} bytes (deduplicated: {})",
-        file3.len(), result3.compressed_size, result3.is_deduplicated);
-
-    // Get deduplication statistics
-    let stats = compressor.get_dedup_stats();
-    println!("Deduplication saved: {} bytes", stats.bytes_saved);
-    println!("Duplicate files found: {}", stats.duplicate_count);
-
-    Ok(())
-}
-```
-
-## 🏗️ Architecture
-
-### Zip Module Architecture
-
-```
-Zip Architecture:
-┌─────────────────────────────────────────────────────────────┐
-│                    Compression API                          │
-├─────────────────────────────────────────────────────────────┤
-│  Algorithm    │  Streaming    │  Archive     │  Adaptive   │
-│  Selection    │  Compression  │  Management  │  Compression│
-├─────────────────────────────────────────────────────────────┤
-│              Compression Engines                            │
-├─────────────────────────────────────────────────────────────┤
-│   Zstd    │    LZ4    │   Gzip   │   Brotli  │   Custom   │
-├─────────────────────────────────────────────────────────────┤
-│              Low-Level Compression                          │
-└─────────────────────────────────────────────────────────────┘
-```
-
-### Compression Algorithms
-
-| Algorithm | Speed | Ratio | Use Case |
-|-----------|-------|-------|----------|
-| LZ4 | Very Fast | Good | Real-time compression |
-| Zstd | Fast | Excellent | General purpose |
-| Gzip | Medium | Good | Web compatibility |
-| Brotli | Slow | Excellent | Text/web content |
-
-## 🧪 Testing
-
-Run the test suite:
-
-```bash
-# Run all tests
-cargo test
-
-# Test compression algorithms
-cargo test algorithms
-
-# Test streaming compression
-cargo test streaming
-
-# Test archive operations
-cargo test archive
-
-# Benchmark compression performance
-cargo bench
-```
-
-## 📋 Requirements
-
-- **Rust**: 1.70.0 or later
-- **Platforms**: Linux, macOS, Windows
-- **Dependencies**: Native compression libraries
-- **Memory**: Sufficient RAM for compression buffers
-
-## 🌍 Related Projects
-
-This module is part of the RustFS ecosystem:
-
-- [RustFS Main](https://github.com/rustfs/rustfs) - Core distributed storage system
-- [RustFS Rio](../rio) - High-performance I/O
-- [RustFS Utils](../utils) - Utility functions
+- Multiple compression algorithms (Zstd, LZ4, Gzip, Brotli)
+- Streaming compression for memory efficiency
+- Parallel processing for large files
+- Archive format support (ZIP, TAR, custom formats)
+- Adaptive compression with content-type detection
+- Compression analytics and performance metrics
 
 ## 📚 Documentation
 
-For comprehensive documentation, visit:
-
-- [RustFS Documentation](https://docs.rustfs.com)
-- [Zip API Reference](https://docs.rustfs.com/zip/)
-- [Compression Guide](https://docs.rustfs.com/compression/)
-
-## 🔗 Links
-
-- [Documentation](https://docs.rustfs.com) - Complete RustFS manual
-- [Changelog](https://github.com/rustfs/rustfs/releases) - Release notes and updates
-- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Community support
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](https://github.com/rustfs/rustfs/blob/main/CONTRIBUTING.md) for details.
+For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
 ## 📄 License
 
-Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/rustfs/rustfs/blob/main/LICENSE) for details.
-
----
-
-<p align="center">
-  <strong>RustFS</strong> is a trademark of RustFS, Inc.<br>
-  All other trademarks are the property of their respective owners.
-</p>
-
-<p align="center">
-  Made with 📦 by the RustFS Team
-</p>
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

pr_description.md

@@ -0,0 +1,57 @@
+## Summary
+
+This PR modifies the GitHub Actions workflows to ensure that **version releases never get skipped** during CI/CD execution, addressing the issue where duplicate action detection could skip important release processes.
+
+## Changes Made
+
+### 🔧 Core Modifications
+
+1. **Modified skip-duplicate-actions configuration**:
+   - Added `skip_after_successful_duplicate: ${{ !startsWith(github.ref, 'refs/tags/') }}` parameter
+   - This ensures tag pushes (version releases) are never skipped due to duplicate detection
+
+2. **Updated workflow job conditions**:
+   - **CI Workflow** (`ci.yml`): Modified `test-and-lint` and `e2e-tests` jobs
+   - **Build Workflow** (`build.yml`): Modified `build-check`, `build-rustfs`, `build-gui`, `release`, and `upload-oss` jobs
+   - All jobs now use condition: `startsWith(github.ref, 'refs/tags/') || needs.skip-check.outputs.should_skip != 'true'`
+
+### 🎯 Problem Solved
+
+- **Before**: Version releases could be skipped if there were concurrent workflows or duplicate actions
+- **After**: Tag pushes always trigger complete CI/CD pipeline execution, ensuring:
+  - ✅ Full test suite execution
+  - ✅ Code quality checks (fmt, clippy)
+  - ✅ Multi-platform builds (Linux, macOS, Windows)
+  - ✅ GUI builds for releases
+  - ✅ Release asset creation
+  - ✅ OSS uploads
+
+### 🚀 Benefits
+
+1. **Release Quality Assurance**: Every version release undergoes complete validation
+2. **Consistency**: No more uncertainty about whether release builds were properly tested
+3. **Multi-platform Support**: Ensures all target platforms are built for every release
+4. **Backward Compatibility**: Non-release workflows still benefit from duplicate skip optimization
+
+## Testing
+
+- [x] Workflow syntax validated
+- [x] Logic conditions verified for both tag and non-tag scenarios
+- [x] Maintains existing optimization for development builds
+- [x] Follows project coding standards and commit conventions
+
+## Related Issues
+
+This resolves the concern about workflow skipping during version releases, ensuring complete CI/CD execution for all published versions.
+
+## Checklist
+
+- [x] Code follows project formatting standards
+- [x] Commit message follows Conventional Commits format
+- [x] Changes are backwards compatible
+- [x] No breaking changes introduced
+- [x] All workflow conditions properly tested
+
+---
+
+**Note**: This change only affects the execution logic for tag pushes (version releases). Regular development workflows continue to benefit from duplicate action skipping for efficiency.

rustfs/Cargo.toml

@@ -19,6 +19,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
+homepage.workspace = true
+description = "RustFS is a high-performance, distributed file system designed for modern cloud-native applications, providing efficient data storage and retrieval with advanced features like S3 Select, IAM, and policy management."
+keywords.workspace = true
+categories.workspace = true
+documentation = "https://docs.rustfs.com/"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [[bin]]

rustfs/README.md

@@ -1,36 +1,129 @@
-# RustFS
+[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
 
-RustFS is a simple file system written in Rust. It is designed to be a learning project for those who want to understand
-how file systems work and how to implement them in Rust.
+<p align="center">RustFS is a high-performance distributed object storage software built using Rust</p>
+
+<p align="center">
+  <a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
+  <a href="https://github.com/rustfs/rustfs/actions/workflows/docker.yml"><img alt="Build and Push Docker Images" src="https://github.com/rustfs/rustfs/actions/workflows/docker.yml/badge.svg" /></a>
+  <img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/rustfs/rustfs"/>
+  <img alt="Github Last Commit" src="https://img.shields.io/github/last-commit/rustfs/rustfs"/>
+</p>
+
+<p align="center">
+  <a href="https://docs.rustfs.com/en/introduction.html">Getting Started</a>
+  · <a href="https://docs.rustfs.com/en/">Docs</a>
+  · <a href="https://github.com/rustfs/rustfs/issues">Bug reports</a>
+  · <a href="https://github.com/rustfs/rustfs/discussions">Discussions</a>
+</p>
+
+<p align="center">
+English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简体中文</a>
+</p>
+
+RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
+worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
+support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
+comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
+RustFS provides faster speed and safer distributed features for high-performance object storage.
 
 ## Features
 
-- Simple file system structure
-- Basic file operations (create, read, write, delete)
-- Directory support
-- File metadata (size, creation time, etc.)
-- Basic error handling
-- Unit tests for core functionality
-- Documentation for public API
-- Example usage
-- License information
-- Contributing guidelines
-- Changelog
-- Code of conduct
-- Acknowledgements
-- Contact information
-- Links to additional resources
+- **High Performance**: Built with Rust, ensuring speed and efficiency.
+- **Distributed Architecture**: Scalable and fault-tolerant design for large-scale deployments.
+- **S3 Compatibility**: Seamless integration with existing S3-compatible applications.
+- **Data Lake Support**: Optimized for big data and AI workloads.
+- **Open Source**: Licensed under Apache 2.0, encouraging community contributions and transparency.
+- **User-Friendly**: Designed with simplicity in mind, making it easy to deploy and manage.
 
-## Getting Started
+## RustFS vs MinIO
 
-To get started with RustFS, clone the repository and build the project:
+Stress test server parameters
 
-```bash
-git clone git@github.com:rustfs/s3-rustfs.git
-cd rustfs
-cargo build
-```
+| Type    | parameter | Remark                                                   |
+|---------|-----------|----------------------------------------------------------|
+| CPU     | 2 Core    | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz |   |
+| Memory  | 4GB       |                                                          |
+| Network | 15Gbp     |                                                          |
+| Driver  | 40GB x 4  | IOPS 3800 / Driver                                       |
 
-## Usage
+<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
 
-To use RustFS, you can create a new file system instance and perform basic file operations. Here is an example:
+### RustFS vs Other object storage
+
+| RustFS                                                                          | Other object storage                                                                                                    |
+|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| Powerful Console                                                                | Simple and useless Console                                                                                              |
+| Developed based on Rust language, memory is safer                               | Developed in Go or C, with potential issues like memory GC/leaks                                                        |
+| Does not report logs to third-party countries                                   | Reporting logs to other third countries may violate national security laws                                              |
+| Licensed under Apache, more business-friendly                                   | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
+| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support                                                                  |
+| Rust-based development, strong support for secure and innovative devices        | Poor support for edge gateways and secure innovative devices                                                            |
+| Stable commercial prices, free community support                                | High pricing, with costs up to $250,000 for 1PiB                                                                        |
+| No risk                                                                         | Intellectual property risks and risks of prohibited uses                                                                |
+
+## Quickstart
+
+To get started with RustFS, follow these steps:
+
+1. **One-click installation script (Option 1)​​**
+
+   ```bash
+   curl -O  https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
+   ```
+
+2. **Docker Quick Start (Option 2)​​**
+
+  ```bash
+   podman run -d -p 9000:9000 -p 9001:9001 -v /data:/data quay.io/rustfs/rustfs
+   ```
+
+3. **Access the Console**: Open your web browser and navigate to `http://localhost:9001` to access the RustFS console,
+   default username and password is `rustfsadmin` .
+4. **Create a Bucket**: Use the console to create a new bucket for your objects.
+5. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
+   RustFS instance.
+
+## Documentation
+
+For detailed documentation, including configuration options, API references, and advanced usage, please visit
+our [Documentation](https://docs.rustfs.com).
+
+## Getting Help
+
+If you have any questions or need assistance, you can:
+
+- Check the [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) for common issues and solutions.
+- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your
+  experiences.
+- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature
+  requests.
+
+## Links
+
+- [Documentation](https://docs.rustfs.com) - The manual you should read
+- [Changelog](https://github.com/rustfs/rustfs/releases) - What we broke and fixed
+- [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) - Where the community lives
+
+## Contact
+
+- **Bugs**: [GitHub Issues](https://github.com/rustfs/rustfs/issues)
+- **Business**: <hello@rustfs.com>
+- **Jobs**: <jobs@rustfs.com>
+- **General Discussion**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
+- **Contributing**: [CONTRIBUTING.md](CONTRIBUTING.md)
+
+## Contributors
+
+RustFS is a community-driven project, and we appreciate all contributions. Check out
+the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped
+make RustFS better.
+
+<a href="https://github.com/rustfs/rustfs/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=rustfs/rustfs" />
+</a>
+
+## License
+
+[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
+
+**RustFS** is a trademark of RustFS, Inc. All other trademarks are the property of their respective owners.

rustfs/src/config/mod.rs

@@ -86,6 +86,9 @@ pub struct Opt {
 
     #[arg(long, env = "RUSTFS_LICENSE")]
     pub license: Option<String>,
+
+    #[arg(long, env = "RUSTFS_REGION")]
+    pub region: Option<String>,
 }
 
 // lazy_static::lazy_static! {

rustfs/src/console.rs

@@ -27,7 +27,7 @@ use std::io;
 
 use axum::response::Redirect;
 use axum_server::tls_rustls::RustlsConfig;
-use http::{Uri, header};
+use http::{HeaderMap, HeaderName, Uri, header};
 use mime_guess::from_path;
 use rust_embed::RustEmbed;
 use serde::Serialize;
@@ -211,21 +211,40 @@ fn _is_private_ip(ip: IpAddr) -> bool {
 
 #[allow(clippy::const_is_empty)]
 #[instrument(fields(host))]
-async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse {
-    let scheme = uri.scheme().map(|s| s.as_str()).unwrap_or("http");
+async fn config_handler(uri: Uri, Host(host): Host, headers: HeaderMap) -> impl IntoResponse {
+    // Get the scheme from the headers or use the URI scheme
+    let scheme = headers
+        .get(HeaderName::from_static("x-forwarded-proto"))
+        .and_then(|value| value.to_str().ok())
+        .unwrap_or_else(|| uri.scheme().map(|s| s.as_str()).unwrap_or("http"));
+
+    // Print logs for debugging
+    info!("Scheme: {}, ", scheme);
 
     // Get the host from the uri and use the value of the host extractor if it doesn't have one
     let host = uri.host().unwrap_or(host.as_str());
 
-    let host = if host.contains(':') {
-        let (host, _) = host.split_once(':').unwrap_or((host, "80"));
-        host
+    let host = if let Ok(socket_addr) = host.parse::<SocketAddr>() {
+        // Successfully parsed, it's in IP:Port format.
+        // For IPv6, we need to enclose it in brackets to form a valid URL.
+        let ip = socket_addr.ip();
+        if ip.is_ipv6() { format!("[{ip}]") } else { format!("{ip}") }
     } else {
-        host
+        // Failed to parse, it might be a domain name or a bare IP, use it as is.
+        host.to_string()
     };
 
     // Make a copy of the current configuration
-    let mut cfg = CONSOLE_CONFIG.get().unwrap().clone();
+    let mut cfg = match CONSOLE_CONFIG.get() {
+        Some(cfg) => cfg.clone(),
+        None => {
+            error!("Console configuration not initialized");
+            return Response::builder()
+                .status(StatusCode::INTERNAL_SERVER_ERROR)
+                .body(Body::from("Console configuration not initialized"))
+                .unwrap();
+        }
+    };
 
     let url = format!("{}://{}:{}", scheme, host, cfg.port);
     cfg.api.base_url = format!("{url}{RUSTFS_ADMIN_PREFIX}");

rustfs/src/main.rs

@@ -174,6 +174,10 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
 async fn run(opt: config::Opt) -> Result<()> {
     debug!("opt: {:?}", &opt);
 
+    if let Some(region) = opt.region {
+        rustfs_ecstore::global::set_global_region(region);
+    }
+
     let server_addr = parse_and_resolve_address(opt.address.as_str()).map_err(Error::other)?;
     let server_port = server_addr.port();
     let server_address = server_addr.to_string();
@@ -189,7 +193,17 @@ async fn run(opt: config::Opt) -> Result<()> {
     let listener = TcpListener::bind(server_address.clone()).await?;
     // Obtain the listener address
     let local_addr: SocketAddr = listener.local_addr()?;
-    let local_ip = rustfs_utils::get_local_ip().ok_or(local_addr.ip()).unwrap();
+    debug!("Listening on: {}", local_addr);
+    let local_ip = match rustfs_utils::get_local_ip() {
+        Some(ip) => {
+            debug!("Obtained local IP address: {}", ip);
+            ip
+        }
+        None => {
+            warn!("Unable to obtain local IP address, using fallback IP: {}", local_addr.ip());
+            local_addr.ip()
+        }
+    };
 
     // For RPC
     let (endpoint_pools, setup_type) =

rustfs/src/storage/ecfs.rs

@@ -703,6 +703,12 @@ impl S3 for FS {
             .await
             .map_err(ApiError::from)?;
 
+        if let Some(region) = rustfs_ecstore::global::get_global_region() {
+            return Ok(S3Response::new(GetBucketLocationOutput {
+                location_constraint: Some(BucketLocationConstraint::from(region)),
+            }));
+        }
+
         let output = GetBucketLocationOutput::default();
         Ok(S3Response::new(output))
     }

scripts/run.sh

@@ -100,6 +100,8 @@ export RUSTFS_NS_SCANNER_INTERVAL=60  # 对象扫描间隔时间，单位为秒
 
 export RUSTFS_COMPRESSION_ENABLED=true # 是否启用压缩
 
+#export RUSTFS_REGION="us-east-1"
+
 # 事件消息配置
 #export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml"