Merge pull request #312 from rustfs/0xdx2-s3s_xmlns

fix: update s3s version to solve xml namespace type attribute bug.

.docker/compose/docker-compose.cluster.yaml

@@ -27,7 +27,7 @@ services:
     ports:
       - "9000:9000" # Map port 9001 of the host to port 9000 of the container
     volumes:
-      - ../../target/x86_64-unknown-linux-musl/release/rustfs:/app/rustfs
+      - ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
     command: "/app/rustfs"
 
   node1:
@@ -44,7 +44,7 @@ services:
     ports:
       - "9001:9000" # Map port 9002 of the host to port 9000 of the container
     volumes:
-      - ../../target/x86_64-unknown-linux-musl/release/rustfs:/app/rustfs
+      - ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
     command: "/app/rustfs"
 
   node2:
@@ -61,7 +61,7 @@ services:
     ports:
       - "9002:9000" # Map port 9003 of the host to port 9000 of the container
     volumes:
-      - ../../target/x86_64-unknown-linux-musl/release/rustfs:/app/rustfs
+      - ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
     command: "/app/rustfs"
 
   node3:
@@ -78,5 +78,5 @@ services:
     ports:
       - "9003:9000" # Map port 9004 of the host to port 9000 of the container
     volumes:
-      - ../../target/x86_64-unknown-linux-musl/release/rustfs:/app/rustfs
+      - ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
     command: "/app/rustfs"

.github/workflows/build.yml

@@ -172,6 +172,14 @@ jobs:
             target: aarch64-unknown-linux-musl
             cross: true
             platform: linux
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+            cross: false
+            platform: linux
+          - os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            cross: true
+            platform: linux
           # macOS builds
           - os: macos-latest
             target: aarch64-apple-darwin
@@ -181,15 +189,15 @@ jobs:
             target: x86_64-apple-darwin
             cross: false
             platform: macos
-          # # Windows builds (temporarily disabled)
-          # - os: windows-latest
-          #   target: x86_64-pc-windows-msvc
-          #   cross: false
-          #   platform: windows
-          # - os: windows-latest
-          #   target: aarch64-pc-windows-msvc
-          #   cross: true
-          #   platform: windows
+          # Windows builds (temporarily disabled)
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
+            cross: false
+            platform: windows
+          #- os: windows-latest
+          #  target: aarch64-pc-windows-msvc
+          #  cross: true
+          #  platform: windows
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -207,6 +215,7 @@ jobs:
           install-cross-tools: ${{ matrix.cross }}
 
       - name: Download static console assets
+        shell: bash
         run: |
           mkdir -p ./rustfs/static
           if [[ "${{ matrix.platform }}" == "windows" ]]; then
@@ -232,6 +241,7 @@ jobs:
           fi
 
       - name: Build RustFS
+        shell: bash
         run: |
           # Force rebuild by touching build.rs
           touch rustfs/build.rs
@@ -260,30 +270,55 @@ jobs:
           # Extract platform and arch from target
           TARGET="${{ matrix.target }}"
           PLATFORM="${{ matrix.platform }}"
-
-          # Map target to architecture
+          # Map target to architecture and variant
           case "$TARGET" in
+            *x86_64*musl*)
+              ARCH="x86_64"
+              VARIANT="musl"
+              ;;
+            *x86_64*gnu*)
+              ARCH="x86_64"
+              VARIANT="gnu"
+              ;;
             *x86_64*)
               ARCH="x86_64"
+              VARIANT=""
+              ;;
+            *aarch64*musl*|*arm64*musl*)
+              ARCH="aarch64"
+              VARIANT="musl"
+              ;;
+            *aarch64*gnu*|*arm64*gnu*)
+              ARCH="aarch64"
+              VARIANT="gnu"
               ;;
             *aarch64*|*arm64*)
               ARCH="aarch64"
+              VARIANT=""
               ;;
             *armv7*)
               ARCH="armv7"
+              VARIANT=""
               ;;
             *)
               ARCH="unknown"
+              VARIANT=""
               ;;
           esac
 
           # Generate package name based on build type
-          if [[ "$BUILD_TYPE" == "development" ]]; then
-            # Development build: rustfs-${platform}-${arch}-dev-${short_sha}.zip
-            PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH}-dev-${SHORT_SHA}"
+          if [[ -n "$VARIANT" ]]; then
+            ARCH_WITH_VARIANT="${ARCH}-${VARIANT}"
           else
-            # Release/Prerelease build: rustfs-${platform}-${arch}-v${version}.zip
-            PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH}-v${VERSION}"
+            ARCH_WITH_VARIANT="${ARCH}"
+          fi
+
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development build: rustfs-${platform}-${arch}-${variant}-dev-${short_sha}.zip
+            PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-dev-${SHORT_SHA}"
+          else
+            # Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
+            PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-v${VERSION}"
           fi
 
           # Create zip packages for all platforms
@@ -295,23 +330,119 @@ jobs:
           fi
 
           cd target/${{ matrix.target }}/release
-          zip "../../../${PACKAGE_NAME}.zip" rustfs
+          # Determine the binary name based on platform
+          if [[ "${{ matrix.platform }}" == "windows" ]]; then
+            BINARY_NAME="rustfs.exe"
+          else
+            BINARY_NAME="rustfs"
+          fi
+
+          # Verify the binary exists before packaging
+          if [[ ! -f "$BINARY_NAME" ]]; then
+            echo "❌ Binary $BINARY_NAME not found in $(pwd)"
+            if [[ "${{ matrix.platform }}" == "windows" ]]; then
+              dir
+            else
+              ls -la
+            fi
+            exit 1
+          fi
+
+          # Universal packaging function
+          package_zip() {
+            local src=$1
+            local dst=$2
+            if [[ "${{ matrix.platform }}" == "windows" ]]; then
+              # Windows uses PowerShell Compress-Archive
+              powershell -Command "Compress-Archive -Path '$src' -DestinationPath '$dst' -Force"
+            elif command -v zip &> /dev/null; then
+              # Unix systems use zip command
+              zip "$dst" "$src"
+            else
+              echo "❌ No zip utility available"
+              exit 1
+            fi
+          }
+
+          # Create the zip package
+          echo "Start packaging: $BINARY_NAME -> ../../../${PACKAGE_NAME}.zip"
+          package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
+
           cd ../../..
 
+          # Verify the package was created
+          if [[ -f "${PACKAGE_NAME}.zip" ]]; then
+            echo "✅ Package created successfully: ${PACKAGE_NAME}.zip"
+            if [[ "${{ matrix.platform }}" == "windows" ]]; then
+              dir
+            else
+              ls -lh ${PACKAGE_NAME}.zip
+            fi
+          else
+            echo "❌ Failed to create package: ${PACKAGE_NAME}.zip"
+            exit 1
+          fi
+
+          # Create latest version files right after the main package
+          LATEST_FILES=""
+          if [[ "$BUILD_TYPE" == "release" ]] || [[ "$BUILD_TYPE" == "prerelease" ]]; then
+            # Create latest version filename
+            # Convert from rustfs-linux-x86_64-musl-v1.0.0 to rustfs-linux-x86_64-musl-latest
+            LATEST_FILE="${PACKAGE_NAME%-v*}-latest.zip"
+
+            echo "🔄 Creating latest version: ${PACKAGE_NAME}.zip -> $LATEST_FILE"
+            cp "${PACKAGE_NAME}.zip" "$LATEST_FILE"
+
+            if [[ -f "$LATEST_FILE" ]]; then
+              echo "✅ Latest version created: $LATEST_FILE"
+              LATEST_FILES="$LATEST_FILE"
+            fi
+          elif [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development builds (only main branch triggers development builds)
+            # Create main-latest version filename
+            # Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-main-latest
+            MAIN_LATEST_FILE="${PACKAGE_NAME%-dev-*}-main-latest.zip"
+
+            echo "🔄 Creating main-latest version: ${PACKAGE_NAME}.zip -> $MAIN_LATEST_FILE"
+            cp "${PACKAGE_NAME}.zip" "$MAIN_LATEST_FILE"
+
+            if [[ -f "$MAIN_LATEST_FILE" ]]; then
+              echo "✅ Main-latest version created: $MAIN_LATEST_FILE"
+              LATEST_FILES="$MAIN_LATEST_FILE"
+
+              # Also create a generic main-latest for Docker builds (Linux only)
+              if [[ "${{ matrix.platform }}" == "linux" ]]; then
+                DOCKER_MAIN_LATEST_FILE="rustfs-linux-${ARCH_WITH_VARIANT}-main-latest.zip"
+
+                echo "🔄 Creating Docker main-latest version: ${PACKAGE_NAME}.zip -> $DOCKER_MAIN_LATEST_FILE"
+                cp "${PACKAGE_NAME}.zip" "$DOCKER_MAIN_LATEST_FILE"
+
+                if [[ -f "$DOCKER_MAIN_LATEST_FILE" ]]; then
+                  echo "✅ Docker main-latest version created: $DOCKER_MAIN_LATEST_FILE"
+                  LATEST_FILES="$LATEST_FILES $DOCKER_MAIN_LATEST_FILE"
+                fi
+              fi
+            fi
+          fi
+
           echo "package_name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT
           echo "package_file=${PACKAGE_NAME}.zip" >> $GITHUB_OUTPUT
+          echo "latest_files=${LATEST_FILES}" >> $GITHUB_OUTPUT
           echo "build_type=${BUILD_TYPE}" >> $GITHUB_OUTPUT
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
 
           echo "📦 Package created: ${PACKAGE_NAME}.zip"
+          if [[ -n "$LATEST_FILES" ]]; then
+            echo "📦 Latest files created: $LATEST_FILES"
+          fi
           echo "🔧 Build type: ${BUILD_TYPE}"
           echo "📊 Version: ${VERSION}"
 
-      - name: Upload artifacts
+      - name: Upload to GitHub artifacts
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.package.outputs.package_name }}
-          path: ${{ steps.package.outputs.package_file }}
+          path: "rustfs-*.zip"
           retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 7 }}
 
       - name: Upload to Aliyun OSS
@@ -321,6 +452,7 @@ jobs:
           OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
           OSS_REGION: cn-beijing
           OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
+        shell: bash
         run: |
           BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
 
@@ -359,6 +491,16 @@ jobs:
               chmod +x /usr/local/bin/ossutil
               OSSUTIL_BIN=ossutil
               ;;
+            windows)
+              OSSUTIL_ZIP="ossutil-${OSSUTIL_VERSION}-windows-amd64.zip"
+              OSSUTIL_DIR="ossutil-${OSSUTIL_VERSION}-windows-amd64"
+
+              curl -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}"
+              unzip "$OSSUTIL_ZIP"
+              mv "${OSSUTIL_DIR}/ossutil.exe" ./ossutil.exe
+              rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
+              OSSUTIL_BIN=./ossutil.exe
+              ;;
           esac
 
           # Determine upload path based on build type
@@ -370,72 +512,15 @@ jobs:
             echo "📤 Uploading release build to OSS release directory"
           fi
 
-          # Upload the package file to OSS
-          echo "Uploading ${{ steps.package.outputs.package_file }} to $OSS_PATH..."
-          $OSSUTIL_BIN cp "${{ steps.package.outputs.package_file }}" "$OSS_PATH" --force
-
-          # For release and prerelease builds, also create a latest version
-          if [[ "$BUILD_TYPE" == "release" ]] || [[ "$BUILD_TYPE" == "prerelease" ]]; then
-            # Extract platform and arch from package name
-            PACKAGE_NAME="${{ steps.package.outputs.package_name }}"
-
-            # Create latest version filename
-            # Convert from rustfs-linux-x86_64-v1.0.0 to rustfs-linux-x86_64-latest
-            LATEST_FILE="${PACKAGE_NAME%-v*}-latest.zip"
-
-            # Copy the original file to latest version
-            cp "${{ steps.package.outputs.package_file }}" "$LATEST_FILE"
-
-            # Upload the latest version
-            echo "Uploading latest version: $LATEST_FILE to $OSS_PATH..."
-            $OSSUTIL_BIN cp "$LATEST_FILE" "$OSS_PATH" --force
-
-            echo "✅ Latest version uploaded: $LATEST_FILE"
-          fi
-
-          # For development builds, create dev-latest version
-          if [[ "$BUILD_TYPE" == "development" ]]; then
-            # Extract platform and arch from package name
-            PACKAGE_NAME="${{ steps.package.outputs.package_name }}"
-
-            # Create dev-latest version filename
-            # Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-dev-latest
-            DEV_LATEST_FILE="${PACKAGE_NAME%-*}-latest.zip"
-
-            # Copy the original file to dev-latest version
-            cp "${{ steps.package.outputs.package_file }}" "$DEV_LATEST_FILE"
-
-            # Upload the dev-latest version
-            echo "Uploading dev-latest version: $DEV_LATEST_FILE to $OSS_PATH..."
-            $OSSUTIL_BIN cp "$DEV_LATEST_FILE" "$OSS_PATH" --force
-
-            echo "✅ Dev-latest version uploaded: $DEV_LATEST_FILE"
-
-            # For main branch builds, also create a main-latest version
-            if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
-              # Create main-latest version filename
-              # Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-main-latest
-              MAIN_LATEST_FILE="${PACKAGE_NAME%-dev-*}-main-latest.zip"
-
-              # Copy the original file to main-latest version
-              cp "${{ steps.package.outputs.package_file }}" "$MAIN_LATEST_FILE"
-
-              # Upload the main-latest version
-              echo "Uploading main-latest version: $MAIN_LATEST_FILE to $OSS_PATH..."
-              $OSSUTIL_BIN cp "$MAIN_LATEST_FILE" "$OSS_PATH" --force
-
-              echo "✅ Main-latest version uploaded: $MAIN_LATEST_FILE"
-
-              # Also create a generic main-latest for Docker builds
-              if [[ "${{ matrix.platform }}" == "linux" ]]; then
-                DOCKER_MAIN_LATEST_FILE="rustfs-linux-${{ matrix.target == 'x86_64-unknown-linux-musl' && 'x86_64' || 'aarch64' }}-main-latest.zip"
-
-                cp "${{ steps.package.outputs.package_file }}" "$DOCKER_MAIN_LATEST_FILE"
-                $OSSUTIL_BIN cp "$DOCKER_MAIN_LATEST_FILE" "$OSS_PATH" --force
-                echo "✅ Docker main-latest version uploaded: $DOCKER_MAIN_LATEST_FILE"
-              fi
+          # Upload all rustfs zip files to OSS using glob pattern
+          echo "📤 Uploading all rustfs-*.zip files to $OSS_PATH..."
+          for zip_file in rustfs-*.zip; do
+            if [[ -f "$zip_file" ]]; then
+              echo "Uploading: $zip_file to $OSS_PATH..."
+              $OSSUTIL_BIN cp "$zip_file" "$OSS_PATH" --force
+              echo "✅ Uploaded: $zip_file"
             fi
-          fi
+          done
 
           echo "✅ Upload completed successfully"
 
@@ -447,6 +532,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Build completion summary
+        shell: bash
         run: |
           BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
           VERSION="${{ needs.build-check.outputs.version }}"
@@ -511,6 +597,7 @@ jobs:
         id: create
         env:
           GH_TOKEN: ${{ github.token }}
+        shell: bash
         run: |
           TAG="${{ needs.build-check.outputs.version }}"
           VERSION="${{ needs.build-check.outputs.version }}"
@@ -597,13 +684,14 @@ jobs:
 
       - name: Prepare release assets
         id: prepare
+        shell: bash
         run: |
           VERSION="${{ needs.build-check.outputs.version }}"
           TAG="${{ needs.build-check.outputs.version }}"
 
           mkdir -p ./release-assets
 
-          # Copy and verify artifacts
+          # Copy and verify artifacts (including latest files created during build)
           ASSETS_COUNT=0
           for file in ./artifacts/*.zip; do
             if [[ -f "$file" ]]; then
@@ -619,7 +707,7 @@ jobs:
 
           cd ./release-assets
 
-          # Generate checksums
+          # Generate checksums for all files (including latest versions)
           if ls *.zip >/dev/null 2>&1; then
             sha256sum *.zip > SHA256SUMS
             sha512sum *.zip > SHA512SUMS
@@ -634,11 +722,12 @@ jobs:
           echo "📦 Prepared assets:"
           ls -la
 
-          echo "🔢 Asset count: $ASSETS_COUNT"
+          echo "🔢 Total asset count: $ASSETS_COUNT"
 
       - name: Upload to GitHub Release
         env:
           GH_TOKEN: ${{ github.token }}
+        shell: bash
         run: |
           TAG="${{ needs.build-check.outputs.version }}"
 
@@ -658,13 +747,16 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [build-check, upload-release-assets]
-    if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.is_prerelease == 'false'
+    if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
       - name: Update latest.json
         env:
           OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
           OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
+          OSS_REGION: cn-beijing
+          OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
+        shell: bash
         run: |
           if [[ -z "$OSS_ACCESS_KEY_ID" ]]; then
             echo "⚠️ OSS credentials not available, skipping latest.json update"
@@ -681,7 +773,9 @@ jobs:
 
           curl -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}"
           unzip "$OSSUTIL_ZIP"
-          chmod +x "${OSSUTIL_DIR}/ossutil"
+          mv "${OSSUTIL_DIR}/ossutil" /usr/local/bin/
+          rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
+          chmod +x /usr/local/bin/ossutil
 
           # Create latest.json
           cat > latest.json << EOF
@@ -695,7 +789,7 @@ jobs:
           EOF
 
           # Upload to OSS
-          ./${OSSUTIL_DIR}/ossutil cp latest.json oss://rustfs-version/latest.json --force
+          ossutil cp latest.json oss://rustfs-version/latest.json --force
 
           echo "✅ Updated latest.json for stable release $VERSION"
 
@@ -714,6 +808,7 @@ jobs:
       - name: Update release notes and publish
         env:
           GH_TOKEN: ${{ github.token }}
+        shell: bash
         run: |
           TAG="${{ needs.build-check.outputs.version }}"
           VERSION="${{ needs.build-check.outputs.version }}"

.github/workflows/ci.yml

@@ -83,6 +83,16 @@ jobs:
           # Never skip release events and tag pushes
           do_not_skip: '["workflow_dispatch", "schedule", "merge_group", "release", "push"]'
 
+
+  typos:
+    name: Typos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Typos check with custom config file
+        uses: crate-ci/typos@master
+
   test-and-lint:
     name: Test and Lint
     needs: skip-check

Cargo.toml

@@ -33,10 +33,12 @@ members = [
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
     "crates/signer", # client signer
+    "crates/checksums", # client checksums
     "crates/utils", # Utility functions and helpers
     "crates/workers", # Worker thread pools and task scheduling
     "crates/zip", # ZIP file handling and compression
     "crates/ahm",
+    "crates/mcp", # MCP server for S3 operations
 ]
 resolver = "2"
 
@@ -84,8 +86,11 @@ rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-rio = { path = "crates/rio", version = "0.0.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
 rustfs-signer = { path = "crates/signer", version = "0.0.5" }
+rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
+rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
 aes-gcm = { version = "0.10.3", features = ["std"] }
+anyhow = "1.0.98"
 arc-swap = "1.7.1"
 argon2 = { version = "0.5.3", features = ["std"] }
 atoi = "2.0.0"
@@ -94,7 +99,8 @@ async-recursion = "1.1.1"
 async-trait = "0.1.88"
 async-compression = { version = "0.4.0" }
 atomic_enum = "0.3.0"
-aws-sdk-s3 = "1.96.0"
+aws-config = { version = "1.8.3" }
+aws-sdk-s3 = "1.100.0"
 axum = "0.8.4"
 axum-extra = "0.10.1"
 axum-server = { version = "0.7.2", features = ["tls-rustls"] }
@@ -104,12 +110,14 @@ brotli = "8.0.1"
 bytes = { version = "1.10.1", features = ["serde"] }
 bytesize = "2.0.1"
 byteorder = "1.5.0"
+bytes-utils = "0.1.4"
 cfg-if = "1.0.1"
+crc-fast = "1.3.0"
 chacha20poly1305 = { version = "0.10.1" }
 chrono = { version = "0.4.41", features = ["serde"] }
-clap = { version = "4.5.41", features = ["derive", "env"] }
-const-str = { version = "0.6.2", features = ["std", "proc"] }
-crc32fast = "1.4.2"
+clap = { version = "4.5.42", features = ["derive", "env"] }
+const-str = { version = "0.6.4", features = ["std", "proc"] }
+crc32fast = "1.5.0"
 criterion = { version = "0.5", features = ["html_reports"] }
 dashmap = "6.1.0"
 datafusion = "46.0.1"
@@ -130,7 +138,7 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 hmac = "0.12.1"
 hyper = "1.6.0"
-hyper-util = { version = "0.1.15", features = [
+hyper-util = { version = "0.1.16", features = [
     "tokio",
     "server-auto",
     "server-graceful",
@@ -141,13 +149,14 @@ http-body = "1.0.1"
 humantime = "2.2.0"
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 jsonwebtoken = "9.3.1"
-keyring = { version = "3.6.2", features = [
+keyring = { version = "3.6.3", features = [
     "apple-native",
     "windows-native",
     "sync-secret-service",
 ] }
 lazy_static = "1.5.0"
 libsystemd = { version = "0.7.2" }
+lru = "0.16"
 local-ip-address = "0.6.5"
 lz4 = "1.28.1"
 matchit = "0.8.4"
@@ -181,9 +190,10 @@ blake3 = { version = "1.8.2" }
 pbkdf2 = "0.12.2"
 percent-encoding = "2.3.1"
 pin-project-lite = "0.2.16"
-prost = "0.13.5"
+prost = "0.14.1"
+pretty_assertions = "1.4.1"
 quick-xml = "0.38.0"
-rand = "0.9.1"
+rand = "0.9.2"
 rdkafka = { version = "0.38.0", features = ["tokio"] }
 reed-solomon-simd = { version = "3.0.1" }
 regex = { version = "1.11.1" }
@@ -196,10 +206,11 @@ reqwest = { version = "0.12.22", default-features = false, features = [
     "json",
     "blocking",
 ] }
-rfd = { version = "0.15.3", default-features = false, features = [
+rfd = { version = "0.15.4", default-features = false, features = [
     "xdg-portal",
     "tokio",
 ] }
+rmcp = { version = "0.3.1" }
 rmp = "0.8.14"
 rmp-serde = "1.3.0"
 rsa = "0.9.8"
@@ -207,24 +218,25 @@ rumqttc = { version = "0.24" }
 rust-embed = { version = "8.7.2" }
 rust-i18n = { version = "3.1.5" }
 rustfs-rsc = "2025.506.1"
-rustls = { version = "0.23.29" }
+rustls = { version = "0.23.31" }
 rustls-pki-types = "1.12.0"
 rustls-pemfile = "2.2.0"
-s3s = { version = "0.12.0-minio-preview.2" }
-shadow-rs = { version = "1.2.0", default-features = false }
+s3s = { version = "0.12.0-minio-preview.3" }
+schemars = "1.0.4"
 serde = { version = "1.0.219", features = ["derive"] }
-serde_json = { version = "1.0.140", features = ["raw_value"] }
-serde-xml-rs = "0.8.1"
+serde_json = { version = "1.0.141", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
+serial_test = "3.2.0"
 sha1 = "0.10.6"
 sha2 = "0.10.9"
+shadow-rs = { version = "1.2.0", default-features = false }
 siphasher = "1.0.1"
 smallvec = { version = "1.15.1", features = ["serde"] }
 snafu = "0.8.6"
 snap = "1.1.1"
 socket2 = "0.6.0"
-strum = { version = "0.27.1", features = ["derive"] }
-sysinfo = "0.36.0"
+strum = { version = "0.27.2", features = ["derive"] }
+sysinfo = "0.36.1"
 sysctl = "0.6.0"
 tempfile = "3.20.0"
 temp-env = "0.3.6"
@@ -237,22 +249,24 @@ time = { version = "0.3.41", features = [
     "macros",
     "serde",
 ] }
-tokio = { version = "1.46.1", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.47.0", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.2", default-features = false }
 tokio-stream = { version = "0.1.17" }
 tokio-tar = "0.3.1"
 tokio-test = "0.4.4"
 tokio-util = { version = "0.7.15", features = ["io", "compat"] }
-tonic = { version = "0.13.1", features = ["gzip"] }
-tonic-build = { version = "0.13.1" }
+tonic = { version = "0.14.0", features = ["gzip"] }
+tonic-prost = { version = "0.14.0" }
+tonic-prost-build = { version = "0.14.0" }
 tower = { version = "0.5.2", features = ["timeout"] }
 tower-http = { version = "0.6.6", features = ["cors"] }
 tracing = "0.1.41"
+tracing-appender = "0.2.3"
 tracing-core = "0.1.34"
 tracing-error = "0.2.1"
-tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
-tracing-appender = "0.2.3"
 tracing-opentelemetry = "0.31.0"
+tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
+tracing-test = "0.2.5"
 transform-stream = "0.3.1"
 url = "2.5.4"
 urlencoding = "2.1.3"
@@ -266,7 +280,10 @@ winapi = { version = "0.3.9" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
 zip = "2.4.2"
 zstd = "0.13.3"
-anyhow = "1.0.98"
+
+
+[workspace.metadata.cargo-shear]
+ignored = ["rustfs", "rust-i18n"]
 
 [profile.wasm-dev]
 inherits = "dev"

Dockerfile

@@ -1,121 +1,101 @@
 # Multi-stage build for RustFS production image
-FROM alpine:latest AS build
 
-# Build arguments - use TARGETPLATFORM for consistency with Dockerfile.source
-ARG TARGETPLATFORM
-ARG BUILDPLATFORM
+# Build stage: Download and extract RustFS binary
+FROM alpine:3.22 AS build
+
+# Build arguments for platform and release
+ARG TARGETARCH
 ARG RELEASE=latest
 
-# Install dependencies for downloading and verifying binaries
-RUN apk add --no-cache \
-    ca-certificates \
-    curl \
-    bash \
-    wget \
-    unzip \
-    jq
+# Install minimal dependencies for downloading and extracting
+RUN apk add --no-cache ca-certificates curl unzip
 
 # Create build directory
 WORKDIR /build
 
-# Map TARGETPLATFORM to architecture format used in builds
-RUN case "${TARGETPLATFORM}" in \
-        "linux/amd64") ARCH="x86_64" ;; \
-        "linux/arm64") ARCH="aarch64" ;; \
-        *) echo "Unsupported platform: ${TARGETPLATFORM}" && exit 1 ;; \
-    esac && \
-    echo "ARCH=${ARCH}" > /build/arch.env
-
-# Download rustfs binary from dl.rustfs.com (release channel only)
-RUN . /build/arch.env && \
-    BASE_URL="https://dl.rustfs.com/artifacts/rustfs/release" && \
-    PLATFORM="linux" && \
-    if [ "${RELEASE}" = "latest" ]; then \
-        # Download latest release version \
-        PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH}-latest.zip"; \
-        DOWNLOAD_URL="${BASE_URL}/${PACKAGE_NAME}"; \
-        echo "📥 Downloading latest release build: ${PACKAGE_NAME}"; \
+# Set architecture-specific variables
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        echo "x86_64-musl" > /tmp/arch; \
+    elif [ "$TARGETARCH" = "arm64" ]; then \
+        echo "aarch64-musl" > /tmp/arch; \
     else \
-        # Download specific release version \
-        PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH}-v${RELEASE}.zip"; \
-        DOWNLOAD_URL="${BASE_URL}/${PACKAGE_NAME}"; \
-        echo "📥 Downloading specific release version: ${PACKAGE_NAME}"; \
+        echo "unsupported" > /tmp/arch; \
+    fi
+RUN ARCH=$(cat /tmp/arch) && \
+    if [ "$ARCH" = "unsupported" ]; then \
+        echo "Unsupported architecture: $TARGETARCH" && exit 1; \
     fi && \
-    echo "🔗 Download URL: ${DOWNLOAD_URL}" && \
-    curl -f -L "${DOWNLOAD_URL}" -o /build/rustfs.zip && \
-    if [ ! -f /build/rustfs.zip ] || [ ! -s /build/rustfs.zip ]; then \
-        echo "❌ Failed to download binary package"; \
-        echo "💡 Make sure the package ${PACKAGE_NAME} exists"; \
-        echo "🔗 Check: ${DOWNLOAD_URL}"; \
-        exit 1; \
+    if [ "${RELEASE}" = "latest" ]; then \
+        VERSION="latest"; \
+    else \
+        VERSION="v${RELEASE#v}"; \
     fi && \
-    unzip /build/rustfs.zip -d /build && \
+    BASE_URL="https://dl.rustfs.com/artifacts/rustfs/release" && \
+    PACKAGE_NAME="rustfs-linux-${ARCH}-${VERSION}.zip" && \
+    DOWNLOAD_URL="${BASE_URL}/${PACKAGE_NAME}" && \
+    echo "Downloading ${PACKAGE_NAME} from ${DOWNLOAD_URL}" >&2 && \
+    curl -f -L "${DOWNLOAD_URL}" -o rustfs.zip && \
+    unzip rustfs.zip -d /build && \
     chmod +x /build/rustfs && \
-    rm /build/rustfs.zip && \
-    echo "✅ Successfully downloaded and extracted rustfs binary"
+    rm rustfs.zip || { echo "Failed to download or extract ${PACKAGE_NAME}" >&2; exit 1; }
 
-# Runtime stage
-FROM alpine:latest
+# Runtime stage: Configure runtime environment
+FROM alpine:3.22.1
 
-# Set build arguments and labels
+# Build arguments and labels
 ARG RELEASE=latest
 ARG BUILD_DATE
 ARG VCS_REF
 
 LABEL name="RustFS" \
-    vendor="RustFS Team" \
-    maintainer="RustFS Team <dev@rustfs.com>" \
-    version="${RELEASE}" \
-    release="${RELEASE}" \
-    build-date="${BUILD_DATE}" \
-    vcs-ref="${VCS_REF}" \
-    summary="RustFS is a high-performance distributed object storage system written in Rust, compatible with S3 API." \
-    description="RustFS is a high-performance distributed object storage software built using Rust. It supports erasure coding storage, multi-tenant management, observability, and other enterprise-level features." \
-    url="https://rustfs.com" \
-    license="Apache-2.0"
+      vendor="RustFS Team" \
+      maintainer="RustFS Team <dev@rustfs.com>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      build-date="${BUILD_DATE}" \
+      vcs-ref="${VCS_REF}" \
+      summary="High-performance distributed object storage system compatible with S3 API" \
+      description="RustFS is a distributed object storage system written in Rust, supporting erasure coding, multi-tenant management, and observability." \
+      url="https://rustfs.com" \
+      license="Apache-2.0"
 
 # Install runtime dependencies
-RUN apk add --no-cache \
-    ca-certificates \
-    curl \
-    tzdata \
-    bash \
-    && addgroup -g 1000 rustfs \
-    && adduser -u 1000 -G rustfs -s /bin/sh -D rustfs
+RUN echo "https://dl-cdn.alpinelinux.org/alpine/v3.20/community" >> /etc/apk/repositories && \
+    apk update && \
+    apk add --no-cache ca-certificates bash gosu coreutils shadow && \
+    addgroup -g 1000 rustfs && \
+    adduser -u 1000 -G rustfs -s /bin/bash -D rustfs
 
-# Environment variables
-ENV RUSTFS_ACCESS_KEY=rustfsadmin \
+# Copy CA certificates and RustFS binary from build stage
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /build/rustfs /usr/bin/rustfs
+
+# Copy entry point script
+COPY entrypoint.sh /entrypoint.sh
+
+# Set permissions
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh && \
+    mkdir -p /data /logs && \
+    chown rustfs:rustfs /data /logs && \
+    chmod 700 /data /logs
+
+# Environment variables (credentials should be set via environment or secrets)
+ENV RUSTFS_ADDRESS=:9000 \
+    RUSTFS_ACCESS_KEY=rustfsadmin \
     RUSTFS_SECRET_KEY=rustfsadmin \
-    RUSTFS_ADDRESS=":9000" \
     RUSTFS_CONSOLE_ENABLE=true \
     RUSTFS_VOLUMES=/data \
-    RUST_LOG=warn
-
-# Set permissions for /usr/bin (similar to MinIO's approach)
-RUN chmod -R 755 /usr/bin
-
-# Copy CA certificates and binaries from build stage
-COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /build/rustfs /usr/bin/
-
-# Set executable permissions
-RUN chmod +x /usr/bin/rustfs
-
-# Create data directory
-RUN mkdir -p /data /config && chown -R rustfs:rustfs /data /config
-
-# Switch to non-root user
-USER rustfs
-
-# Set working directory
-WORKDIR /data
+    RUST_LOG=warn \
+    RUSTFS_OBS_LOG_DIRECTORY=/logs \
+    RUSTFS_SINKS_FILE_PATH=/logs
 
 # Expose port
 EXPOSE 9000
 
+# Volumes for data and logs
+VOLUME ["/data", "/logs"]
 
-# Volume for data
-VOLUME ["/data"]
+# Set entry point
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/usr/bin/rustfs"]
 
-# Set entrypoint
-ENTRYPOINT ["/usr/bin/rustfs"]

Dockerfile.source

@@ -112,6 +112,8 @@ RUN apt-get update && apt-get install -y \
     ca-certificates \
     tzdata \
     wget \
+    coreutils \
+    passwd \
     && rm -rf /var/lib/apt/lists/*
 
 # Create rustfs user and group
@@ -128,6 +130,10 @@ RUN mkdir -p /data/rustfs{0,1,2,3} && \
 COPY --from=builder /usr/local/bin/rustfs /app/rustfs
 RUN chmod +x /app/rustfs && chown rustfs:rustfs /app/rustfs
 
+# Copy entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
 # Switch to non-root user
 USER rustfs
 
@@ -142,9 +148,9 @@ ENV RUSTFS_ACCESS_KEY=rustfsadmin \
     RUSTFS_VOLUMES=/data \
     RUST_LOG=warn
 
-
 # Volume for data
 VOLUME ["/data"]
 
-# Set default command
+# Set entrypoint and default command
+ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/app/rustfs"]

Makefile

@@ -23,7 +23,7 @@ fmt-check:
 .PHONY: clippy
 clippy:
 	@echo "🔍 Running clippy checks..."
-	cargo clippy --all-targets --all-features -- -D warnings
+	cargo clippy --all-targets --all-features --fix --allow-dirty -- -D warnings
 
 .PHONY: check
 check:
@@ -90,6 +90,18 @@ build-gnu:
 	@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
 	./build-rustfs.sh --platform x86_64-unknown-linux-gnu
 
+.PHONY: build-musl-arm64
+build-musl-arm64:
+	@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
+	@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
+	./build-rustfs.sh --platform aarch64-unknown-linux-musl
+
+.PHONY: build-gnu-arm64
+build-gnu-arm64:
+	@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
+	@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
+	./build-rustfs.sh --platform aarch64-unknown-linux-gnu
+
 .PHONY: deploy-dev
 deploy-dev: build-musl
 	@echo "🚀 Deploying to dev server: $${IP}"
@@ -248,10 +260,14 @@ build-cross-all:
 	@echo "💡 On macOS/Windows, use 'make docker-dev' for reliable multi-arch builds"
 	@echo "🔨 Generating protobuf code..."
 	cargo run --bin gproto || true
-	@echo "🔨 Building x86_64-unknown-linux-musl..."
-	./build-rustfs.sh --platform x86_64-unknown-linux-musl
+	@echo "🔨 Building x86_64-unknown-linux-gnu..."
+	./build-rustfs.sh --platform x86_64-unknown-linux-gnu
 	@echo "🔨 Building aarch64-unknown-linux-gnu..."
 	./build-rustfs.sh --platform aarch64-unknown-linux-gnu
+	@echo "🔨 Building x86_64-unknown-linux-musl..."
+	./build-rustfs.sh --platform x86_64-unknown-linux-musl
+	@echo "🔨 Building aarch64-unknown-linux-musl..."
+	./build-rustfs.sh --platform aarch64-unknown-linux-musl
 	@echo "✅ All architectures built successfully!"
 
 # ========================================================================================
@@ -265,8 +281,10 @@ help-build:
 	@echo "🚀 本地构建 (推荐使用):"
 	@echo "  make build                               # 构建 RustFS 二进制文件 (默认包含 console)"
 	@echo "  make build-dev                           # 开发模式构建"
-	@echo "  make build-musl                          # 构建 musl 版本"
-	@echo "  make build-gnu                           # 构建 GNU 版本"
+	@echo "  make build-musl                          # 构建 x86_64 musl 版本"
+	@echo "  make build-gnu                           # 构建 x86_64 GNU 版本"
+	@echo "  make build-musl-arm64                    # 构建 aarch64 musl 版本"
+	@echo "  make build-gnu-arm64                     # 构建 aarch64 GNU 版本"
 	@echo ""
 	@echo "🐳 Docker 构建:"
 	@echo "  make build-docker                        # 使用 Docker 容器构建"
@@ -281,7 +299,7 @@ help-build:
 	@echo "  ./build-rustfs.sh --force-console-update # 强制更新 console 资源"
 	@echo "  ./build-rustfs.sh --dev                  # 开发模式构建"
 	@echo "  ./build-rustfs.sh --sign                 # 签名二进制文件"
-	@echo "  ./build-rustfs.sh --platform x86_64-unknown-linux-musl  # 指定目标平台"
+	@echo "  ./build-rustfs.sh --platform x86_64-unknown-linux-gnu   # 指定目标平台"
 	@echo "  ./build-rustfs.sh --skip-verification    # 跳过二进制验证"
 	@echo ""
 	@echo "💡 build-rustfs.sh 脚本提供了更多选项、智能检测和二进制验证功能"

README.md

@@ -11,8 +11,8 @@
 </p>
 
 <p align="center">
-  <a href="https://docs.rustfs.com/en/introduction.html">Getting Started</a>
-  · <a href="https://docs.rustfs.com/en/">Docs</a>
+  <a href="https://docs.rustfs.com/introduction.html">Getting Started</a>
+  · <a href="https://docs.rustfs.com/">Docs</a>
   · <a href="https://github.com/rustfs/rustfs/issues">Bug reports</a>
   · <a href="https://github.com/rustfs/rustfs/discussions">Discussions</a>
 </p>

_typos.toml

@@ -0,0 +1,41 @@
+[default]
+# # Ignore specific spell checking patterns
+# extend-ignore-identifiers-re = [
+#     # Ignore common patterns in base64 encoding and hash values
+#     "[A-Za-z0-9+/]{8,}={0,2}",  # base64 encoding
+#     "[A-Fa-f0-9]{8,}",          # hexadecimal hash
+#     "[A-Za-z0-9_-]{20,}",       # long random strings
+# ]
+
+# # Ignore specific regex patterns in content
+# extend-ignore-re = [
+#     # Ignore hash values and encoded strings (base64 patterns)
+#     "(?i)[A-Za-z0-9+/]{8,}={0,2}",
+#     # Ignore long strings in quotes (usually hash or base64)
+#     '"[A-Za-z0-9+/=_-]{8,}"',
+#     # Ignore IV values and similar cryptographic strings
+#     '"[A-Za-z0-9+/=]{12,}"',
+#     # Ignore cryptographic signatures and keys (including partial strings)
+#     "[A-Za-z0-9+/]{6,}[A-Za-z0-9+/=]*",
+#     # Ignore base64-like strings in comments (common in examples)
+#     "//.*[A-Za-z0-9+/]{8,}[A-Za-z0-9+/=]*",
+# ]
+extend-ignore-re = [
+    # Ignore long strings in quotes (usually hash or base64)
+    '"[A-Za-z0-9+/=_-]{32,}"',
+    # Ignore IV values and similar cryptographic strings
+    '"[A-Za-z0-9+/=]{12,}"',
+    # Ignore cryptographic signatures and keys (including partial strings)
+    "[A-Za-z0-9+/]{16,}[A-Za-z0-9+/=]*",
+]
+
+[default.extend-words]
+bui = "bui"
+typ = "typ"
+clen = "clen"
+datas = "datas"
+bre = "bre"
+abd = "abd"
+
+[files]
+extend-exclude = []

build-rustfs.sh

@@ -21,13 +21,17 @@ detect_platform() {
         "linux")
             case "$arch" in
                 "x86_64")
-                    echo "x86_64-unknown-linux-musl"
+                    # Default to GNU for better compatibility
+                    echo "x86_64-unknown-linux-gnu"
                     ;;
                 "aarch64"|"arm64")
-                    echo "aarch64-unknown-linux-musl"
+                    echo "aarch64-unknown-linux-gnu"
                     ;;
                 "armv7l")
-                    echo "armv7-unknown-linux-musleabihf"
+                    echo "armv7-unknown-linux-gnueabihf"
+                    ;;
+                "loongarch64")
+                    echo "loongarch64-unknown-linux-musl"
                     ;;
                 *)
                     echo "unknown-platform"
@@ -119,6 +123,17 @@ usage() {
     echo "  -o, --output-dir DIR       Output directory (default: target/release)"
     echo "  -b, --binary-name NAME     Binary name (default: rustfs)"
     echo "  -p, --platform TARGET      Target platform (default: auto-detect)"
+    echo "                              Supported platforms:"
+    echo "                                x86_64-unknown-linux-gnu"
+    echo "                                aarch64-unknown-linux-gnu"
+    echo "                                armv7-unknown-linux-gnueabihf"
+    echo "                                x86_64-unknown-linux-musl"
+    echo "                                aarch64-unknown-linux-musl"
+    echo "                                armv7-unknown-linux-musleabihf"
+    echo "                                x86_64-apple-darwin"
+    echo "                                aarch64-apple-darwin"
+    echo "                                x86_64-pc-windows-msvc"
+    echo "                                aarch64-pc-windows-msvc"
     echo "  --dev                      Build in dev mode"
     echo "  --sign                     Sign binaries after build"
     echo "  --with-console             Download console static assets (default)"
@@ -385,7 +400,7 @@ build_binary() {
         fi
     else
         # Native compilation
-        build_cmd="cargo build"
+        build_cmd="RUSTFLAGS=-Clink-arg=-lm cargo build"
     fi
 
     if [ "$BUILD_TYPE" = "release" ]; then

cli/rustfs-gui/src/utils/helper.rs

@@ -371,7 +371,7 @@ impl ServiceManager {
                 StdCommand::new("taskkill")
                     .arg("/F")
                     .arg("/PID")
-                    .arg(&service_pid.to_string())
+                    .arg(service_pid.to_string())
                     .output()?;
             }
 

crates/ahm/Cargo.toml

@@ -34,8 +34,14 @@ url = { workspace = true }
 rustfs-lock = { workspace = true }
 
 lazy_static = { workspace = true }
+chrono = { workspace = true }
 
 [dev-dependencies]
 rmp-serde = { workspace = true }
 tokio-test = { workspace = true }
 serde_json = { workspace = true }
+serial_test = "3.2.0"
+once_cell = { workspace = true }
+tracing-subscriber = { workspace = true }
+walkdir = "2.5.0"
+tempfile = { workspace = true }

crates/ahm/src/error.rs

@@ -14,30 +14,79 @@
 
 use thiserror::Error;
 
+/// RustFS AHM/Heal/Scanner 统一错误类型
 #[derive(Debug, Error)]
 pub enum Error {
+    // 通用
     #[error("I/O error: {0}")]
     Io(#[from] std::io::Error),
 
     #[error("Storage error: {0}")]
     Storage(#[from] rustfs_ecstore::error::Error),
 
+    #[error("Disk error: {0}")]
+    Disk(#[from] rustfs_ecstore::disk::error::DiskError),
+
     #[error("Configuration error: {0}")]
     Config(String),
 
+    #[error("Heal configuration error: {message}")]
+    ConfigurationError { message: String },
+
+    #[error("Other error: {0}")]
+    Other(String),
+
+    #[error(transparent)]
+    Anyhow(#[from] anyhow::Error),
+
+    // Scanner相关
     #[error("Scanner error: {0}")]
     Scanner(String),
 
     #[error("Metrics error: {0}")]
     Metrics(String),
 
-    #[error(transparent)]
-    Other(#[from] anyhow::Error),
+    // Heal相关
+    #[error("Heal task not found: {task_id}")]
+    TaskNotFound { task_id: String },
+
+    #[error("Heal task already exists: {task_id}")]
+    TaskAlreadyExists { task_id: String },
+
+    #[error("Heal manager is not running")]
+    ManagerNotRunning,
+
+    #[error("Heal task execution failed: {message}")]
+    TaskExecutionFailed { message: String },
+
+    #[error("Invalid heal type: {heal_type}")]
+    InvalidHealType { heal_type: String },
+
+    #[error("Heal task cancelled")]
+    TaskCancelled,
+
+    #[error("Heal task timeout")]
+    TaskTimeout,
+
+    #[error("Heal event processing failed: {message}")]
+    EventProcessingFailed { message: String },
+
+    #[error("Heal progress tracking failed: {message}")]
+    ProgressTrackingFailed { message: String },
 }
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
-// Implement conversion from ahm::Error to std::io::Error for use in main.rs
+impl Error {
+    pub fn other<E>(error: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        Error::Other(error.into().to_string())
+    }
+}
+
+// 可选：实现与 std::io::Error 的互转
 impl From<Error> for std::io::Error {
     fn from(err: Error) -> Self {
         std::io::Error::other(err)

crates/ahm/src/heal/channel.rs

@@ -0,0 +1,233 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::Result;
+use crate::heal::{
+    manager::HealManager,
+    task::{HealOptions, HealPriority, HealRequest, HealType},
+};
+
+use rustfs_common::heal_channel::{
+    HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse, HealScanMode,
+};
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use tracing::{error, info};
+
+/// Heal channel processor
+pub struct HealChannelProcessor {
+    /// Heal manager
+    heal_manager: Arc<HealManager>,
+    /// Response sender
+    response_sender: mpsc::UnboundedSender<HealChannelResponse>,
+    /// Response receiver
+    response_receiver: mpsc::UnboundedReceiver<HealChannelResponse>,
+}
+
+impl HealChannelProcessor {
+    /// Create new HealChannelProcessor
+    pub fn new(heal_manager: Arc<HealManager>) -> Self {
+        let (response_tx, response_rx) = mpsc::unbounded_channel();
+        Self {
+            heal_manager,
+            response_sender: response_tx,
+            response_receiver: response_rx,
+        }
+    }
+
+    /// Start processing heal channel requests
+    pub async fn start(&mut self, mut receiver: HealChannelReceiver) -> Result<()> {
+        info!("Starting heal channel processor");
+
+        loop {
+            tokio::select! {
+                command = receiver.recv() => {
+                    match command {
+                        Some(command) => {
+                            if let Err(e) = self.process_command(command).await {
+                                error!("Failed to process heal command: {}", e);
+                            }
+                        }
+                        None => {
+                            info!("Heal channel receiver closed, stopping processor");
+                            break;
+                        }
+                    }
+                }
+                response = self.response_receiver.recv() => {
+                    if let Some(response) = response {
+                        // Handle response if needed
+                        info!("Received heal response for request: {}", response.request_id);
+                    }
+                }
+            }
+        }
+
+        info!("Heal channel processor stopped");
+        Ok(())
+    }
+
+    /// Process heal command
+    async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
+        match command {
+            HealChannelCommand::Start(request) => self.process_start_request(request).await,
+            HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await,
+            HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await,
+        }
+    }
+
+    /// Process start request
+    async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> {
+        info!("Processing heal start request: {} for bucket: {}", request.id, request.bucket);
+
+        // Convert channel request to heal request
+        let heal_request = self.convert_to_heal_request(request.clone())?;
+
+        // Submit to heal manager
+        match self.heal_manager.submit_heal_request(heal_request).await {
+            Ok(task_id) => {
+                info!("Successfully submitted heal request: {} as task: {}", request.id, task_id);
+
+                // Send success response
+                let response = HealChannelResponse {
+                    request_id: request.id,
+                    success: true,
+                    data: Some(format!("Task ID: {task_id}").into_bytes()),
+                    error: None,
+                };
+
+                if let Err(e) = self.response_sender.send(response) {
+                    error!("Failed to send heal response: {}", e);
+                }
+            }
+            Err(e) => {
+                error!("Failed to submit heal request: {} - {}", request.id, e);
+
+                // Send error response
+                let response = HealChannelResponse {
+                    request_id: request.id,
+                    success: false,
+                    data: None,
+                    error: Some(e.to_string()),
+                };
+
+                if let Err(e) = self.response_sender.send(response) {
+                    error!("Failed to send heal error response: {}", e);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Process query request
+    async fn process_query_request(&self, heal_path: String, client_token: String) -> Result<()> {
+        info!("Processing heal query request for path: {}", heal_path);
+
+        // TODO: Implement query logic based on heal_path and client_token
+        // For now, return a placeholder response
+        let response = HealChannelResponse {
+            request_id: client_token,
+            success: true,
+            data: Some(format!("Query result for path: {heal_path}").into_bytes()),
+            error: None,
+        };
+
+        if let Err(e) = self.response_sender.send(response) {
+            error!("Failed to send query response: {}", e);
+        }
+
+        Ok(())
+    }
+
+    /// Process cancel request
+    async fn process_cancel_request(&self, heal_path: String) -> Result<()> {
+        info!("Processing heal cancel request for path: {}", heal_path);
+
+        // TODO: Implement cancel logic based on heal_path
+        // For now, return a placeholder response
+        let response = HealChannelResponse {
+            request_id: heal_path.clone(),
+            success: true,
+            data: Some(format!("Cancel request for path: {heal_path}").into_bytes()),
+            error: None,
+        };
+
+        if let Err(e) = self.response_sender.send(response) {
+            error!("Failed to send cancel response: {}", e);
+        }
+
+        Ok(())
+    }
+
+    /// Convert channel request to heal request
+    fn convert_to_heal_request(&self, request: HealChannelRequest) -> Result<HealRequest> {
+        let heal_type = if let Some(disk_id) = &request.disk {
+            HealType::ErasureSet {
+                buckets: vec![],
+                set_disk_id: disk_id.clone(),
+            }
+        } else if let Some(prefix) = &request.object_prefix {
+            if !prefix.is_empty() {
+                HealType::Object {
+                    bucket: request.bucket.clone(),
+                    object: prefix.clone(),
+                    version_id: None,
+                }
+            } else {
+                HealType::Bucket {
+                    bucket: request.bucket.clone(),
+                }
+            }
+        } else {
+            HealType::Bucket {
+                bucket: request.bucket.clone(),
+            }
+        };
+
+        let priority = match request.priority {
+            HealChannelPriority::Low => HealPriority::Low,
+            HealChannelPriority::Normal => HealPriority::Normal,
+            HealChannelPriority::High => HealPriority::High,
+            HealChannelPriority::Critical => HealPriority::Urgent,
+        };
+
+        // Build HealOptions with all available fields
+        let mut options = HealOptions {
+            scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
+            remove_corrupted: request.remove_corrupted.unwrap_or(false),
+            recreate_missing: request.recreate_missing.unwrap_or(true),
+            update_parity: request.update_parity.unwrap_or(true),
+            recursive: request.recursive.unwrap_or(false),
+            dry_run: request.dry_run.unwrap_or(false),
+            timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
+            pool_index: request.pool_index,
+            set_index: request.set_index,
+        };
+
+        // Apply force_start overrides
+        if request.force_start {
+            options.remove_corrupted = true;
+            options.recreate_missing = true;
+            options.update_parity = true;
+        }
+
+        Ok(HealRequest::new(heal_type, options, priority))
+    }
+
+    /// Get response sender for external use
+    pub fn get_response_sender(&self) -> mpsc::UnboundedSender<HealChannelResponse> {
+        self.response_sender.clone()
+    }
+}

crates/ahm/src/heal/erasure_healer.rs

@@ -0,0 +1,456 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::heal::{
+    progress::HealProgress,
+    resume::{CheckpointManager, ResumeManager, ResumeUtils},
+    storage::HealStorageAPI,
+};
+use futures::future::join_all;
+use rustfs_common::heal_channel::{HealOpts, HealScanMode};
+use rustfs_ecstore::disk::DiskStore;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{error, info, warn};
+
+/// Erasure Set Healer
+pub struct ErasureSetHealer {
+    storage: Arc<dyn HealStorageAPI>,
+    progress: Arc<RwLock<HealProgress>>,
+    cancel_token: tokio_util::sync::CancellationToken,
+    disk: DiskStore,
+}
+
+impl ErasureSetHealer {
+    pub fn new(
+        storage: Arc<dyn HealStorageAPI>,
+        progress: Arc<RwLock<HealProgress>>,
+        cancel_token: tokio_util::sync::CancellationToken,
+        disk: DiskStore,
+    ) -> Self {
+        Self {
+            storage,
+            progress,
+            cancel_token,
+            disk,
+        }
+    }
+
+    /// execute erasure set heal with resume
+    pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
+        info!("Starting erasure set heal for {} buckets on set disk {}", buckets.len(), set_disk_id);
+
+        // 1. generate or get task id
+        let task_id = self.get_or_create_task_id(set_disk_id).await?;
+
+        // 2. initialize or resume resume state
+        let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, buckets).await?;
+
+        // 3. execute heal with resume
+        let result = self
+            .execute_heal_with_resume(buckets, &resume_manager, &checkpoint_manager)
+            .await;
+
+        // 4. cleanup resume state
+        if result.is_ok() {
+            if let Err(e) = resume_manager.cleanup().await {
+                warn!("Failed to cleanup resume state: {}", e);
+            }
+            if let Err(e) = checkpoint_manager.cleanup().await {
+                warn!("Failed to cleanup checkpoint: {}", e);
+            }
+        }
+
+        result
+    }
+
+    /// get or create task id
+    async fn get_or_create_task_id(&self, _set_disk_id: &str) -> Result<String> {
+        // check if there are resumable tasks
+        let resumable_tasks = ResumeUtils::get_resumable_tasks(&self.disk).await?;
+
+        for task_id in resumable_tasks {
+            if ResumeUtils::can_resume_task(&self.disk, &task_id).await {
+                info!("Found resumable task: {}", task_id);
+                return Ok(task_id);
+            }
+        }
+
+        // create new task id
+        let task_id = ResumeUtils::generate_task_id();
+        info!("Created new heal task: {}", task_id);
+        Ok(task_id)
+    }
+
+    /// initialize or resume resume state
+    async fn initialize_resume_state(&self, task_id: &str, buckets: &[String]) -> Result<(ResumeManager, CheckpointManager)> {
+        // check if resume state exists
+        if ResumeManager::has_resume_state(&self.disk, task_id).await {
+            info!("Loading existing resume state for task: {}", task_id);
+
+            let resume_manager = ResumeManager::load_from_disk(self.disk.clone(), task_id).await?;
+            let checkpoint_manager = if CheckpointManager::has_checkpoint(&self.disk, task_id).await {
+                CheckpointManager::load_from_disk(self.disk.clone(), task_id).await?
+            } else {
+                CheckpointManager::new(self.disk.clone(), task_id.to_string()).await?
+            };
+
+            Ok((resume_manager, checkpoint_manager))
+        } else {
+            info!("Creating new resume state for task: {}", task_id);
+
+            let resume_manager =
+                ResumeManager::new(self.disk.clone(), task_id.to_string(), "erasure_set".to_string(), buckets.to_vec()).await?;
+
+            let checkpoint_manager = CheckpointManager::new(self.disk.clone(), task_id.to_string()).await?;
+
+            Ok((resume_manager, checkpoint_manager))
+        }
+    }
+
+    /// execute heal with resume
+    async fn execute_heal_with_resume(
+        &self,
+        buckets: &[String],
+        resume_manager: &ResumeManager,
+        checkpoint_manager: &CheckpointManager,
+    ) -> Result<()> {
+        // 1. get current state
+        let state = resume_manager.get_state().await;
+        let checkpoint = checkpoint_manager.get_checkpoint().await;
+
+        info!(
+            "Resuming from bucket {} object {}",
+            checkpoint.current_bucket_index, checkpoint.current_object_index
+        );
+
+        // 2. initialize progress
+        self.initialize_progress(buckets, &state).await;
+
+        // 3. continue from checkpoint
+        let current_bucket_index = checkpoint.current_bucket_index;
+        let mut current_object_index = checkpoint.current_object_index;
+
+        let mut processed_objects = state.processed_objects;
+        let mut successful_objects = state.successful_objects;
+        let mut failed_objects = state.failed_objects;
+        let mut skipped_objects = state.skipped_objects;
+
+        // 4. process remaining buckets
+        for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) {
+            // check if completed
+            if state.completed_buckets.contains(bucket) {
+                continue;
+            }
+
+            // update current bucket
+            resume_manager.set_current_item(Some(bucket.clone()), None).await?;
+
+            // process objects in bucket
+            let bucket_result = self
+                .heal_bucket_with_resume(
+                    bucket,
+                    &mut current_object_index,
+                    &mut processed_objects,
+                    &mut successful_objects,
+                    &mut failed_objects,
+                    &mut skipped_objects,
+                    resume_manager,
+                    checkpoint_manager,
+                )
+                .await;
+
+            // update checkpoint position
+            checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
+
+            // update progress
+            resume_manager
+                .update_progress(processed_objects, successful_objects, failed_objects, skipped_objects)
+                .await?;
+
+            // check cancel status
+            if self.cancel_token.is_cancelled() {
+                info!("Heal task cancelled");
+                return Err(Error::TaskCancelled);
+            }
+
+            // process bucket result
+            match bucket_result {
+                Ok(_) => {
+                    resume_manager.complete_bucket(bucket).await?;
+                    info!("Completed heal for bucket: {}", bucket);
+                }
+                Err(e) => {
+                    error!("Failed to heal bucket {}: {}", bucket, e);
+                    // continue to next bucket, do not interrupt the whole process
+                }
+            }
+
+            // reset object index
+            current_object_index = 0;
+        }
+
+        // 5. mark task completed
+        resume_manager.mark_completed().await?;
+
+        info!("Erasure set heal completed successfully");
+        Ok(())
+    }
+
+    /// heal single bucket with resume
+    #[allow(clippy::too_many_arguments)]
+    async fn heal_bucket_with_resume(
+        &self,
+        bucket: &str,
+        current_object_index: &mut usize,
+        processed_objects: &mut u64,
+        successful_objects: &mut u64,
+        failed_objects: &mut u64,
+        _skipped_objects: &mut u64,
+        resume_manager: &ResumeManager,
+        checkpoint_manager: &CheckpointManager,
+    ) -> Result<()> {
+        info!("Starting heal for bucket: {} from object index {}", bucket, current_object_index);
+
+        // 1. get bucket info
+        let _bucket_info = match self.storage.get_bucket_info(bucket).await? {
+            Some(info) => info,
+            None => {
+                warn!("Bucket {} not found, skipping", bucket);
+                return Ok(());
+            }
+        };
+
+        // 2. get objects to heal
+        let objects = self.storage.list_objects_for_heal(bucket, "").await?;
+
+        // 3. continue from checkpoint
+        for (obj_idx, object) in objects.iter().enumerate().skip(*current_object_index) {
+            // check if already processed
+            if checkpoint_manager.get_checkpoint().await.processed_objects.contains(object) {
+                continue;
+            }
+
+            // update current object
+            resume_manager
+                .set_current_item(Some(bucket.to_string()), Some(object.clone()))
+                .await?;
+
+            // heal object
+            let heal_opts = HealOpts {
+                scan_mode: HealScanMode::Normal,
+                remove: true,
+                recreate: true,
+                ..Default::default()
+            };
+
+            match self.storage.heal_object(bucket, object, None, &heal_opts).await {
+                Ok((_result, None)) => {
+                    *successful_objects += 1;
+                    checkpoint_manager.add_processed_object(object.clone()).await?;
+                    info!("Successfully healed object {}/{}", bucket, object);
+                }
+                Ok((_, Some(err))) => {
+                    *failed_objects += 1;
+                    checkpoint_manager.add_failed_object(object.clone()).await?;
+                    warn!("Failed to heal object {}/{}: {}", bucket, object, err);
+                }
+                Err(err) => {
+                    *failed_objects += 1;
+                    checkpoint_manager.add_failed_object(object.clone()).await?;
+                    warn!("Error healing object {}/{}: {}", bucket, object, err);
+                }
+            }
+
+            *processed_objects += 1;
+            *current_object_index = obj_idx + 1;
+
+            // check cancel status
+            if self.cancel_token.is_cancelled() {
+                info!("Heal task cancelled during object processing");
+                return Err(Error::TaskCancelled);
+            }
+
+            // save checkpoint periodically
+            if obj_idx % 100 == 0 {
+                checkpoint_manager.update_position(0, *current_object_index).await?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// initialize progress tracking
+    async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) {
+        let mut progress = self.progress.write().await;
+        progress.objects_scanned = state.total_objects;
+        progress.objects_healed = state.successful_objects;
+        progress.objects_failed = state.failed_objects;
+        progress.bytes_processed = 0; // set to 0 for now, can be extended later
+        progress.set_current_object(state.current_object.clone());
+    }
+
+    /// heal all buckets concurrently
+    #[allow(dead_code)]
+    async fn heal_buckets_concurrently(&self, buckets: &[String]) -> Vec<Result<()>> {
+        // use semaphore to control concurrency, avoid too many concurrent healings
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(4)); // max 4 concurrent healings
+
+        let heal_futures = buckets.iter().map(|bucket| {
+            let bucket = bucket.clone();
+            let storage = self.storage.clone();
+            let progress = self.progress.clone();
+            let semaphore = semaphore.clone();
+            let cancel_token = self.cancel_token.clone();
+
+            async move {
+                let _permit = semaphore.acquire().await.unwrap();
+
+                if cancel_token.is_cancelled() {
+                    return Err(Error::TaskCancelled);
+                }
+
+                Self::heal_single_bucket(&storage, &bucket, &progress).await
+            }
+        });
+
+        // use join_all to process concurrently
+        join_all(heal_futures).await
+    }
+
+    /// heal single bucket
+    #[allow(dead_code)]
+    async fn heal_single_bucket(
+        storage: &Arc<dyn HealStorageAPI>,
+        bucket: &str,
+        progress: &Arc<RwLock<HealProgress>>,
+    ) -> Result<()> {
+        info!("Starting heal for bucket: {}", bucket);
+
+        // 1. get bucket info
+        let _bucket_info = match storage.get_bucket_info(bucket).await? {
+            Some(info) => info,
+            None => {
+                warn!("Bucket {} not found, skipping", bucket);
+                return Ok(());
+            }
+        };
+
+        // 2. get objects to heal
+        let objects = storage.list_objects_for_heal(bucket, "").await?;
+
+        // 3. update progress
+        {
+            let mut p = progress.write().await;
+            p.objects_scanned += objects.len() as u64;
+        }
+
+        // 4. heal objects concurrently
+        let heal_opts = HealOpts {
+            scan_mode: HealScanMode::Normal,
+            remove: true,   // remove corrupted data
+            recreate: true, // recreate missing data
+            ..Default::default()
+        };
+
+        let object_results = Self::heal_objects_concurrently(storage, bucket, &objects, &heal_opts, progress).await;
+
+        // 5. count results
+        let (success_count, failure_count) = object_results
+            .into_iter()
+            .fold((0, 0), |(success, failure), result| match result {
+                Ok(_) => (success + 1, failure),
+                Err(_) => (success, failure + 1),
+            });
+
+        // 6. update progress
+        {
+            let mut p = progress.write().await;
+            p.objects_healed += success_count;
+            p.objects_failed += failure_count;
+            p.set_current_object(Some(format!("completed bucket: {bucket}")));
+        }
+
+        info!(
+            "Completed heal for bucket {}: {} success, {} failures",
+            bucket, success_count, failure_count
+        );
+
+        Ok(())
+    }
+
+    /// heal objects concurrently
+    #[allow(dead_code)]
+    async fn heal_objects_concurrently(
+        storage: &Arc<dyn HealStorageAPI>,
+        bucket: &str,
+        objects: &[String],
+        heal_opts: &HealOpts,
+        _progress: &Arc<RwLock<HealProgress>>,
+    ) -> Vec<Result<()>> {
+        // use semaphore to control object healing concurrency
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(8)); // max 8 concurrent object healings
+
+        let heal_futures = objects.iter().map(|object| {
+            let object = object.clone();
+            let bucket = bucket.to_string();
+            let storage = storage.clone();
+            let heal_opts = *heal_opts;
+            let semaphore = semaphore.clone();
+
+            async move {
+                let _permit = semaphore.acquire().await.unwrap();
+
+                match storage.heal_object(&bucket, &object, None, &heal_opts).await {
+                    Ok((_result, None)) => {
+                        info!("Successfully healed object {}/{}", bucket, object);
+                        Ok(())
+                    }
+                    Ok((_, Some(err))) => {
+                        warn!("Failed to heal object {}/{}: {}", bucket, object, err);
+                        Err(Error::other(err))
+                    }
+                    Err(err) => {
+                        warn!("Error healing object {}/{}: {}", bucket, object, err);
+                        Err(err)
+                    }
+                }
+            }
+        });
+
+        join_all(heal_futures).await
+    }
+
+    /// process results
+    #[allow(dead_code)]
+    async fn process_results(&self, results: Vec<Result<()>>) -> Result<()> {
+        let (success_count, failure_count): (usize, usize) =
+            results.into_iter().fold((0, 0), |(success, failure), result| match result {
+                Ok(_) => (success + 1, failure),
+                Err(_) => (success, failure + 1),
+            });
+
+        let total = success_count + failure_count;
+
+        info!("Erasure set heal completed: {}/{} buckets successful", success_count, total);
+
+        if failure_count > 0 {
+            warn!("{} buckets failed to heal", failure_count);
+            return Err(Error::other(format!("{failure_count} buckets failed to heal")));
+        }
+
+        Ok(())
+    }
+}

crates/ahm/src/heal/event.rs

@@ -0,0 +1,359 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
+use rustfs_ecstore::disk::endpoint::Endpoint;
+use serde::{Deserialize, Serialize};
+use std::time::SystemTime;
+
+/// Corruption type
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CorruptionType {
+    /// Data corruption
+    DataCorruption,
+    /// Metadata corruption
+    MetadataCorruption,
+    /// Partial corruption
+    PartialCorruption,
+    /// Complete corruption
+    CompleteCorruption,
+}
+
+/// Severity level
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub enum Severity {
+    /// Low severity
+    Low = 0,
+    /// Medium severity
+    Medium = 1,
+    /// High severity
+    High = 2,
+    /// Critical severity
+    Critical = 3,
+}
+
+/// Heal event
+#[derive(Debug, Clone)]
+pub enum HealEvent {
+    /// Object corruption event
+    ObjectCorruption {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        corruption_type: CorruptionType,
+        severity: Severity,
+    },
+    /// Object missing event
+    ObjectMissing {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        expected_locations: Vec<usize>,
+        available_locations: Vec<usize>,
+    },
+    /// Metadata corruption event
+    MetadataCorruption {
+        bucket: String,
+        object: String,
+        corruption_type: CorruptionType,
+    },
+    /// Disk status change event
+    DiskStatusChange {
+        endpoint: Endpoint,
+        old_status: String,
+        new_status: String,
+    },
+    /// EC decode failure event
+    ECDecodeFailure {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        missing_shards: Vec<usize>,
+        available_shards: Vec<usize>,
+    },
+    /// Checksum mismatch event
+    ChecksumMismatch {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        expected_checksum: String,
+        actual_checksum: String,
+    },
+    /// Bucket metadata corruption event
+    BucketMetadataCorruption {
+        bucket: String,
+        corruption_type: CorruptionType,
+    },
+    /// MRF metadata corruption event
+    MRFMetadataCorruption {
+        meta_path: String,
+        corruption_type: CorruptionType,
+    },
+}
+
+impl HealEvent {
+    /// Convert HealEvent to HealRequest
+    pub fn to_heal_request(&self) -> HealRequest {
+        match self {
+            HealEvent::ObjectCorruption {
+                bucket,
+                object,
+                version_id,
+                severity,
+                ..
+            } => HealRequest::new(
+                HealType::Object {
+                    bucket: bucket.clone(),
+                    object: object.clone(),
+                    version_id: version_id.clone(),
+                },
+                HealOptions::default(),
+                Self::severity_to_priority(severity),
+            ),
+            HealEvent::ObjectMissing {
+                bucket,
+                object,
+                version_id,
+                ..
+            } => HealRequest::new(
+                HealType::Object {
+                    bucket: bucket.clone(),
+                    object: object.clone(),
+                    version_id: version_id.clone(),
+                },
+                HealOptions::default(),
+                HealPriority::High,
+            ),
+            HealEvent::MetadataCorruption { bucket, object, .. } => HealRequest::new(
+                HealType::Metadata {
+                    bucket: bucket.clone(),
+                    object: object.clone(),
+                },
+                HealOptions::default(),
+                HealPriority::High,
+            ),
+            HealEvent::DiskStatusChange { endpoint, .. } => {
+                // Convert disk status change to erasure set heal
+                // Note: This requires access to storage to get bucket list, which is not available here
+                // The actual bucket list will need to be provided by the caller or retrieved differently
+                HealRequest::new(
+                    HealType::ErasureSet {
+                        buckets: vec![], // Empty bucket list - caller should populate this
+                        set_disk_id: format!("{}_{}", endpoint.pool_idx, endpoint.set_idx),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::ECDecodeFailure {
+                bucket,
+                object,
+                version_id,
+                ..
+            } => HealRequest::new(
+                HealType::ECDecode {
+                    bucket: bucket.clone(),
+                    object: object.clone(),
+                    version_id: version_id.clone(),
+                },
+                HealOptions::default(),
+                HealPriority::Urgent,
+            ),
+            HealEvent::ChecksumMismatch {
+                bucket,
+                object,
+                version_id,
+                ..
+            } => HealRequest::new(
+                HealType::Object {
+                    bucket: bucket.clone(),
+                    object: object.clone(),
+                    version_id: version_id.clone(),
+                },
+                HealOptions::default(),
+                HealPriority::High,
+            ),
+            HealEvent::BucketMetadataCorruption { bucket, .. } => {
+                HealRequest::new(HealType::Bucket { bucket: bucket.clone() }, HealOptions::default(), HealPriority::High)
+            }
+            HealEvent::MRFMetadataCorruption { meta_path, .. } => HealRequest::new(
+                HealType::MRF {
+                    meta_path: meta_path.clone(),
+                },
+                HealOptions::default(),
+                HealPriority::High,
+            ),
+        }
+    }
+
+    /// Convert severity to priority
+    fn severity_to_priority(severity: &Severity) -> HealPriority {
+        match severity {
+            Severity::Low => HealPriority::Low,
+            Severity::Medium => HealPriority::Normal,
+            Severity::High => HealPriority::High,
+            Severity::Critical => HealPriority::Urgent,
+        }
+    }
+
+    /// Get event description
+    pub fn description(&self) -> String {
+        match self {
+            HealEvent::ObjectCorruption {
+                bucket,
+                object,
+                corruption_type,
+                ..
+            } => {
+                format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}")
+            }
+            HealEvent::ObjectMissing { bucket, object, .. } => {
+                format!("Object missing: {bucket}/{object}")
+            }
+            HealEvent::MetadataCorruption {
+                bucket,
+                object,
+                corruption_type,
+                ..
+            } => {
+                format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}")
+            }
+            HealEvent::DiskStatusChange {
+                endpoint,
+                old_status,
+                new_status,
+                ..
+            } => {
+                format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}")
+            }
+            HealEvent::ECDecodeFailure {
+                bucket,
+                object,
+                missing_shards,
+                ..
+            } => {
+                format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}")
+            }
+            HealEvent::ChecksumMismatch {
+                bucket,
+                object,
+                expected_checksum,
+                actual_checksum,
+                ..
+            } => {
+                format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}")
+            }
+            HealEvent::BucketMetadataCorruption {
+                bucket, corruption_type, ..
+            } => {
+                format!("Bucket metadata corruption: {bucket} - {corruption_type:?}")
+            }
+            HealEvent::MRFMetadataCorruption {
+                meta_path,
+                corruption_type,
+                ..
+            } => {
+                format!("MRF metadata corruption: {meta_path} - {corruption_type:?}")
+            }
+        }
+    }
+
+    /// Get event severity
+    pub fn severity(&self) -> Severity {
+        match self {
+            HealEvent::ObjectCorruption { severity, .. } => severity.clone(),
+            HealEvent::ObjectMissing { .. } => Severity::High,
+            HealEvent::MetadataCorruption { .. } => Severity::High,
+            HealEvent::DiskStatusChange { .. } => Severity::High,
+            HealEvent::ECDecodeFailure { .. } => Severity::Critical,
+            HealEvent::ChecksumMismatch { .. } => Severity::High,
+            HealEvent::BucketMetadataCorruption { .. } => Severity::High,
+            HealEvent::MRFMetadataCorruption { .. } => Severity::High,
+        }
+    }
+
+    /// Get event timestamp
+    pub fn timestamp(&self) -> SystemTime {
+        SystemTime::now()
+    }
+}
+
+/// Heal event handler
+pub struct HealEventHandler {
+    /// Event queue
+    events: Vec<HealEvent>,
+    /// Maximum number of events
+    max_events: usize,
+}
+
+impl HealEventHandler {
+    pub fn new(max_events: usize) -> Self {
+        Self {
+            events: Vec::new(),
+            max_events,
+        }
+    }
+
+    /// Add event
+    pub fn add_event(&mut self, event: HealEvent) {
+        if self.events.len() >= self.max_events {
+            // Remove oldest event
+            self.events.remove(0);
+        }
+        self.events.push(event);
+    }
+
+    /// Get all events
+    pub fn get_events(&self) -> &[HealEvent] {
+        &self.events
+    }
+
+    /// Clear events
+    pub fn clear_events(&mut self) {
+        self.events.clear();
+    }
+
+    /// Get event count
+    pub fn event_count(&self) -> usize {
+        self.events.len()
+    }
+
+    /// Filter events by severity
+    pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> {
+        self.events.iter().filter(|event| event.severity() >= min_severity).collect()
+    }
+
+    /// Filter events by type
+    pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> {
+        self.events
+            .iter()
+            .filter(|event| match event {
+                HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption",
+                HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing",
+                HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption",
+                HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange",
+                HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure",
+                HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch",
+                HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption",
+                HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption",
+            })
+            .collect()
+    }
+}
+
+impl Default for HealEventHandler {
+    fn default() -> Self {
+        Self::new(1000)
+    }
+}

crates/ahm/src/heal/manager.rs

@@ -0,0 +1,422 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::heal::{
+    progress::{HealProgress, HealStatistics},
+    storage::HealStorageAPI,
+    task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
+};
+use rustfs_ecstore::disk::DiskAPI;
+use rustfs_ecstore::disk::error::DiskError;
+use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
+use std::{
+    collections::{HashMap, VecDeque},
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+use tokio::{
+    sync::{Mutex, RwLock},
+    time::interval,
+};
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info, warn};
+
+/// Heal config
+#[derive(Debug, Clone)]
+pub struct HealConfig {
+    /// Whether to enable auto heal
+    pub enable_auto_heal: bool,
+    /// Heal interval
+    pub heal_interval: Duration,
+    /// Maximum concurrent heal tasks
+    pub max_concurrent_heals: usize,
+    /// Task timeout
+    pub task_timeout: Duration,
+    /// Queue size
+    pub queue_size: usize,
+}
+
+impl Default for HealConfig {
+    fn default() -> Self {
+        Self {
+            enable_auto_heal: true,
+            heal_interval: Duration::from_secs(10), // 10 seconds
+            max_concurrent_heals: 4,
+            task_timeout: Duration::from_secs(300), // 5 minutes
+            queue_size: 1000,
+        }
+    }
+}
+
+/// Heal state
+#[derive(Debug, Default)]
+pub struct HealState {
+    /// Whether running
+    pub is_running: bool,
+    /// Current heal cycle
+    pub current_cycle: u64,
+    /// Last heal time
+    pub last_heal_time: Option<SystemTime>,
+    /// Total healed objects
+    pub total_healed_objects: u64,
+    /// Total heal failures
+    pub total_heal_failures: u64,
+    /// Current active heal tasks
+    pub active_heal_count: usize,
+}
+
+/// Heal manager
+pub struct HealManager {
+    /// Heal config
+    config: Arc<RwLock<HealConfig>>,
+    /// Heal state
+    state: Arc<RwLock<HealState>>,
+    /// Active heal tasks
+    active_heals: Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
+    /// Heal queue
+    heal_queue: Arc<Mutex<VecDeque<HealRequest>>>,
+    /// Storage layer interface
+    storage: Arc<dyn HealStorageAPI>,
+    /// Cancel token
+    cancel_token: CancellationToken,
+    /// Statistics
+    statistics: Arc<RwLock<HealStatistics>>,
+}
+
+impl HealManager {
+    /// Create new HealManager
+    pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
+        let config = config.unwrap_or_default();
+        Self {
+            config: Arc::new(RwLock::new(config)),
+            state: Arc::new(RwLock::new(HealState::default())),
+            active_heals: Arc::new(Mutex::new(HashMap::new())),
+            heal_queue: Arc::new(Mutex::new(VecDeque::new())),
+            storage,
+            cancel_token: CancellationToken::new(),
+            statistics: Arc::new(RwLock::new(HealStatistics::new())),
+        }
+    }
+
+    /// Start HealManager
+    pub async fn start(&self) -> Result<()> {
+        let mut state = self.state.write().await;
+        if state.is_running {
+            warn!("HealManager is already running");
+            return Ok(());
+        }
+        state.is_running = true;
+        drop(state);
+
+        info!("Starting HealManager");
+
+        // start scheduler
+        self.start_scheduler().await?;
+
+        // start auto disk scanner
+        self.start_auto_disk_scanner().await?;
+
+        info!("HealManager started successfully");
+        Ok(())
+    }
+
+    /// Stop HealManager
+    pub async fn stop(&self) -> Result<()> {
+        info!("Stopping HealManager");
+
+        // cancel all tasks
+        self.cancel_token.cancel();
+
+        // wait for all tasks to complete
+        let mut active_heals = self.active_heals.lock().await;
+        for task in active_heals.values() {
+            if let Err(e) = task.cancel().await {
+                warn!("Failed to cancel task {}: {}", task.id, e);
+            }
+        }
+        active_heals.clear();
+
+        // update state
+        let mut state = self.state.write().await;
+        state.is_running = false;
+
+        info!("HealManager stopped successfully");
+        Ok(())
+    }
+
+    /// Submit heal request
+    pub async fn submit_heal_request(&self, request: HealRequest) -> Result<String> {
+        let config = self.config.read().await;
+        let mut queue = self.heal_queue.lock().await;
+
+        if queue.len() >= config.queue_size {
+            return Err(Error::ConfigurationError {
+                message: "Heal queue is full".to_string(),
+            });
+        }
+
+        let request_id = request.id.clone();
+        queue.push_back(request);
+        drop(queue);
+
+        info!("Submitted heal request: {}", request_id);
+        Ok(request_id)
+    }
+
+    /// Get task status
+    pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
+        let active_heals = self.active_heals.lock().await;
+        if let Some(task) = active_heals.get(task_id) {
+            Ok(task.get_status().await)
+        } else {
+            Err(Error::TaskNotFound {
+                task_id: task_id.to_string(),
+            })
+        }
+    }
+
+    /// Get task progress
+    pub async fn get_active_tasks_count(&self) -> usize {
+        self.active_heals.lock().await.len()
+    }
+
+    pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
+        let active_heals = self.active_heals.lock().await;
+        if let Some(task) = active_heals.get(task_id) {
+            Ok(task.get_progress().await)
+        } else {
+            Err(Error::TaskNotFound {
+                task_id: task_id.to_string(),
+            })
+        }
+    }
+
+    /// Cancel task
+    pub async fn cancel_task(&self, task_id: &str) -> Result<()> {
+        let mut active_heals = self.active_heals.lock().await;
+        if let Some(task) = active_heals.get(task_id) {
+            task.cancel().await?;
+            active_heals.remove(task_id);
+            info!("Cancelled heal task: {}", task_id);
+            Ok(())
+        } else {
+            Err(Error::TaskNotFound {
+                task_id: task_id.to_string(),
+            })
+        }
+    }
+
+    /// Get statistics
+    pub async fn get_statistics(&self) -> HealStatistics {
+        self.statistics.read().await.clone()
+    }
+
+    /// Get active task count
+    pub async fn get_active_task_count(&self) -> usize {
+        let active_heals = self.active_heals.lock().await;
+        active_heals.len()
+    }
+
+    /// Get queue length
+    pub async fn get_queue_length(&self) -> usize {
+        let queue = self.heal_queue.lock().await;
+        queue.len()
+    }
+
+    /// Start scheduler
+    async fn start_scheduler(&self) -> Result<()> {
+        let config = self.config.clone();
+        let heal_queue = self.heal_queue.clone();
+        let active_heals = self.active_heals.clone();
+        let cancel_token = self.cancel_token.clone();
+        let statistics = self.statistics.clone();
+        let storage = self.storage.clone();
+
+        tokio::spawn(async move {
+            let mut interval = interval(config.read().await.heal_interval);
+
+            loop {
+                tokio::select! {
+                    _ = cancel_token.cancelled() => {
+                        info!("Heal scheduler received shutdown signal");
+                        break;
+                    }
+                    _ = interval.tick() => {
+                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage).await;
+                    }
+                }
+            }
+        });
+
+        Ok(())
+    }
+
+    /// Start background task to auto scan local disks and enqueue erasure set heal requests
+    async fn start_auto_disk_scanner(&self) -> Result<()> {
+        let config = self.config.clone();
+        let heal_queue = self.heal_queue.clone();
+        let active_heals = self.active_heals.clone();
+        let cancel_token = self.cancel_token.clone();
+        let storage = self.storage.clone();
+
+        tokio::spawn(async move {
+            let mut interval = interval(config.read().await.heal_interval);
+
+            loop {
+                tokio::select! {
+                    _ = cancel_token.cancelled() => {
+                        info!("Auto disk scanner received shutdown signal");
+                        break;
+                    }
+                    _ = interval.tick() => {
+                        // Build list of endpoints that need healing
+                        let mut endpoints = Vec::new();
+                        for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
+                            if let Some(disk) = disk_opt {
+                                // detect unformatted disk via get_disk_id()
+                                if let Err(err) = disk.get_disk_id().await {
+                                    if err == DiskError::UnformattedDisk {
+                                        endpoints.push(disk.endpoint());
+                                        continue;
+                                    }
+                                }
+                            }
+                        }
+
+                        if endpoints.is_empty() {
+                            continue;
+                        }
+
+                        // Get bucket list for erasure set healing
+                        let buckets = match storage.list_buckets().await {
+                            Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
+                            Err(e) => {
+                                error!("Failed to get bucket list for auto healing: {}", e);
+                                continue;
+                            }
+                        };
+
+                        // Create erasure set heal requests for each endpoint
+                        for ep in endpoints {
+                            // skip if already queued or healing
+                            let mut skip = false;
+                            {
+                                let queue = heal_queue.lock().await;
+                                if queue.iter().any(|req| matches!(&req.heal_type, crate::heal::task::HealType::ErasureSet { set_disk_id, .. } if set_disk_id == &format!("{}_{}", ep.pool_idx, ep.set_idx))) {
+                                    skip = true;
+                                }
+                            }
+                            if !skip {
+                                let active = active_heals.lock().await;
+                                if active.values().any(|task| matches!(&task.heal_type, crate::heal::task::HealType::ErasureSet { set_disk_id, .. } if set_disk_id == &format!("{}_{}", ep.pool_idx, ep.set_idx))) {
+                                    skip = true;
+                                }
+                            }
+
+                            if skip {
+                                continue;
+                            }
+
+                            // enqueue erasure set heal request for this disk
+                            let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx);
+                            let req = HealRequest::new(
+                                HealType::ErasureSet {
+                                    buckets: buckets.clone(),
+                                    set_disk_id: set_disk_id.clone()
+                                },
+                                HealOptions::default(),
+                                HealPriority::Normal,
+                            );
+                            let mut queue = heal_queue.lock().await;
+                            queue.push_back(req);
+                            info!("Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
+                        }
+                    }
+                }
+            }
+        });
+        Ok(())
+    }
+
+    /// Process heal queue
+    async fn process_heal_queue(
+        heal_queue: &Arc<Mutex<VecDeque<HealRequest>>>,
+        active_heals: &Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
+        config: &Arc<RwLock<HealConfig>>,
+        statistics: &Arc<RwLock<HealStatistics>>,
+        storage: &Arc<dyn HealStorageAPI>,
+    ) {
+        let config = config.read().await;
+        let mut active_heals_guard = active_heals.lock().await;
+
+        // check if new heal tasks can be started
+        if active_heals_guard.len() >= config.max_concurrent_heals {
+            return;
+        }
+
+        let mut queue = heal_queue.lock().await;
+        if let Some(request) = queue.pop_front() {
+            let task = Arc::new(HealTask::from_request(request, storage.clone()));
+            let task_id = task.id.clone();
+            active_heals_guard.insert(task_id.clone(), task.clone());
+            drop(active_heals_guard);
+            let active_heals_clone = active_heals.clone();
+            let statistics_clone = statistics.clone();
+
+            // start heal task
+            tokio::spawn(async move {
+                info!("Starting heal task: {}", task_id);
+                let result = task.execute().await;
+                match result {
+                    Ok(_) => {
+                        info!("Heal task completed successfully: {}", task_id);
+                    }
+                    Err(e) => {
+                        error!("Heal task failed: {} - {}", task_id, e);
+                    }
+                }
+                let mut active_heals_guard = active_heals_clone.lock().await;
+                if let Some(completed_task) = active_heals_guard.remove(&task_id) {
+                    // update statistics
+                    let mut stats = statistics_clone.write().await;
+                    match completed_task.get_status().await {
+                        HealTaskStatus::Completed => {
+                            stats.update_task_completion(true);
+                        }
+                        _ => {
+                            stats.update_task_completion(false);
+                        }
+                    }
+                    stats.update_running_tasks(active_heals_guard.len() as u64);
+                }
+            });
+
+            // update statistics
+            let mut stats = statistics.write().await;
+            stats.total_tasks += 1;
+        }
+    }
+}
+
+impl std::fmt::Debug for HealManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HealManager")
+            .field("config", &"<config>")
+            .field("state", &"<state>")
+            .field("active_heals_count", &"<active_heals>")
+            .field("queue_length", &"<queue>")
+            .finish()
+    }
+}

crates/ahm/src/heal/mod.rs

@@ -12,24 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Serialize};
+pub mod channel;
+pub mod erasure_healer;
+pub mod event;
+pub mod manager;
+pub mod progress;
+pub mod resume;
+pub mod storage;
+pub mod task;
 
-/// Logger configuration
-#[derive(Debug, Deserialize, Serialize, Clone)]
-pub struct LoggerConfig {
-    pub queue_capacity: Option<usize>,
-}
-
-impl LoggerConfig {
-    pub fn new() -> Self {
-        Self {
-            queue_capacity: Some(10000),
-        }
-    }
-}
-
-impl Default for LoggerConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}
+pub use erasure_healer::ErasureSetHealer;
+pub use manager::HealManager;
+pub use resume::{CheckpointManager, ResumeCheckpoint, ResumeManager, ResumeState, ResumeUtils};
+pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};

crates/ahm/src/heal/progress.rs

@@ -0,0 +1,148 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+use std::time::SystemTime;
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct HealProgress {
+    /// Objects scanned
+    pub objects_scanned: u64,
+    /// Objects healed
+    pub objects_healed: u64,
+    /// Objects failed
+    pub objects_failed: u64,
+    /// Bytes processed
+    pub bytes_processed: u64,
+    /// Current object
+    pub current_object: Option<String>,
+    /// Progress percentage
+    pub progress_percentage: f64,
+    /// Start time
+    pub start_time: Option<SystemTime>,
+    /// Last update time
+    pub last_update_time: Option<SystemTime>,
+    /// Estimated completion time
+    pub estimated_completion_time: Option<SystemTime>,
+}
+
+impl HealProgress {
+    pub fn new() -> Self {
+        Self {
+            start_time: Some(SystemTime::now()),
+            last_update_time: Some(SystemTime::now()),
+            ..Default::default()
+        }
+    }
+
+    pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
+        self.objects_scanned = scanned;
+        self.objects_healed = healed;
+        self.objects_failed = failed;
+        self.bytes_processed = bytes;
+        self.last_update_time = Some(SystemTime::now());
+
+        // calculate progress percentage
+        let total = scanned + healed + failed;
+        if total > 0 {
+            self.progress_percentage = (healed as f64 / total as f64) * 100.0;
+        }
+    }
+
+    pub fn set_current_object(&mut self, object: Option<String>) {
+        self.current_object = object;
+        self.last_update_time = Some(SystemTime::now());
+    }
+
+    pub fn is_completed(&self) -> bool {
+        self.progress_percentage >= 100.0
+            || self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned
+    }
+
+    pub fn get_success_rate(&self) -> f64 {
+        let total = self.objects_healed + self.objects_failed;
+        if total > 0 {
+            (self.objects_healed as f64 / total as f64) * 100.0
+        } else {
+            0.0
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HealStatistics {
+    /// Total heal tasks
+    pub total_tasks: u64,
+    /// Successful tasks
+    pub successful_tasks: u64,
+    /// Failed tasks
+    pub failed_tasks: u64,
+    /// Running tasks
+    pub running_tasks: u64,
+    /// Total healed objects
+    pub total_objects_healed: u64,
+    /// Total healed bytes
+    pub total_bytes_healed: u64,
+    /// Last update time
+    pub last_update_time: SystemTime,
+}
+
+impl Default for HealStatistics {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl HealStatistics {
+    pub fn new() -> Self {
+        Self {
+            total_tasks: 0,
+            successful_tasks: 0,
+            failed_tasks: 0,
+            running_tasks: 0,
+            total_objects_healed: 0,
+            total_bytes_healed: 0,
+            last_update_time: SystemTime::now(),
+        }
+    }
+
+    pub fn update_task_completion(&mut self, success: bool) {
+        if success {
+            self.successful_tasks += 1;
+        } else {
+            self.failed_tasks += 1;
+        }
+        self.last_update_time = SystemTime::now();
+    }
+
+    pub fn update_running_tasks(&mut self, count: u64) {
+        self.running_tasks = count;
+        self.last_update_time = SystemTime::now();
+    }
+
+    pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
+        self.total_objects_healed += count;
+        self.total_bytes_healed += bytes;
+        self.last_update_time = SystemTime::now();
+    }
+
+    pub fn get_success_rate(&self) -> f64 {
+        let total = self.successful_tasks + self.failed_tasks;
+        if total > 0 {
+            (self.successful_tasks as f64 / total as f64) * 100.0
+        } else {
+            0.0
+        }
+    }
+}

crates/ahm/src/heal/resume.rs

@@ -0,0 +1,696 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use rustfs_ecstore::disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET};
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::sync::RwLock;
+use tracing::{debug, info, warn};
+use uuid::Uuid;
+
+/// resume state file constants
+const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
+const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
+const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
+
+/// resume state
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ResumeState {
+    /// task id
+    pub task_id: String,
+    /// task type
+    pub task_type: String,
+    /// start time
+    pub start_time: u64,
+    /// last update time
+    pub last_update: u64,
+    /// completed
+    pub completed: bool,
+    /// total objects
+    pub total_objects: u64,
+    /// processed objects
+    pub processed_objects: u64,
+    /// successful objects
+    pub successful_objects: u64,
+    /// failed objects
+    pub failed_objects: u64,
+    /// skipped objects
+    pub skipped_objects: u64,
+    /// current bucket
+    pub current_bucket: Option<String>,
+    /// current object
+    pub current_object: Option<String>,
+    /// completed buckets
+    pub completed_buckets: Vec<String>,
+    /// pending buckets
+    pub pending_buckets: Vec<String>,
+    /// error message
+    pub error_message: Option<String>,
+    /// retry count
+    pub retry_count: u32,
+    /// max retries
+    pub max_retries: u32,
+}
+
+impl ResumeState {
+    pub fn new(task_id: String, task_type: String, buckets: Vec<String>) -> Self {
+        Self {
+            task_id,
+            task_type,
+            start_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
+            last_update: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
+            completed: false,
+            total_objects: 0,
+            processed_objects: 0,
+            successful_objects: 0,
+            failed_objects: 0,
+            skipped_objects: 0,
+            current_bucket: None,
+            current_object: None,
+            completed_buckets: Vec::new(),
+            pending_buckets: buckets,
+            error_message: None,
+            retry_count: 0,
+            max_retries: 3,
+        }
+    }
+
+    pub fn update_progress(&mut self, processed: u64, successful: u64, failed: u64, skipped: u64) {
+        self.processed_objects = processed;
+        self.successful_objects = successful;
+        self.failed_objects = failed;
+        self.skipped_objects = skipped;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn set_current_item(&mut self, bucket: Option<String>, object: Option<String>) {
+        self.current_bucket = bucket;
+        self.current_object = object;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn complete_bucket(&mut self, bucket: &str) {
+        if !self.completed_buckets.contains(&bucket.to_string()) {
+            self.completed_buckets.push(bucket.to_string());
+        }
+        if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) {
+            self.pending_buckets.remove(pos);
+        }
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn mark_completed(&mut self) {
+        self.completed = true;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn set_error(&mut self, error: String) {
+        self.error_message = Some(error);
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn increment_retry(&mut self) {
+        self.retry_count += 1;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn can_retry(&self) -> bool {
+        self.retry_count < self.max_retries
+    }
+
+    pub fn get_progress_percentage(&self) -> f64 {
+        if self.total_objects == 0 {
+            return 0.0;
+        }
+        (self.processed_objects as f64 / self.total_objects as f64) * 100.0
+    }
+
+    pub fn get_success_rate(&self) -> f64 {
+        let total = self.successful_objects + self.failed_objects;
+        if total == 0 {
+            return 0.0;
+        }
+        (self.successful_objects as f64 / total as f64) * 100.0
+    }
+}
+
+/// resume manager
+pub struct ResumeManager {
+    disk: DiskStore,
+    state: Arc<RwLock<ResumeState>>,
+}
+
+impl ResumeManager {
+    /// create new resume manager
+    pub async fn new(disk: DiskStore, task_id: String, task_type: String, buckets: Vec<String>) -> Result<Self> {
+        let state = ResumeState::new(task_id, task_type, buckets);
+        let manager = Self {
+            disk,
+            state: Arc::new(RwLock::new(state)),
+        };
+
+        // save initial state
+        manager.save_state().await?;
+        Ok(manager)
+    }
+
+    /// load resume state from disk
+    pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
+        let state_data = Self::read_state_file(&disk, task_id).await?;
+        let state: ResumeState = serde_json::from_slice(&state_data).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to deserialize resume state: {e}"),
+        })?;
+
+        Ok(Self {
+            disk,
+            state: Arc::new(RwLock::new(state)),
+        })
+    }
+
+    /// check if resume state exists
+    pub async fn has_resume_state(disk: &DiskStore, task_id: &str) -> bool {
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}"));
+        match disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await {
+            Ok(data) => !data.is_empty(),
+            Err(_) => false,
+        }
+    }
+
+    /// get current state
+    pub async fn get_state(&self) -> ResumeState {
+        self.state.read().await.clone()
+    }
+
+    /// update progress
+    pub async fn update_progress(&self, processed: u64, successful: u64, failed: u64, skipped: u64) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.update_progress(processed, successful, failed, skipped);
+        drop(state);
+        self.save_state().await
+    }
+
+    /// set current item
+    pub async fn set_current_item(&self, bucket: Option<String>, object: Option<String>) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.set_current_item(bucket, object);
+        drop(state);
+        self.save_state().await
+    }
+
+    /// complete bucket
+    pub async fn complete_bucket(&self, bucket: &str) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.complete_bucket(bucket);
+        drop(state);
+        self.save_state().await
+    }
+
+    /// mark task completed
+    pub async fn mark_completed(&self) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.mark_completed();
+        drop(state);
+        self.save_state().await
+    }
+
+    /// set error message
+    pub async fn set_error(&self, error: String) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.set_error(error);
+        drop(state);
+        self.save_state().await
+    }
+
+    /// increment retry count
+    pub async fn increment_retry(&self) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.increment_retry();
+        drop(state);
+        self.save_state().await
+    }
+
+    /// cleanup resume state
+    pub async fn cleanup(&self) -> Result<()> {
+        let state = self.state.read().await;
+        let task_id = &state.task_id;
+
+        // delete state files
+        let state_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}"));
+        let progress_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_PROGRESS_FILE}"));
+        let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
+
+        // ignore delete errors, files may not exist
+        let _ = self
+            .disk
+            .delete(RUSTFS_META_BUCKET, state_file.to_str().unwrap(), Default::default())
+            .await;
+        let _ = self
+            .disk
+            .delete(RUSTFS_META_BUCKET, progress_file.to_str().unwrap(), Default::default())
+            .await;
+        let _ = self
+            .disk
+            .delete(RUSTFS_META_BUCKET, checkpoint_file.to_str().unwrap(), Default::default())
+            .await;
+
+        info!("Cleaned up resume state for task: {}", task_id);
+        Ok(())
+    }
+
+    /// save state to disk
+    async fn save_state(&self) -> Result<()> {
+        let state = self.state.read().await;
+        let state_data = serde_json::to_vec(&*state).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to serialize resume state: {e}"),
+        })?;
+
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", state.task_id, RESUME_STATE_FILE));
+
+        self.disk
+            .write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), state_data.into())
+            .await
+            .map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to save resume state: {e}"),
+            })?;
+
+        debug!("Saved resume state for task: {}", state.task_id);
+        Ok(())
+    }
+
+    /// read state file from disk
+    async fn read_state_file(disk: &DiskStore, task_id: &str) -> Result<Vec<u8>> {
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_STATE_FILE}"));
+
+        disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap())
+            .await
+            .map(|bytes| bytes.to_vec())
+            .map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to read resume state file: {e}"),
+            })
+    }
+}
+
+/// resume checkpoint
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ResumeCheckpoint {
+    /// task id
+    pub task_id: String,
+    /// checkpoint time
+    pub checkpoint_time: u64,
+    /// current bucket index
+    pub current_bucket_index: usize,
+    /// current object index
+    pub current_object_index: usize,
+    /// processed objects
+    pub processed_objects: Vec<String>,
+    /// failed objects
+    pub failed_objects: Vec<String>,
+    /// skipped objects
+    pub skipped_objects: Vec<String>,
+}
+
+impl ResumeCheckpoint {
+    pub fn new(task_id: String) -> Self {
+        Self {
+            task_id,
+            checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
+            current_bucket_index: 0,
+            current_object_index: 0,
+            processed_objects: Vec::new(),
+            failed_objects: Vec::new(),
+            skipped_objects: Vec::new(),
+        }
+    }
+
+    pub fn update_position(&mut self, bucket_index: usize, object_index: usize) {
+        self.current_bucket_index = bucket_index;
+        self.current_object_index = object_index;
+        self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+    }
+
+    pub fn add_processed_object(&mut self, object: String) {
+        if !self.processed_objects.contains(&object) {
+            self.processed_objects.push(object);
+        }
+    }
+
+    pub fn add_failed_object(&mut self, object: String) {
+        if !self.failed_objects.contains(&object) {
+            self.failed_objects.push(object);
+        }
+    }
+
+    pub fn add_skipped_object(&mut self, object: String) {
+        if !self.skipped_objects.contains(&object) {
+            self.skipped_objects.push(object);
+        }
+    }
+}
+
+/// resume checkpoint manager
+pub struct CheckpointManager {
+    disk: DiskStore,
+    checkpoint: Arc<RwLock<ResumeCheckpoint>>,
+}
+
+impl CheckpointManager {
+    /// create new checkpoint manager
+    pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
+        let checkpoint = ResumeCheckpoint::new(task_id);
+        let manager = Self {
+            disk,
+            checkpoint: Arc::new(RwLock::new(checkpoint)),
+        };
+
+        // save initial checkpoint
+        manager.save_checkpoint().await?;
+        Ok(manager)
+    }
+
+    /// load checkpoint from disk
+    pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
+        let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
+        let checkpoint: ResumeCheckpoint = serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to deserialize checkpoint: {e}"),
+        })?;
+
+        Ok(Self {
+            disk,
+            checkpoint: Arc::new(RwLock::new(checkpoint)),
+        })
+    }
+
+    /// check if checkpoint exists
+    pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool {
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
+        match disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await {
+            Ok(data) => !data.is_empty(),
+            Err(_) => false,
+        }
+    }
+
+    /// get current checkpoint
+    pub async fn get_checkpoint(&self) -> ResumeCheckpoint {
+        self.checkpoint.read().await.clone()
+    }
+
+    /// update position
+    pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.update_position(bucket_index, object_index);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// add processed object
+    pub async fn add_processed_object(&self, object: String) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.add_processed_object(object);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// add failed object
+    pub async fn add_failed_object(&self, object: String) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.add_failed_object(object);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// add skipped object
+    pub async fn add_skipped_object(&self, object: String) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.add_skipped_object(object);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// cleanup checkpoint
+    pub async fn cleanup(&self) -> Result<()> {
+        let checkpoint = self.checkpoint.read().await;
+        let task_id = &checkpoint.task_id;
+
+        let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
+        let _ = self
+            .disk
+            .delete(RUSTFS_META_BUCKET, checkpoint_file.to_str().unwrap(), Default::default())
+            .await;
+
+        info!("Cleaned up checkpoint for task: {}", task_id);
+        Ok(())
+    }
+
+    /// save checkpoint to disk
+    async fn save_checkpoint(&self) -> Result<()> {
+        let checkpoint = self.checkpoint.read().await;
+        let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to serialize checkpoint: {e}"),
+        })?;
+
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
+
+        self.disk
+            .write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), checkpoint_data.into())
+            .await
+            .map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to save checkpoint: {e}"),
+            })?;
+
+        debug!("Saved checkpoint for task: {}", checkpoint.task_id);
+        Ok(())
+    }
+
+    /// read checkpoint file from disk
+    async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result<Vec<u8>> {
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
+
+        disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap())
+            .await
+            .map(|bytes| bytes.to_vec())
+            .map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to read checkpoint file: {e}"),
+            })
+    }
+}
+
+/// resume utils
+pub struct ResumeUtils;
+
+impl ResumeUtils {
+    /// generate unique task id
+    pub fn generate_task_id() -> String {
+        Uuid::new_v4().to_string()
+    }
+
+    /// check if task can be resumed
+    pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool {
+        ResumeManager::has_resume_state(disk, task_id).await
+    }
+
+    /// get all resumable task ids
+    pub async fn get_resumable_tasks(disk: &DiskStore) -> Result<Vec<String>> {
+        // List all files in the buckets metadata directory
+        let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await {
+            Ok(entries) => entries,
+            Err(e) => {
+                debug!("Failed to list resume state files: {}", e);
+                return Ok(Vec::new());
+            }
+        };
+
+        let mut task_ids = Vec::new();
+
+        // Filter files that end with ahm_resume_state.json and extract task IDs
+        for entry in entries {
+            if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
+                // Extract task ID from filename: {task_id}_ahm_resume_state.json
+                if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) {
+                    if !task_id.is_empty() {
+                        task_ids.push(task_id.to_string());
+                    }
+                }
+            }
+        }
+
+        debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids);
+        Ok(task_ids)
+    }
+
+    /// cleanup expired resume states
+    pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> {
+        let task_ids = Self::get_resumable_tasks(disk).await?;
+        let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+
+        for task_id in task_ids {
+            if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await {
+                let state = resume_manager.get_state().await;
+                let age_hours = (current_time - state.last_update) / 3600;
+
+                if age_hours > max_age_hours {
+                    info!("Cleaning up expired resume state for task: {} (age: {} hours)", task_id, age_hours);
+                    if let Err(e) = resume_manager.cleanup().await {
+                        warn!("Failed to cleanup expired resume state for task {}: {}", task_id, e);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_resume_state_creation() {
+        let task_id = ResumeUtils::generate_task_id();
+        let buckets = vec!["bucket1".to_string(), "bucket2".to_string()];
+        let state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), buckets);
+
+        assert_eq!(state.task_id, task_id);
+        assert_eq!(state.task_type, "erasure_set");
+        assert!(!state.completed);
+        assert_eq!(state.processed_objects, 0);
+        assert_eq!(state.pending_buckets.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_resume_state_progress() {
+        let task_id = ResumeUtils::generate_task_id();
+        let buckets = vec!["bucket1".to_string()];
+        let mut state = ResumeState::new(task_id, "erasure_set".to_string(), buckets);
+
+        state.update_progress(10, 8, 1, 1);
+        assert_eq!(state.processed_objects, 10);
+        assert_eq!(state.successful_objects, 8);
+        assert_eq!(state.failed_objects, 1);
+        assert_eq!(state.skipped_objects, 1);
+
+        let progress = state.get_progress_percentage();
+        assert_eq!(progress, 0.0); // total_objects is 0
+
+        state.total_objects = 100;
+        let progress = state.get_progress_percentage();
+        assert_eq!(progress, 10.0);
+    }
+
+    #[tokio::test]
+    async fn test_resume_state_bucket_completion() {
+        let task_id = ResumeUtils::generate_task_id();
+        let buckets = vec!["bucket1".to_string(), "bucket2".to_string()];
+        let mut state = ResumeState::new(task_id, "erasure_set".to_string(), buckets);
+
+        assert_eq!(state.pending_buckets.len(), 2);
+        assert_eq!(state.completed_buckets.len(), 0);
+
+        state.complete_bucket("bucket1");
+        assert_eq!(state.pending_buckets.len(), 1);
+        assert_eq!(state.completed_buckets.len(), 1);
+        assert!(state.completed_buckets.contains(&"bucket1".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_resume_utils() {
+        let task_id1 = ResumeUtils::generate_task_id();
+        let task_id2 = ResumeUtils::generate_task_id();
+
+        assert_ne!(task_id1, task_id2);
+        assert_eq!(task_id1.len(), 36); // UUID length
+        assert_eq!(task_id2.len(), 36);
+    }
+
+    #[tokio::test]
+    async fn test_get_resumable_tasks_integration() {
+        use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
+        use tempfile::TempDir;
+
+        // Create a temporary directory for testing
+        let temp_dir = TempDir::new().unwrap();
+        let disk_path = temp_dir.path().join("test_disk");
+        std::fs::create_dir_all(&disk_path).unwrap();
+
+        // Create a local disk for testing
+        let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap();
+        let disk_option = DiskOption {
+            cleanup: false,
+            health_check: false,
+        };
+        let disk = new_disk(&endpoint, &disk_option).await.unwrap();
+
+        // Create necessary directories first (ignore if already exist)
+        let _ = disk.make_volume(RUSTFS_META_BUCKET).await;
+        let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await;
+
+        // Create some test resume state files
+        let task_ids = vec![
+            "test-task-1".to_string(),
+            "test-task-2".to_string(),
+            "test-task-3".to_string(),
+        ];
+
+        // Save resume state files for each task
+        for task_id in &task_ids {
+            let state = ResumeState::new(
+                task_id.clone(),
+                "erasure_set".to_string(),
+                vec!["bucket1".to_string(), "bucket2".to_string()],
+            );
+
+            let state_data = serde_json::to_vec(&state).unwrap();
+            let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}");
+
+            disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into())
+                .await
+                .unwrap();
+        }
+
+        // Also create some non-resume state files to test filtering
+        let non_resume_files = vec![
+            "other_file.txt",
+            "task4_ahm_checkpoint.json",
+            "task5_ahm_progress.json",
+            "_ahm_resume_state.json", // Invalid: empty task ID
+        ];
+
+        for file_name in non_resume_files {
+            let file_path = format!("{BUCKET_META_PREFIX}/{file_name}");
+            disk.write_all(RUSTFS_META_BUCKET, &file_path, b"test data".to_vec().into())
+                .await
+                .unwrap();
+        }
+
+        // Now call get_resumable_tasks to see if it finds the correct files
+        let found_task_ids = ResumeUtils::get_resumable_tasks(&disk).await.unwrap();
+
+        // Verify that only the valid resume state files are found
+        assert_eq!(found_task_ids.len(), 3);
+        for task_id in &task_ids {
+            assert!(found_task_ids.contains(task_id), "Task ID {task_id} not found");
+        }
+
+        // Verify that invalid files are not included
+        assert!(!found_task_ids.contains(&"".to_string()));
+        assert!(!found_task_ids.contains(&"task4".to_string()));
+        assert!(!found_task_ids.contains(&"task5".to_string()));
+
+        // Clean up
+        temp_dir.close().unwrap();
+    }
+}

crates/ahm/src/heal/storage.rs

@@ -0,0 +1,506 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use async_trait::async_trait;
+use rustfs_common::heal_channel::{HealOpts, HealScanMode};
+use rustfs_ecstore::{
+    disk::{DiskStore, endpoint::Endpoint},
+    store::ECStore,
+    store_api::{BucketInfo, ObjectIO, StorageAPI},
+};
+use rustfs_madmin::heal_commands::HealResultItem;
+use std::sync::Arc;
+use tracing::{debug, error, info, warn};
+
+/// Disk status for heal operations
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DiskStatus {
+    /// Ok
+    Ok,
+    /// Offline
+    Offline,
+    /// Corrupt
+    Corrupt,
+    /// Missing
+    Missing,
+    /// Permission denied
+    PermissionDenied,
+    /// Faulty
+    Faulty,
+    /// Root mount
+    RootMount,
+    /// Unknown
+    Unknown,
+    /// Unformatted
+    Unformatted,
+}
+
+/// Heal storage layer interface
+#[async_trait]
+pub trait HealStorageAPI: Send + Sync {
+    /// Get object meta
+    async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>>;
+
+    /// Get object data
+    async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>>;
+
+    /// Put object data
+    async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>;
+
+    /// Delete object
+    async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>;
+
+    /// Check object integrity
+    async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool>;
+
+    /// EC decode rebuild
+    async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result<Vec<u8>>;
+
+    /// Get disk status
+    async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus>;
+
+    /// Format disk
+    async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>;
+
+    /// Get bucket info
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
+
+    /// Fix bucket metadata
+    async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
+
+    /// Get all buckets
+    async fn list_buckets(&self) -> Result<Vec<BucketInfo>>;
+
+    /// Check object exists
+    async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
+
+    /// Get object size
+    async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>>;
+
+    /// Get object checksum
+    async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>>;
+
+    /// Heal object using ecstore
+    async fn heal_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: Option<&str>,
+        opts: &HealOpts,
+    ) -> Result<(HealResultItem, Option<Error>)>;
+
+    /// Heal bucket using ecstore
+    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
+
+    /// Heal format using ecstore
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)>;
+
+    /// List objects for healing
+    async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<String>>;
+
+    /// Get disk for resume functionality
+    async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result<DiskStore>;
+}
+
+/// ECStore Heal storage layer implementation
+pub struct ECStoreHealStorage {
+    ecstore: Arc<ECStore>,
+}
+
+impl ECStoreHealStorage {
+    pub fn new(ecstore: Arc<ECStore>) -> Self {
+        Self { ecstore }
+    }
+}
+
+#[async_trait]
+impl HealStorageAPI for ECStoreHealStorage {
+    async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
+        debug!("Getting object meta: {}/{}", bucket, object);
+
+        match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
+            Ok(info) => Ok(Some(info)),
+            Err(e) => {
+                error!("Failed to get object meta: {}/{} - {}", bucket, object, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>> {
+        debug!("Getting object data: {}/{}", bucket, object);
+
+        match (*self.ecstore)
+            .get_object_reader(bucket, object, None, Default::default(), &Default::default())
+            .await
+        {
+            Ok(mut reader) => match reader.read_all().await {
+                Ok(data) => Ok(Some(data)),
+                Err(e) => {
+                    error!("Failed to read object data: {}/{} - {}", bucket, object, e);
+                    Err(Error::other(e))
+                }
+            },
+            Err(e) => {
+                error!("Failed to get object: {}/{} - {}", bucket, object, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
+        debug!("Putting object data: {}/{} ({} bytes)", bucket, object, data.len());
+
+        let mut reader = rustfs_ecstore::store_api::PutObjReader::from_vec(data.to_vec());
+        match (*self.ecstore)
+            .put_object(bucket, object, &mut reader, &Default::default())
+            .await
+        {
+            Ok(_) => {
+                info!("Successfully put object: {}/{}", bucket, object);
+                Ok(())
+            }
+            Err(e) => {
+                error!("Failed to put object: {}/{} - {}", bucket, object, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> {
+        debug!("Deleting object: {}/{}", bucket, object);
+
+        match self.ecstore.delete_object(bucket, object, Default::default()).await {
+            Ok(_) => {
+                info!("Successfully deleted object: {}/{}", bucket, object);
+                Ok(())
+            }
+            Err(e) => {
+                error!("Failed to delete object: {}/{} - {}", bucket, object, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool> {
+        debug!("Verifying object integrity: {}/{}", bucket, object);
+
+        // Try to get object info and data to verify integrity
+        match self.get_object_meta(bucket, object).await? {
+            Some(obj_info) => {
+                // Check if object has valid metadata
+                if obj_info.size < 0 {
+                    warn!("Object has invalid size: {}/{}", bucket, object);
+                    return Ok(false);
+                }
+
+                // Try to read object data to verify it's accessible
+                match self.get_object_data(bucket, object).await {
+                    Ok(Some(_)) => {
+                        info!("Object integrity check passed: {}/{}", bucket, object);
+                        Ok(true)
+                    }
+                    Ok(None) => {
+                        warn!("Object data not found: {}/{}", bucket, object);
+                        Ok(false)
+                    }
+                    Err(_) => {
+                        warn!("Object data read failed: {}/{}", bucket, object);
+                        Ok(false)
+                    }
+                }
+            }
+            None => {
+                warn!("Object metadata not found: {}/{}", bucket, object);
+                Ok(false)
+            }
+        }
+    }
+
+    async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result<Vec<u8>> {
+        debug!("EC decode rebuild: {}/{}", bucket, object);
+
+        // Use ecstore's heal_object to rebuild the object
+        let heal_opts = HealOpts {
+            recursive: false,
+            dry_run: false,
+            remove: false,
+            recreate: true,
+            scan_mode: HealScanMode::Deep,
+            update_parity: true,
+            no_lock: false,
+            pool: None,
+            set: None,
+        };
+
+        match self.heal_object(bucket, object, None, &heal_opts).await {
+            Ok((_result, error)) => {
+                if error.is_some() {
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Heal failed: {error:?}"),
+                    });
+                }
+
+                // After healing, try to read the object data
+                match self.get_object_data(bucket, object).await? {
+                    Some(data) => {
+                        info!("EC decode rebuild successful: {}/{} ({} bytes)", bucket, object, data.len());
+                        Ok(data)
+                    }
+                    None => {
+                        error!("Object not found after heal: {}/{}", bucket, object);
+                        Err(Error::TaskExecutionFailed {
+                            message: format!("Object not found after heal: {bucket}/{object}"),
+                        })
+                    }
+                }
+            }
+            Err(e) => {
+                error!("Heal operation failed: {}/{} - {}", bucket, object, e);
+                Err(e)
+            }
+        }
+    }
+
+    async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus> {
+        debug!("Getting disk status: {:?}", endpoint);
+
+        // TODO: implement disk status check using ecstore
+        // For now, return Ok status
+        info!("Disk status check: {:?} - OK", endpoint);
+        Ok(DiskStatus::Ok)
+    }
+
+    async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> {
+        debug!("Formatting disk: {:?}", endpoint);
+
+        // Use ecstore's heal_format
+        match self.heal_format(false).await {
+            Ok((_, error)) => {
+                if error.is_some() {
+                    return Err(Error::other(format!("Format failed: {error:?}")));
+                }
+                info!("Successfully formatted disk: {:?}", endpoint);
+                Ok(())
+            }
+            Err(e) => {
+                error!("Failed to format disk: {:?} - {}", endpoint, e);
+                Err(e)
+            }
+        }
+    }
+
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+        debug!("Getting bucket info: {}", bucket);
+
+        match self.ecstore.get_bucket_info(bucket, &Default::default()).await {
+            Ok(info) => Ok(Some(info)),
+            Err(e) => {
+                error!("Failed to get bucket info: {} - {}", bucket, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
+        debug!("Healing bucket metadata: {}", bucket);
+
+        let heal_opts = HealOpts {
+            recursive: true,
+            dry_run: false,
+            remove: false,
+            recreate: false,
+            scan_mode: HealScanMode::Normal,
+            update_parity: false,
+            no_lock: false,
+            pool: None,
+            set: None,
+        };
+
+        match self.heal_bucket(bucket, &heal_opts).await {
+            Ok(_) => {
+                info!("Successfully healed bucket metadata: {}", bucket);
+                Ok(())
+            }
+            Err(e) => {
+                error!("Failed to heal bucket metadata: {} - {}", bucket, e);
+                Err(e)
+            }
+        }
+    }
+
+    async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
+        debug!("Listing buckets");
+
+        match self.ecstore.list_bucket(&Default::default()).await {
+            Ok(buckets) => Ok(buckets),
+            Err(e) => {
+                error!("Failed to list buckets: {}", e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool> {
+        debug!("Checking object exists: {}/{}", bucket, object);
+
+        match self.get_object_meta(bucket, object).await {
+            Ok(Some(_)) => Ok(true),
+            Ok(None) => Ok(false),
+            Err(_) => Ok(false),
+        }
+    }
+
+    async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>> {
+        debug!("Getting object size: {}/{}", bucket, object);
+
+        match self.get_object_meta(bucket, object).await {
+            Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)),
+            Ok(None) => Ok(None),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>> {
+        debug!("Getting object checksum: {}/{}", bucket, object);
+
+        match self.get_object_meta(bucket, object).await {
+            Ok(Some(obj_info)) => {
+                // Convert checksum bytes to hex string
+                let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::<String>();
+                Ok(Some(checksum))
+            }
+            Ok(None) => Ok(None),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn heal_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: Option<&str>,
+        opts: &HealOpts,
+    ) -> Result<(HealResultItem, Option<Error>)> {
+        debug!("Healing object: {}/{}", bucket, object);
+
+        let version_id_str = version_id.unwrap_or("");
+
+        match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
+            Ok((result, ecstore_error)) => {
+                let error = ecstore_error.map(Error::other);
+                info!("Heal object completed: {}/{} - result: {:?}, error: {:?}", bucket, object, result, error);
+                Ok((result, error))
+            }
+            Err(e) => {
+                error!("Heal object failed: {}/{} - {}", bucket, object, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+        debug!("Healing bucket: {}", bucket);
+
+        match self.ecstore.heal_bucket(bucket, opts).await {
+            Ok(result) => {
+                info!("Heal bucket completed: {} - result: {:?}", bucket, result);
+                Ok(result)
+            }
+            Err(e) => {
+                error!("Heal bucket failed: {} - {}", bucket, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        debug!("Healing format (dry_run: {})", dry_run);
+
+        match self.ecstore.heal_format(dry_run).await {
+            Ok((result, ecstore_error)) => {
+                let error = ecstore_error.map(Error::other);
+                info!("Heal format completed - result: {:?}, error: {:?}", result, error);
+                Ok((result, error))
+            }
+            Err(e) => {
+                error!("Heal format failed: {}", e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<String>> {
+        debug!("Listing objects for heal: {}/{}", bucket, prefix);
+
+        // Use list_objects_v2 to get objects
+        match self
+            .ecstore
+            .clone()
+            .list_objects_v2(bucket, prefix, None, None, 1000, false, None)
+            .await
+        {
+            Ok(list_info) => {
+                let objects: Vec<String> = list_info.objects.into_iter().map(|obj| obj.name).collect();
+                info!("Found {} objects for heal in {}/{}", objects.len(), bucket, prefix);
+                Ok(objects)
+            }
+            Err(e) => {
+                error!("Failed to list objects for heal: {}/{} - {}", bucket, prefix, e);
+                Err(Error::other(e))
+            }
+        }
+    }
+
+    async fn get_disk_for_resume(&self, set_disk_id: &str) -> Result<DiskStore> {
+        debug!("Getting disk for resume: {}", set_disk_id);
+
+        // Parse set_disk_id to extract pool and set indices
+        // Format: "pool_{pool_idx}_set_{set_idx}"
+        let parts: Vec<&str> = set_disk_id.split('_').collect();
+        if parts.len() != 4 || parts[0] != "pool" || parts[2] != "set" {
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Invalid set_disk_id format: {set_disk_id}"),
+            });
+        }
+
+        let pool_idx: usize = parts[1].parse().map_err(|_| Error::TaskExecutionFailed {
+            message: format!("Invalid pool index in set_disk_id: {set_disk_id}"),
+        })?;
+
+        let set_idx: usize = parts[3].parse().map_err(|_| Error::TaskExecutionFailed {
+            message: format!("Invalid set index in set_disk_id: {set_disk_id}"),
+        })?;
+
+        // Get the first available disk from the set
+        let disks = self
+            .ecstore
+            .get_disks(pool_idx, set_idx)
+            .await
+            .map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to get disks for pool {pool_idx} set {set_idx}: {e}"),
+            })?;
+
+        // Find the first available disk
+        if let Some(disk_store) = disks.into_iter().flatten().next() {
+            info!("Found disk for resume: {:?}", disk_store);
+            return Ok(disk_store);
+        }
+
+        Err(Error::TaskExecutionFailed {
+            message: format!("No available disk found for set_disk_id: {set_disk_id}"),
+        })
+    }
+}

crates/ahm/src/heal/task.rs

@@ -0,0 +1,855 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::heal::ErasureSetHealer;
+use crate::heal::{progress::HealProgress, storage::HealStorageAPI};
+use rustfs_common::heal_channel::{HealOpts, HealScanMode};
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use std::time::{Duration, SystemTime};
+use tokio::sync::RwLock;
+use tracing::{error, info, warn};
+use uuid::Uuid;
+
+/// Heal type
+#[derive(Debug, Clone)]
+pub enum HealType {
+    /// Object heal
+    Object {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+    },
+    /// Bucket heal
+    Bucket { bucket: String },
+    /// Erasure Set heal (includes disk format repair)
+    ErasureSet { buckets: Vec<String>, set_disk_id: String },
+    /// Metadata heal
+    Metadata { bucket: String, object: String },
+    /// MRF heal
+    MRF { meta_path: String },
+    /// EC decode heal
+    ECDecode {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+    },
+}
+
+/// Heal priority
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub enum HealPriority {
+    /// Low priority
+    Low = 0,
+    /// Normal priority
+    Normal = 1,
+    /// High priority
+    High = 2,
+    /// Urgent priority
+    Urgent = 3,
+}
+
+impl Default for HealPriority {
+    fn default() -> Self {
+        Self::Normal
+    }
+}
+
+/// Heal options
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HealOptions {
+    /// Scan mode
+    pub scan_mode: HealScanMode,
+    /// Whether to remove corrupted data
+    pub remove_corrupted: bool,
+    /// Whether to recreate
+    pub recreate_missing: bool,
+    /// Whether to update parity
+    pub update_parity: bool,
+    /// Whether to recursively process
+    pub recursive: bool,
+    /// Whether to dry run
+    pub dry_run: bool,
+    /// Timeout
+    pub timeout: Option<Duration>,
+    /// pool index
+    pub pool_index: Option<usize>,
+    /// set index
+    pub set_index: Option<usize>,
+}
+
+impl Default for HealOptions {
+    fn default() -> Self {
+        Self {
+            scan_mode: HealScanMode::Normal,
+            remove_corrupted: false,
+            recreate_missing: true,
+            update_parity: true,
+            recursive: false,
+            dry_run: false,
+            timeout: Some(Duration::from_secs(300)), // 5 minutes default timeout
+            pool_index: None,
+            set_index: None,
+        }
+    }
+}
+
+/// Heal task status
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub enum HealTaskStatus {
+    /// Pending
+    Pending,
+    /// Running
+    Running,
+    /// Completed
+    Completed,
+    /// Failed
+    Failed { error: String },
+    /// Cancelled
+    Cancelled,
+    /// Timeout
+    Timeout,
+}
+
+/// Heal request
+#[derive(Debug, Clone)]
+pub struct HealRequest {
+    /// Request ID
+    pub id: String,
+    /// Heal type
+    pub heal_type: HealType,
+    /// Heal options
+    pub options: HealOptions,
+    /// Priority
+    pub priority: HealPriority,
+    /// Created time
+    pub created_at: SystemTime,
+}
+
+impl HealRequest {
+    pub fn new(heal_type: HealType, options: HealOptions, priority: HealPriority) -> Self {
+        Self {
+            id: Uuid::new_v4().to_string(),
+            heal_type,
+            options,
+            priority,
+            created_at: SystemTime::now(),
+        }
+    }
+
+    pub fn object(bucket: String, object: String, version_id: Option<String>) -> Self {
+        Self::new(
+            HealType::Object {
+                bucket,
+                object,
+                version_id,
+            },
+            HealOptions::default(),
+            HealPriority::Normal,
+        )
+    }
+
+    pub fn bucket(bucket: String) -> Self {
+        Self::new(HealType::Bucket { bucket }, HealOptions::default(), HealPriority::Normal)
+    }
+
+    pub fn metadata(bucket: String, object: String) -> Self {
+        Self::new(HealType::Metadata { bucket, object }, HealOptions::default(), HealPriority::High)
+    }
+
+    pub fn ec_decode(bucket: String, object: String, version_id: Option<String>) -> Self {
+        Self::new(
+            HealType::ECDecode {
+                bucket,
+                object,
+                version_id,
+            },
+            HealOptions::default(),
+            HealPriority::Urgent,
+        )
+    }
+}
+
+/// Heal task
+pub struct HealTask {
+    /// Task ID
+    pub id: String,
+    /// Heal type
+    pub heal_type: HealType,
+    /// Heal options
+    pub options: HealOptions,
+    /// Task status
+    pub status: Arc<RwLock<HealTaskStatus>>,
+    /// Progress tracking
+    pub progress: Arc<RwLock<HealProgress>>,
+    /// Created time
+    pub created_at: SystemTime,
+    /// Started time
+    pub started_at: Arc<RwLock<Option<SystemTime>>>,
+    /// Completed time
+    pub completed_at: Arc<RwLock<Option<SystemTime>>>,
+    /// Cancel token
+    pub cancel_token: tokio_util::sync::CancellationToken,
+    /// Storage layer interface
+    pub storage: Arc<dyn HealStorageAPI>,
+}
+
+impl HealTask {
+    pub fn from_request(request: HealRequest, storage: Arc<dyn HealStorageAPI>) -> Self {
+        Self {
+            id: request.id,
+            heal_type: request.heal_type,
+            options: request.options,
+            status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
+            progress: Arc::new(RwLock::new(HealProgress::new())),
+            created_at: request.created_at,
+            started_at: Arc::new(RwLock::new(None)),
+            completed_at: Arc::new(RwLock::new(None)),
+            cancel_token: tokio_util::sync::CancellationToken::new(),
+            storage,
+        }
+    }
+
+    pub async fn execute(&self) -> Result<()> {
+        // update status to running
+        {
+            let mut status = self.status.write().await;
+            *status = HealTaskStatus::Running;
+        }
+        {
+            let mut started_at = self.started_at.write().await;
+            *started_at = Some(SystemTime::now());
+        }
+
+        info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type);
+
+        let result = match &self.heal_type {
+            HealType::Object {
+                bucket,
+                object,
+                version_id,
+            } => self.heal_object(bucket, object, version_id.as_deref()).await,
+            HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
+
+            HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
+            HealType::MRF { meta_path } => self.heal_mrf(meta_path).await,
+            HealType::ECDecode {
+                bucket,
+                object,
+                version_id,
+            } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
+            HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
+        };
+
+        // update completed time and status
+        {
+            let mut completed_at = self.completed_at.write().await;
+            *completed_at = Some(SystemTime::now());
+        }
+
+        match &result {
+            Ok(_) => {
+                let mut status = self.status.write().await;
+                *status = HealTaskStatus::Completed;
+                info!("Heal task completed successfully: {}", self.id);
+            }
+            Err(e) => {
+                let mut status = self.status.write().await;
+                *status = HealTaskStatus::Failed { error: e.to_string() };
+                error!("Heal task failed: {} with error: {}", self.id, e);
+            }
+        }
+
+        result
+    }
+
+    pub async fn cancel(&self) -> Result<()> {
+        self.cancel_token.cancel();
+        let mut status = self.status.write().await;
+        *status = HealTaskStatus::Cancelled;
+        info!("Heal task cancelled: {}", self.id);
+        Ok(())
+    }
+
+    pub async fn get_status(&self) -> HealTaskStatus {
+        self.status.read().await.clone()
+    }
+
+    pub async fn get_progress(&self) -> HealProgress {
+        self.progress.read().await.clone()
+    }
+
+    // specific heal implementation method
+    async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
+        info!("Healing object: {}/{}", bucket, object);
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("{bucket}/{object}")));
+            progress.update_progress(0, 4, 0, 0); // 开始heal，总共4个步骤
+        }
+
+        // Step 1: Check if object exists and get metadata
+        info!("Step 1: Checking object existence and metadata");
+        let object_exists = self.storage.object_exists(bucket, object).await?;
+        if !object_exists {
+            warn!("Object does not exist: {}/{}", bucket, object);
+            if self.options.recreate_missing {
+                info!("Attempting to recreate missing object: {}/{}", bucket, object);
+                return self.recreate_missing_object(bucket, object, version_id).await;
+            } else {
+                return Err(Error::TaskExecutionFailed {
+                    message: format!("Object not found: {bucket}/{object}"),
+                });
+            }
+        }
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(1, 3, 0, 0);
+        }
+
+        // Step 2: directly call ecstore to perform heal
+        info!("Step 2: Performing heal using ecstore");
+        let heal_opts = HealOpts {
+            recursive: self.options.recursive,
+            dry_run: self.options.dry_run,
+            remove: self.options.remove_corrupted,
+            recreate: self.options.recreate_missing,
+            scan_mode: self.options.scan_mode,
+            update_parity: self.options.update_parity,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+
+        match self.storage.heal_object(bucket, object, version_id, &heal_opts).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("Heal operation failed: {}/{} - {}", bucket, object, e);
+
+                    // If heal failed and remove_corrupted is enabled, delete the corrupted object
+                    if self.options.remove_corrupted {
+                        warn!("Removing corrupted object: {}/{}", bucket, object);
+                        if !self.options.dry_run {
+                            self.storage.delete_object(bucket, object).await?;
+                            info!("Successfully deleted corrupted object: {}/{}", bucket, object);
+                        } else {
+                            info!("Dry run mode - would delete corrupted object: {}/{}", bucket, object);
+                        }
+                    }
+
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.update_progress(3, 3, 0, 0);
+                    }
+
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to heal object {bucket}/{object}: {e}"),
+                    });
+                }
+
+                // Step 3: Verify heal result
+                info!("Step 3: Verifying heal result");
+                let object_size = result.object_size as u64;
+                info!(
+                    "Heal completed successfully: {}/{} ({} bytes, {} drives healed)",
+                    bucket,
+                    object,
+                    object_size,
+                    result.after.drives.len()
+                );
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, object_size, object_size);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("Heal operation failed: {}/{} - {}", bucket, object, e);
+
+                // If heal failed and remove_corrupted is enabled, delete the corrupted object
+                if self.options.remove_corrupted {
+                    warn!("Removing corrupted object: {}/{}", bucket, object);
+                    if !self.options.dry_run {
+                        self.storage.delete_object(bucket, object).await?;
+                        info!("Successfully deleted corrupted object: {}/{}", bucket, object);
+                    } else {
+                        info!("Dry run mode - would delete corrupted object: {}/{}", bucket, object);
+                    }
+                }
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal object {bucket}/{object}: {e}"),
+                })
+            }
+        }
+    }
+
+    /// Recreate missing object (for EC decode scenarios)
+    async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
+        info!("Attempting to recreate missing object: {}/{}", bucket, object);
+
+        // Use ecstore's heal_object with recreate option
+        let heal_opts = HealOpts {
+            recursive: false,
+            dry_run: self.options.dry_run,
+            remove: false,
+            recreate: true,
+            scan_mode: HealScanMode::Deep,
+            update_parity: true,
+            no_lock: false,
+            pool: None,
+            set: None,
+        };
+
+        match self.storage.heal_object(bucket, object, version_id, &heal_opts).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e);
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
+                    });
+                }
+
+                let object_size = result.object_size as u64;
+                info!("Successfully recreated missing object: {}/{} ({} bytes)", bucket, object, object_size);
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(4, 4, object_size, object_size);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e);
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
+                })
+            }
+        }
+    }
+
+    async fn heal_bucket(&self, bucket: &str) -> Result<()> {
+        info!("Healing bucket: {}", bucket);
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("bucket: {bucket}")));
+            progress.update_progress(0, 3, 0, 0);
+        }
+
+        // Step 1: Check if bucket exists
+        info!("Step 1: Checking bucket existence");
+        let bucket_exists = self.storage.get_bucket_info(bucket).await?.is_some();
+        if !bucket_exists {
+            warn!("Bucket does not exist: {}", bucket);
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Bucket not found: {bucket}"),
+            });
+        }
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(1, 3, 0, 0);
+        }
+
+        // Step 2: Perform bucket heal using ecstore
+        info!("Step 2: Performing bucket heal using ecstore");
+        let heal_opts = HealOpts {
+            recursive: self.options.recursive,
+            dry_run: self.options.dry_run,
+            remove: self.options.remove_corrupted,
+            recreate: self.options.recreate_missing,
+            scan_mode: self.options.scan_mode,
+            update_parity: self.options.update_parity,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+
+        match self.storage.heal_bucket(bucket, &heal_opts).await {
+            Ok(result) => {
+                info!("Bucket heal completed successfully: {} ({} drives)", bucket, result.after.drives.len());
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("Bucket heal failed: {} - {}", bucket, e);
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal bucket {bucket}: {e}"),
+                })
+            }
+        }
+    }
+
+    async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
+        info!("Healing metadata: {}/{}", bucket, object);
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
+            progress.update_progress(0, 3, 0, 0);
+        }
+
+        // Step 1: Check if object exists
+        info!("Step 1: Checking object existence");
+        let object_exists = self.storage.object_exists(bucket, object).await?;
+        if !object_exists {
+            warn!("Object does not exist: {}/{}", bucket, object);
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Object not found: {bucket}/{object}"),
+            });
+        }
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(1, 3, 0, 0);
+        }
+
+        // Step 2: Perform metadata heal using ecstore
+        info!("Step 2: Performing metadata heal using ecstore");
+        let heal_opts = HealOpts {
+            recursive: false,
+            dry_run: self.options.dry_run,
+            remove: false,
+            recreate: false,
+            scan_mode: HealScanMode::Deep,
+            update_parity: false,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+
+        match self.storage.heal_object(bucket, object, None, &heal_opts).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("Metadata heal failed: {}/{} - {}", bucket, object, e);
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.update_progress(3, 3, 0, 0);
+                    }
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
+                    });
+                }
+
+                info!(
+                    "Metadata heal completed successfully: {}/{} ({} drives)",
+                    bucket,
+                    object,
+                    result.after.drives.len()
+                );
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("Metadata heal failed: {}/{} - {}", bucket, object, e);
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
+                })
+            }
+        }
+    }
+
+    async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
+        info!("Healing MRF: {}", meta_path);
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("mrf: {meta_path}")));
+            progress.update_progress(0, 2, 0, 0);
+        }
+
+        // Parse meta_path to extract bucket and object
+        let parts: Vec<&str> = meta_path.split('/').collect();
+        if parts.len() < 2 {
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Invalid meta path format: {meta_path}"),
+            });
+        }
+
+        let bucket = parts[0];
+        let object = parts[1..].join("/");
+
+        // Step 1: Perform MRF heal using ecstore
+        info!("Step 1: Performing MRF heal using ecstore");
+        let heal_opts = HealOpts {
+            recursive: true,
+            dry_run: self.options.dry_run,
+            remove: self.options.remove_corrupted,
+            recreate: self.options.recreate_missing,
+            scan_mode: HealScanMode::Deep,
+            update_parity: true,
+            no_lock: false,
+            pool: None,
+            set: None,
+        };
+
+        match self.storage.heal_object(bucket, &object, None, &heal_opts).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("MRF heal failed: {} - {}", meta_path, e);
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.update_progress(2, 2, 0, 0);
+                    }
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to heal MRF {meta_path}: {e}"),
+                    });
+                }
+
+                info!("MRF heal completed successfully: {} ({} drives)", meta_path, result.after.drives.len());
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(2, 2, 0, 0);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("MRF heal failed: {} - {}", meta_path, e);
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(2, 2, 0, 0);
+                }
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal MRF {meta_path}: {e}"),
+                })
+            }
+        }
+    }
+
+    async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
+        info!("Healing EC decode: {}/{}", bucket, object);
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
+            progress.update_progress(0, 3, 0, 0);
+        }
+
+        // Step 1: Check if object exists
+        info!("Step 1: Checking object existence");
+        let object_exists = self.storage.object_exists(bucket, object).await?;
+        if !object_exists {
+            warn!("Object does not exist: {}/{}", bucket, object);
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Object not found: {bucket}/{object}"),
+            });
+        }
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(1, 3, 0, 0);
+        }
+
+        // Step 2: Perform EC decode heal using ecstore
+        info!("Step 2: Performing EC decode heal using ecstore");
+        let heal_opts = HealOpts {
+            recursive: false,
+            dry_run: self.options.dry_run,
+            remove: false,
+            recreate: true,
+            scan_mode: HealScanMode::Deep,
+            update_parity: true,
+            no_lock: false,
+            pool: None,
+            set: None,
+        };
+
+        match self.storage.heal_object(bucket, object, version_id, &heal_opts).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("EC decode heal failed: {}/{} - {}", bucket, object, e);
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.update_progress(3, 3, 0, 0);
+                    }
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
+                    });
+                }
+
+                let object_size = result.object_size as u64;
+                info!(
+                    "EC decode heal completed successfully: {}/{} ({} bytes, {} drives)",
+                    bucket,
+                    object,
+                    object_size,
+                    result.after.drives.len()
+                );
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, object_size, object_size);
+                }
+                Ok(())
+            }
+            Err(e) => {
+                error!("EC decode heal failed: {}/{} - {}", bucket, object, e);
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(3, 3, 0, 0);
+                }
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
+                })
+            }
+        }
+    }
+
+    async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
+        info!("Healing Erasure Set: {} ({} buckets)", set_disk_id, buckets.len());
+
+        // update progress
+        {
+            let mut progress = self.progress.write().await;
+            progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
+            progress.update_progress(0, 4, 0, 0);
+        }
+
+        let buckets = if buckets.is_empty() {
+            info!("No buckets specified, listing all buckets");
+            let bucket_infos = self.storage.list_buckets().await?;
+            bucket_infos.into_iter().map(|info| info.name).collect()
+        } else {
+            buckets
+        };
+
+        // Step 1: Perform disk format heal using ecstore
+        info!("Step 1: Performing disk format heal using ecstore");
+        match self.storage.heal_format(self.options.dry_run).await {
+            Ok((result, error)) => {
+                if let Some(e) = error {
+                    error!("Disk format heal failed: {} - {}", set_disk_id, e);
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.update_progress(4, 4, 0, 0);
+                    }
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
+                    });
+                }
+
+                info!(
+                    "Disk format heal completed successfully: {} ({} drives)",
+                    set_disk_id,
+                    result.after.drives.len()
+                );
+            }
+            Err(e) => {
+                error!("Disk format heal failed: {} - {}", set_disk_id, e);
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(4, 4, 0, 0);
+                }
+                return Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
+                });
+            }
+        }
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(1, 4, 0, 0);
+        }
+
+        // Step 2: Get disk for resume functionality
+        info!("Step 2: Getting disk for resume functionality");
+        let disk = self.storage.get_disk_for_resume(&set_disk_id).await?;
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(2, 4, 0, 0);
+        }
+
+        // Step 3: Heal bucket structure
+        for bucket in buckets.iter() {
+            if let Err(err) = self.heal_bucket(bucket).await {
+                info!("{}", err.to_string());
+            }
+        }
+
+        // Step 3: Create erasure set healer with resume support
+        info!("Step 3: Creating erasure set healer with resume support");
+        let erasure_healer = ErasureSetHealer::new(self.storage.clone(), self.progress.clone(), self.cancel_token.clone(), disk);
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(3, 4, 0, 0);
+        }
+
+        // Step 4: Execute erasure set heal with resume
+        info!("Step 4: Executing erasure set heal with resume");
+        let result = erasure_healer.heal_erasure_set(&buckets, &set_disk_id).await;
+
+        {
+            let mut progress = self.progress.write().await;
+            progress.update_progress(4, 4, 0, 0);
+        }
+
+        match result {
+            Ok(_) => {
+                info!("Erasure set heal completed successfully: {} ({} buckets)", set_disk_id, buckets.len());
+                Ok(())
+            }
+            Err(e) => {
+                error!("Erasure set heal failed: {} - {}", set_disk_id, e);
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal erasure set {set_disk_id}: {e}"),
+                })
+            }
+        }
+    }
+}
+
+impl std::fmt::Debug for HealTask {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HealTask")
+            .field("id", &self.id)
+            .field("heal_type", &self.heal_type)
+            .field("options", &self.options)
+            .field("created_at", &self.created_at)
+            .finish()
+    }
+}

crates/ahm/src/lib.rs

@@ -12,17 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 use tokio_util::sync::CancellationToken;
+use tracing::{error, info};
 
 pub mod error;
+pub mod heal;
 pub mod scanner;
 
 pub use error::{Error, Result};
-pub use scanner::{
-    BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, Scanner, ScannerMetrics, load_data_usage_from_backend,
-    store_data_usage_in_backend,
-};
+pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
+pub use scanner::Scanner;
 
 // Global cancellation token for AHM services (scanner and other background tasks)
 static GLOBAL_AHM_SERVICES_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();
@@ -52,3 +52,61 @@ pub fn shutdown_ahm_services() {
         cancel_token.cancel();
     }
 }
+
+/// Global heal manager instance
+static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
+
+/// Global heal channel processor instance
+static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
+
+/// Initialize and start heal manager with channel processor
+pub async fn init_heal_manager(
+    storage: Arc<dyn heal::storage::HealStorageAPI>,
+    config: Option<heal::manager::HealConfig>,
+) -> Result<Arc<HealManager>> {
+    // Create heal manager
+    let heal_manager = Arc::new(HealManager::new(storage, config));
+
+    // Start heal manager
+    heal_manager.start().await?;
+
+    // Store global instance
+    GLOBAL_HEAL_MANAGER
+        .set(heal_manager.clone())
+        .map_err(|_| Error::Config("Heal manager already initialized".to_string()))?;
+
+    // Initialize heal channel
+    let channel_receiver = rustfs_common::heal_channel::init_heal_channel();
+
+    // Create channel processor
+    let channel_processor = HealChannelProcessor::new(heal_manager.clone());
+
+    // Store channel processor instance first
+    GLOBAL_HEAL_CHANNEL_PROCESSOR
+        .set(Arc::new(tokio::sync::Mutex::new(channel_processor)))
+        .map_err(|_| Error::Config("Heal channel processor already initialized".to_string()))?;
+
+    // Start channel processor in background
+    let receiver = channel_receiver;
+    tokio::spawn(async move {
+        if let Some(processor_guard) = GLOBAL_HEAL_CHANNEL_PROCESSOR.get() {
+            let mut processor = processor_guard.lock().await;
+            if let Err(e) = processor.start(receiver).await {
+                error!("Heal channel processor failed: {}", e);
+            }
+        }
+    });
+
+    info!("Heal manager with channel processor initialized successfully");
+    Ok(heal_manager)
+}
+
+/// Get global heal manager instance
+pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
+    GLOBAL_HEAL_MANAGER.get()
+}
+
+/// Get global heal channel processor instance
+pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
+    GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
+}

crates/ahm/src/scanner/data_usage.rs

@@ -1,671 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::{collections::HashMap, sync::Arc, time::SystemTime};
-
-use rustfs_ecstore::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore};
-use rustfs_utils::path::SLASH_SEPARATOR;
-use serde::{Deserialize, Serialize};
-use tracing::{error, info, warn};
-
-use crate::error::{Error, Result};
-
-// Data usage storage constants
-pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
-const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
-const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
-pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
-
-// Data usage storage paths
-lazy_static::lazy_static! {
-    pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
-        rustfs_ecstore::disk::RUSTFS_META_BUCKET,
-        SLASH_SEPARATOR,
-        rustfs_ecstore::disk::BUCKET_META_PREFIX
-    );
-    pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
-        rustfs_ecstore::disk::BUCKET_META_PREFIX,
-        SLASH_SEPARATOR,
-        DATA_USAGE_OBJ_NAME
-    );
-    pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
-        rustfs_ecstore::disk::BUCKET_META_PREFIX,
-        SLASH_SEPARATOR,
-        DATA_USAGE_BLOOM_NAME
-    );
-}
-
-/// Bucket target usage info provides replication statistics
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct BucketTargetUsageInfo {
-    pub replication_pending_size: u64,
-    pub replication_failed_size: u64,
-    pub replicated_size: u64,
-    pub replica_size: u64,
-    pub replication_pending_count: u64,
-    pub replication_failed_count: u64,
-    pub replicated_count: u64,
-}
-
-/// Bucket usage info provides bucket-level statistics
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct BucketUsageInfo {
-    pub size: u64,
-    // Following five fields suffixed with V1 are here for backward compatibility
-    // Total Size for objects that have not yet been replicated
-    pub replication_pending_size_v1: u64,
-    // Total size for objects that have witness one or more failures and will be retried
-    pub replication_failed_size_v1: u64,
-    // Total size for objects that have been replicated to destination
-    pub replicated_size_v1: u64,
-    // Total number of objects pending replication
-    pub replication_pending_count_v1: u64,
-    // Total number of objects that failed replication
-    pub replication_failed_count_v1: u64,
-
-    pub objects_count: u64,
-    pub object_size_histogram: HashMap<String, u64>,
-    pub object_versions_histogram: HashMap<String, u64>,
-    pub versions_count: u64,
-    pub delete_markers_count: u64,
-    pub replica_size: u64,
-    pub replica_count: u64,
-    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
-}
-
-/// DataUsageInfo represents data usage stats of the underlying storage
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct DataUsageInfo {
-    /// Total capacity
-    pub total_capacity: u64,
-    /// Total used capacity
-    pub total_used_capacity: u64,
-    /// Total free capacity
-    pub total_free_capacity: u64,
-
-    /// LastUpdate is the timestamp of when the data usage info was last updated
-    pub last_update: Option<SystemTime>,
-
-    /// Objects total count across all buckets
-    pub objects_total_count: u64,
-    /// Versions total count across all buckets
-    pub versions_total_count: u64,
-    /// Delete markers total count across all buckets
-    pub delete_markers_total_count: u64,
-    /// Objects total size across all buckets
-    pub objects_total_size: u64,
-    /// Replication info across all buckets
-    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
-
-    /// Total number of buckets in this cluster
-    pub buckets_count: u64,
-    /// Buckets usage info provides following information across all buckets
-    pub buckets_usage: HashMap<String, BucketUsageInfo>,
-    /// Deprecated kept here for backward compatibility reasons
-    pub bucket_sizes: HashMap<String, u64>,
-}
-
-/// Size summary for a single object or group of objects
-#[derive(Debug, Default, Clone)]
-pub struct SizeSummary {
-    /// Total size
-    pub total_size: usize,
-    /// Number of versions
-    pub versions: usize,
-    /// Number of delete markers
-    pub delete_markers: usize,
-    /// Replicated size
-    pub replicated_size: usize,
-    /// Replicated count
-    pub replicated_count: usize,
-    /// Pending size
-    pub pending_size: usize,
-    /// Failed size
-    pub failed_size: usize,
-    /// Replica size
-    pub replica_size: usize,
-    /// Replica count
-    pub replica_count: usize,
-    /// Pending count
-    pub pending_count: usize,
-    /// Failed count
-    pub failed_count: usize,
-    /// Replication target stats
-    pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
-}
-
-/// Replication target size summary
-#[derive(Debug, Default, Clone)]
-pub struct ReplTargetSizeSummary {
-    /// Replicated size
-    pub replicated_size: usize,
-    /// Replicated count
-    pub replicated_count: usize,
-    /// Pending size
-    pub pending_size: usize,
-    /// Failed size
-    pub failed_size: usize,
-    /// Pending count
-    pub pending_count: usize,
-    /// Failed count
-    pub failed_count: usize,
-}
-
-impl DataUsageInfo {
-    /// Create a new DataUsageInfo
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    /// Add object metadata to data usage statistics
-    pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
-        // This method is kept for backward compatibility
-        // For accurate version counting, use add_object_from_file_meta instead
-        let bucket_name = match self.extract_bucket_from_path(object_path) {
-            Ok(name) => name,
-            Err(_) => return,
-        };
-
-        // Update bucket statistics
-        if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
-            bucket_usage.size += meta_object.size as u64;
-            bucket_usage.objects_count += 1;
-            bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
-
-            // Update size histogram
-            let total_size = meta_object.size as u64;
-            let size_ranges = [
-                ("0-1KB", 0, 1024),
-                ("1KB-1MB", 1024, 1024 * 1024),
-                ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024),
-                ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024),
-                ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024),
-                ("1GB+", 1024 * 1024 * 1024, u64::MAX),
-            ];
-
-            for (range_name, min_size, max_size) in size_ranges {
-                if total_size >= min_size && total_size < max_size {
-                    *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1;
-                    break;
-                }
-            }
-
-            // Update version histogram (simplified - count as single version)
-            *bucket_usage
-                .object_versions_histogram
-                .entry("SINGLE_VERSION".to_string())
-                .or_insert(0) += 1;
-        } else {
-            // Create new bucket usage
-            let mut bucket_usage = BucketUsageInfo {
-                size: meta_object.size as u64,
-                objects_count: 1,
-                versions_count: 1,
-                ..Default::default()
-            };
-            bucket_usage.object_size_histogram.insert("0-1KB".to_string(), 1);
-            bucket_usage.object_versions_histogram.insert("SINGLE_VERSION".to_string(), 1);
-            self.buckets_usage.insert(bucket_name, bucket_usage);
-        }
-
-        // Update global statistics
-        self.objects_total_size += meta_object.size as u64;
-        self.objects_total_count += 1;
-        self.versions_total_count += 1;
-    }
-
-    /// Add object from FileMeta for accurate version counting
-    pub fn add_object_from_file_meta(&mut self, object_path: &str, file_meta: &rustfs_filemeta::FileMeta) {
-        let bucket_name = match self.extract_bucket_from_path(object_path) {
-            Ok(name) => name,
-            Err(_) => return,
-        };
-
-        // Calculate accurate statistics from all versions
-        let mut total_size = 0u64;
-        let mut versions_count = 0u64;
-        let mut delete_markers_count = 0u64;
-        let mut latest_object_size = 0u64;
-
-        // Process all versions to get accurate counts
-        for version in &file_meta.versions {
-            match rustfs_filemeta::FileMetaVersion::try_from(version.clone()) {
-                Ok(ver) => {
-                    if let Some(obj) = ver.object {
-                        total_size += obj.size as u64;
-                        versions_count += 1;
-                        latest_object_size = obj.size as u64; // Keep track of latest object size
-                    } else if ver.delete_marker.is_some() {
-                        delete_markers_count += 1;
-                    }
-                }
-                Err(_) => {
-                    // Skip invalid versions
-                    continue;
-                }
-            }
-        }
-
-        // Update bucket statistics
-        if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
-            bucket_usage.size += total_size;
-            bucket_usage.objects_count += 1;
-            bucket_usage.versions_count += versions_count;
-            bucket_usage.delete_markers_count += delete_markers_count;
-
-            // Update size histogram based on latest object size
-            let size_ranges = [
-                ("0-1KB", 0, 1024),
-                ("1KB-1MB", 1024, 1024 * 1024),
-                ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024),
-                ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024),
-                ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024),
-                ("1GB+", 1024 * 1024 * 1024, u64::MAX),
-            ];
-
-            for (range_name, min_size, max_size) in size_ranges {
-                if latest_object_size >= min_size && latest_object_size < max_size {
-                    *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1;
-                    break;
-                }
-            }
-
-            // Update version histogram based on actual version count
-            let version_ranges = [
-                ("1", 1, 1),
-                ("2-5", 2, 5),
-                ("6-10", 6, 10),
-                ("11-50", 11, 50),
-                ("51-100", 51, 100),
-                ("100+", 101, usize::MAX),
-            ];
-
-            for (range_name, min_versions, max_versions) in version_ranges {
-                if versions_count as usize >= min_versions && versions_count as usize <= max_versions {
-                    *bucket_usage
-                        .object_versions_histogram
-                        .entry(range_name.to_string())
-                        .or_insert(0) += 1;
-                    break;
-                }
-            }
-        } else {
-            // Create new bucket usage
-            let mut bucket_usage = BucketUsageInfo {
-                size: total_size,
-                objects_count: 1,
-                versions_count,
-                delete_markers_count,
-                ..Default::default()
-            };
-
-            // Set size histogram
-            let size_ranges = [
-                ("0-1KB", 0, 1024),
-                ("1KB-1MB", 1024, 1024 * 1024),
-                ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024),
-                ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024),
-                ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024),
-                ("1GB+", 1024 * 1024 * 1024, u64::MAX),
-            ];
-
-            for (range_name, min_size, max_size) in size_ranges {
-                if latest_object_size >= min_size && latest_object_size < max_size {
-                    bucket_usage.object_size_histogram.insert(range_name.to_string(), 1);
-                    break;
-                }
-            }
-
-            // Set version histogram
-            let version_ranges = [
-                ("1", 1, 1),
-                ("2-5", 2, 5),
-                ("6-10", 6, 10),
-                ("11-50", 11, 50),
-                ("51-100", 51, 100),
-                ("100+", 101, usize::MAX),
-            ];
-
-            for (range_name, min_versions, max_versions) in version_ranges {
-                if versions_count as usize >= min_versions && versions_count as usize <= max_versions {
-                    bucket_usage.object_versions_histogram.insert(range_name.to_string(), 1);
-                    break;
-                }
-            }
-
-            self.buckets_usage.insert(bucket_name, bucket_usage);
-            // Update buckets count when adding new bucket
-            self.buckets_count = self.buckets_usage.len() as u64;
-        }
-
-        // Update global statistics
-        self.objects_total_size += total_size;
-        self.objects_total_count += 1;
-        self.versions_total_count += versions_count;
-        self.delete_markers_total_count += delete_markers_count;
-    }
-
-    /// Extract bucket name from object path
-    fn extract_bucket_from_path(&self, object_path: &str) -> Result<String> {
-        let parts: Vec<&str> = object_path.split('/').collect();
-        if parts.is_empty() {
-            return Err(Error::Scanner("Invalid object path: empty".to_string()));
-        }
-        Ok(parts[0].to_string())
-    }
-
-    /// Update capacity information
-    pub fn update_capacity(&mut self, total: u64, used: u64, free: u64) {
-        self.total_capacity = total;
-        self.total_used_capacity = used;
-        self.total_free_capacity = free;
-        self.last_update = Some(SystemTime::now());
-    }
-
-    /// Add bucket usage info
-    pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) {
-        self.buckets_usage.insert(bucket.clone(), usage);
-        self.buckets_count = self.buckets_usage.len() as u64;
-        self.last_update = Some(SystemTime::now());
-    }
-
-    /// Get bucket usage info
-    pub fn get_bucket_usage(&self, bucket: &str) -> Option<&BucketUsageInfo> {
-        self.buckets_usage.get(bucket)
-    }
-
-    /// Calculate total statistics from all buckets
-    pub fn calculate_totals(&mut self) {
-        self.objects_total_count = 0;
-        self.versions_total_count = 0;
-        self.delete_markers_total_count = 0;
-        self.objects_total_size = 0;
-
-        for usage in self.buckets_usage.values() {
-            self.objects_total_count += usage.objects_count;
-            self.versions_total_count += usage.versions_count;
-            self.delete_markers_total_count += usage.delete_markers_count;
-            self.objects_total_size += usage.size;
-        }
-    }
-
-    /// Merge another DataUsageInfo into this one
-    pub fn merge(&mut self, other: &DataUsageInfo) {
-        // Merge bucket usage
-        for (bucket, usage) in &other.buckets_usage {
-            if let Some(existing) = self.buckets_usage.get_mut(bucket) {
-                existing.merge(usage);
-            } else {
-                self.buckets_usage.insert(bucket.clone(), usage.clone());
-            }
-        }
-
-        // Recalculate totals
-        self.calculate_totals();
-
-        // Ensure buckets_count stays consistent with buckets_usage
-        self.buckets_count = self.buckets_usage.len() as u64;
-
-        // Update last update time
-        if let Some(other_update) = other.last_update {
-            if self.last_update.is_none() || other_update > self.last_update.unwrap() {
-                self.last_update = Some(other_update);
-            }
-        }
-    }
-}
-
-impl BucketUsageInfo {
-    /// Create a new BucketUsageInfo
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    /// Add size summary to this bucket usage
-    pub fn add_size_summary(&mut self, summary: &SizeSummary) {
-        self.size += summary.total_size as u64;
-        self.versions_count += summary.versions as u64;
-        self.delete_markers_count += summary.delete_markers as u64;
-        self.replica_size += summary.replica_size as u64;
-        self.replica_count += summary.replica_count as u64;
-    }
-
-    /// Merge another BucketUsageInfo into this one
-    pub fn merge(&mut self, other: &BucketUsageInfo) {
-        self.size += other.size;
-        self.objects_count += other.objects_count;
-        self.versions_count += other.versions_count;
-        self.delete_markers_count += other.delete_markers_count;
-        self.replica_size += other.replica_size;
-        self.replica_count += other.replica_count;
-
-        // Merge histograms
-        for (key, value) in &other.object_size_histogram {
-            *self.object_size_histogram.entry(key.clone()).or_insert(0) += value;
-        }
-
-        for (key, value) in &other.object_versions_histogram {
-            *self.object_versions_histogram.entry(key.clone()).or_insert(0) += value;
-        }
-
-        // Merge replication info
-        for (target, info) in &other.replication_info {
-            let entry = self.replication_info.entry(target.clone()).or_default();
-            entry.replicated_size += info.replicated_size;
-            entry.replica_size += info.replica_size;
-            entry.replication_pending_size += info.replication_pending_size;
-            entry.replication_failed_size += info.replication_failed_size;
-            entry.replication_pending_count += info.replication_pending_count;
-            entry.replication_failed_count += info.replication_failed_count;
-            entry.replicated_count += info.replicated_count;
-        }
-
-        // Merge backward compatibility fields
-        self.replication_pending_size_v1 += other.replication_pending_size_v1;
-        self.replication_failed_size_v1 += other.replication_failed_size_v1;
-        self.replicated_size_v1 += other.replicated_size_v1;
-        self.replication_pending_count_v1 += other.replication_pending_count_v1;
-        self.replication_failed_count_v1 += other.replication_failed_count_v1;
-    }
-}
-
-impl SizeSummary {
-    /// Create a new SizeSummary
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    /// Add another SizeSummary to this one
-    pub fn add(&mut self, other: &SizeSummary) {
-        self.total_size += other.total_size;
-        self.versions += other.versions;
-        self.delete_markers += other.delete_markers;
-        self.replicated_size += other.replicated_size;
-        self.replicated_count += other.replicated_count;
-        self.pending_size += other.pending_size;
-        self.failed_size += other.failed_size;
-        self.replica_size += other.replica_size;
-        self.replica_count += other.replica_count;
-        self.pending_count += other.pending_count;
-        self.failed_count += other.failed_count;
-
-        // Merge replication target stats
-        for (target, stats) in &other.repl_target_stats {
-            let entry = self.repl_target_stats.entry(target.clone()).or_default();
-            entry.replicated_size += stats.replicated_size;
-            entry.replicated_count += stats.replicated_count;
-            entry.pending_size += stats.pending_size;
-            entry.failed_size += stats.failed_size;
-            entry.pending_count += stats.pending_count;
-            entry.failed_count += stats.failed_count;
-        }
-    }
-}
-
-/// Store data usage info to backend storage
-pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<()> {
-    let data =
-        serde_json::to_vec(&data_usage_info).map_err(|e| Error::Config(format!("Failed to serialize data usage info: {e}")))?;
-
-    // Save to backend using the same mechanism as original code
-    rustfs_ecstore::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
-        .await
-        .map_err(Error::Storage)?;
-
-    Ok(())
-}
-
-/// Load data usage info from backend storage
-pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo> {
-    let buf = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await {
-        Ok(data) => data,
-        Err(e) => {
-            error!("Failed to read data usage info from backend: {}", e);
-            if e == rustfs_ecstore::error::Error::ConfigNotFound {
-                return Ok(DataUsageInfo::default());
-            }
-            return Err(Error::Storage(e));
-        }
-    };
-
-    let mut data_usage_info: DataUsageInfo =
-        serde_json::from_slice(&buf).map_err(|e| Error::Config(format!("Failed to deserialize data usage info: {e}")))?;
-
-    warn!("Loaded data usage info from backend {:?}", &data_usage_info);
-
-    // Handle backward compatibility like original code
-    if data_usage_info.buckets_usage.is_empty() {
-        data_usage_info.buckets_usage = data_usage_info
-            .bucket_sizes
-            .iter()
-            .map(|(bucket, &size)| {
-                (
-                    bucket.clone(),
-                    BucketUsageInfo {
-                        size,
-                        ..Default::default()
-                    },
-                )
-            })
-            .collect();
-    }
-
-    if data_usage_info.bucket_sizes.is_empty() {
-        data_usage_info.bucket_sizes = data_usage_info
-            .buckets_usage
-            .iter()
-            .map(|(bucket, bui)| (bucket.clone(), bui.size))
-            .collect();
-    }
-
-    for (bucket, bui) in &data_usage_info.buckets_usage {
-        if bui.replicated_size_v1 > 0
-            || bui.replication_failed_count_v1 > 0
-            || bui.replication_failed_size_v1 > 0
-            || bui.replication_pending_count_v1 > 0
-        {
-            if let Ok((cfg, _)) = get_replication_config(bucket).await {
-                if !cfg.role.is_empty() {
-                    data_usage_info.replication_info.insert(
-                        cfg.role.clone(),
-                        BucketTargetUsageInfo {
-                            replication_failed_size: bui.replication_failed_size_v1,
-                            replication_failed_count: bui.replication_failed_count_v1,
-                            replicated_size: bui.replicated_size_v1,
-                            replication_pending_count: bui.replication_pending_count_v1,
-                            replication_pending_size: bui.replication_pending_size_v1,
-                            ..Default::default()
-                        },
-                    );
-                }
-            }
-        }
-    }
-
-    Ok(data_usage_info)
-}
-
-/// Example function showing how to use AHM data usage functionality
-/// This demonstrates the integration pattern for DataUsageInfoHandler
-pub async fn example_data_usage_integration() -> Result<()> {
-    // Get the global storage instance
-    let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
-        return Err(Error::Config("Storage not initialized".to_string()));
-    };
-
-    // Load data usage from backend (this replaces the original load_data_usage_from_backend)
-    let data_usage = load_data_usage_from_backend(store).await?;
-
-    info!(
-        "Loaded data usage info: {} buckets, {} total objects",
-        data_usage.buckets_count, data_usage.objects_total_count
-    );
-
-    // Example: Store updated data usage back to backend
-    // This would typically be called by the scanner after collecting new statistics
-    // store_data_usage_in_backend(data_usage, store).await?;
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_data_usage_info_creation() {
-        let mut info = DataUsageInfo::new();
-        info.update_capacity(1000, 500, 500);
-
-        assert_eq!(info.total_capacity, 1000);
-        assert_eq!(info.total_used_capacity, 500);
-        assert_eq!(info.total_free_capacity, 500);
-        assert!(info.last_update.is_some());
-    }
-
-    #[test]
-    fn test_bucket_usage_info_merge() {
-        let mut usage1 = BucketUsageInfo::new();
-        usage1.size = 100;
-        usage1.objects_count = 10;
-        usage1.versions_count = 5;
-
-        let mut usage2 = BucketUsageInfo::new();
-        usage2.size = 200;
-        usage2.objects_count = 20;
-        usage2.versions_count = 10;
-
-        usage1.merge(&usage2);
-
-        assert_eq!(usage1.size, 300);
-        assert_eq!(usage1.objects_count, 30);
-        assert_eq!(usage1.versions_count, 15);
-    }
-
-    #[test]
-    fn test_size_summary_add() {
-        let mut summary1 = SizeSummary::new();
-        summary1.total_size = 100;
-        summary1.versions = 5;
-
-        let mut summary2 = SizeSummary::new();
-        summary2.total_size = 200;
-        summary2.versions = 10;
-
-        summary1.add(&summary2);
-
-        assert_eq!(summary1.total_size, 300);
-        assert_eq!(summary1.versions, 15);
-    }
-}

crates/ahm/src/scanner/histogram.rs

@@ -12,197 +12,258 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::{
+    collections::HashMap,
+    sync::atomic::{AtomicU64, Ordering},
+    time::{Duration, SystemTime},
+};
 
-/// Size interval for object size histogram
-#[derive(Debug, Clone)]
-pub struct SizeInterval {
-    pub start: u64,
-    pub end: u64,
-    pub name: &'static str,
+use serde::{Deserialize, Serialize};
+use tracing::info;
+
+/// Scanner metrics
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct ScannerMetrics {
+    /// Total objects scanned since server start
+    pub objects_scanned: u64,
+    /// Total object versions scanned since server start
+    pub versions_scanned: u64,
+    /// Total directories scanned since server start
+    pub directories_scanned: u64,
+    /// Total bucket scans started since server start
+    pub bucket_scans_started: u64,
+    /// Total bucket scans finished since server start
+    pub bucket_scans_finished: u64,
+    /// Total objects with health issues found
+    pub objects_with_issues: u64,
+    /// Total heal tasks queued
+    pub heal_tasks_queued: u64,
+    /// Total heal tasks completed
+    pub heal_tasks_completed: u64,
+    /// Total heal tasks failed
+    pub heal_tasks_failed: u64,
+    /// Total healthy objects found
+    pub healthy_objects: u64,
+    /// Total corrupted objects found
+    pub corrupted_objects: u64,
+    /// Last scan activity time
+    pub last_activity: Option<SystemTime>,
+    /// Current scan cycle
+    pub current_cycle: u64,
+    /// Total scan cycles completed
+    pub total_cycles: u64,
+    /// Current scan duration
+    pub current_scan_duration: Option<Duration>,
+    /// Average scan duration
+    pub avg_scan_duration: Duration,
+    /// Objects scanned per second
+    pub objects_per_second: f64,
+    /// Buckets scanned per second
+    pub buckets_per_second: f64,
+    /// Storage metrics by bucket
+    pub bucket_metrics: HashMap<String, BucketMetrics>,
+    /// Disk metrics
+    pub disk_metrics: HashMap<String, DiskMetrics>,
 }
 
-/// Version interval for object versions histogram
-#[derive(Debug, Clone)]
-pub struct VersionInterval {
-    pub start: u64,
-    pub end: u64,
-    pub name: &'static str,
+/// Bucket-specific metrics
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct BucketMetrics {
+    /// Bucket name
+    pub bucket: String,
+    /// Total objects in bucket
+    pub total_objects: u64,
+    /// Total size of objects in bucket (bytes)
+    pub total_size: u64,
+    /// Objects with health issues
+    pub objects_with_issues: u64,
+    /// Last scan time
+    pub last_scan_time: Option<SystemTime>,
+    /// Scan duration
+    pub scan_duration: Option<Duration>,
+    /// Heal tasks queued for this bucket
+    pub heal_tasks_queued: u64,
+    /// Heal tasks completed for this bucket
+    pub heal_tasks_completed: u64,
+    /// Heal tasks failed for this bucket
+    pub heal_tasks_failed: u64,
 }
 
-/// Object size histogram intervals
-pub const OBJECTS_HISTOGRAM_INTERVALS: &[SizeInterval] = &[
-    SizeInterval {
-        start: 0,
-        end: 1024 - 1,
-        name: "LESS_THAN_1_KiB",
-    },
-    SizeInterval {
-        start: 1024,
-        end: 1024 * 1024 - 1,
-        name: "1_KiB_TO_1_MiB",
-    },
-    SizeInterval {
-        start: 1024 * 1024,
-        end: 10 * 1024 * 1024 - 1,
-        name: "1_MiB_TO_10_MiB",
-    },
-    SizeInterval {
-        start: 10 * 1024 * 1024,
-        end: 64 * 1024 * 1024 - 1,
-        name: "10_MiB_TO_64_MiB",
-    },
-    SizeInterval {
-        start: 64 * 1024 * 1024,
-        end: 128 * 1024 * 1024 - 1,
-        name: "64_MiB_TO_128_MiB",
-    },
-    SizeInterval {
-        start: 128 * 1024 * 1024,
-        end: 512 * 1024 * 1024 - 1,
-        name: "128_MiB_TO_512_MiB",
-    },
-    SizeInterval {
-        start: 512 * 1024 * 1024,
-        end: u64::MAX,
-        name: "MORE_THAN_512_MiB",
-    },
-];
-
-/// Object version count histogram intervals
-pub const OBJECTS_VERSION_COUNT_INTERVALS: &[VersionInterval] = &[
-    VersionInterval {
-        start: 1,
-        end: 1,
-        name: "1_VERSION",
-    },
-    VersionInterval {
-        start: 2,
-        end: 10,
-        name: "2_TO_10_VERSIONS",
-    },
-    VersionInterval {
-        start: 11,
-        end: 100,
-        name: "11_TO_100_VERSIONS",
-    },
-    VersionInterval {
-        start: 101,
-        end: 1000,
-        name: "101_TO_1000_VERSIONS",
-    },
-    VersionInterval {
-        start: 1001,
-        end: u64::MAX,
-        name: "MORE_THAN_1000_VERSIONS",
-    },
-];
-
-/// Size histogram for object size distribution
-#[derive(Debug, Clone, Default)]
-pub struct SizeHistogram {
-    counts: Vec<u64>,
+/// Disk-specific metrics
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct DiskMetrics {
+    /// Disk path
+    pub disk_path: String,
+    /// Total disk space (bytes)
+    pub total_space: u64,
+    /// Used disk space (bytes)
+    pub used_space: u64,
+    /// Free disk space (bytes)
+    pub free_space: u64,
+    /// Objects scanned on this disk
+    pub objects_scanned: u64,
+    /// Objects with issues on this disk
+    pub objects_with_issues: u64,
+    /// Last scan time
+    pub last_scan_time: Option<SystemTime>,
+    /// Whether disk is online
+    pub is_online: bool,
+    /// Whether disk is being scanned
+    pub is_scanning: bool,
 }
 
-/// Versions histogram for object version count distribution
-#[derive(Debug, Clone, Default)]
-pub struct VersionsHistogram {
-    counts: Vec<u64>,
+/// Thread-safe metrics collector
+pub struct MetricsCollector {
+    /// Atomic counters for real-time metrics
+    objects_scanned: AtomicU64,
+    versions_scanned: AtomicU64,
+    directories_scanned: AtomicU64,
+    bucket_scans_started: AtomicU64,
+    bucket_scans_finished: AtomicU64,
+    objects_with_issues: AtomicU64,
+    heal_tasks_queued: AtomicU64,
+    heal_tasks_completed: AtomicU64,
+    heal_tasks_failed: AtomicU64,
+    current_cycle: AtomicU64,
+    total_cycles: AtomicU64,
+    healthy_objects: AtomicU64,
+    corrupted_objects: AtomicU64,
 }
 
-impl SizeHistogram {
-    /// Create a new size histogram
+impl MetricsCollector {
+    /// Create a new metrics collector
     pub fn new() -> Self {
         Self {
-            counts: vec![0; OBJECTS_HISTOGRAM_INTERVALS.len()],
+            objects_scanned: AtomicU64::new(0),
+            versions_scanned: AtomicU64::new(0),
+            directories_scanned: AtomicU64::new(0),
+            bucket_scans_started: AtomicU64::new(0),
+            bucket_scans_finished: AtomicU64::new(0),
+            objects_with_issues: AtomicU64::new(0),
+            heal_tasks_queued: AtomicU64::new(0),
+            heal_tasks_completed: AtomicU64::new(0),
+            heal_tasks_failed: AtomicU64::new(0),
+            current_cycle: AtomicU64::new(0),
+            total_cycles: AtomicU64::new(0),
+            healthy_objects: AtomicU64::new(0),
+            corrupted_objects: AtomicU64::new(0),
         }
     }
 
-    /// Add a size to the histogram
-    pub fn add(&mut self, size: u64) {
-        for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() {
-            if size >= interval.start && size <= interval.end {
-                self.counts[idx] += 1;
-                break;
-            }
+    /// Increment objects scanned count
+    pub fn increment_objects_scanned(&self, count: u64) {
+        self.objects_scanned.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment versions scanned count
+    pub fn increment_versions_scanned(&self, count: u64) {
+        self.versions_scanned.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment directories scanned count
+    pub fn increment_directories_scanned(&self, count: u64) {
+        self.directories_scanned.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment bucket scans started count
+    pub fn increment_bucket_scans_started(&self, count: u64) {
+        self.bucket_scans_started.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment bucket scans finished count
+    pub fn increment_bucket_scans_finished(&self, count: u64) {
+        self.bucket_scans_finished.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment objects with issues count
+    pub fn increment_objects_with_issues(&self, count: u64) {
+        self.objects_with_issues.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment heal tasks queued count
+    pub fn increment_heal_tasks_queued(&self, count: u64) {
+        self.heal_tasks_queued.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment heal tasks completed count
+    pub fn increment_heal_tasks_completed(&self, count: u64) {
+        self.heal_tasks_completed.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Increment heal tasks failed count
+    pub fn increment_heal_tasks_failed(&self, count: u64) {
+        self.heal_tasks_failed.fetch_add(count, Ordering::Relaxed);
+    }
+
+    /// Set current cycle
+    pub fn set_current_cycle(&self, cycle: u64) {
+        self.current_cycle.store(cycle, Ordering::Relaxed);
+    }
+
+    /// Increment total cycles
+    pub fn increment_total_cycles(&self) {
+        self.total_cycles.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Increment healthy objects count
+    pub fn increment_healthy_objects(&self) {
+        self.healthy_objects.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Increment corrupted objects count
+    pub fn increment_corrupted_objects(&self) {
+        self.corrupted_objects.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Get current metrics snapshot
+    pub fn get_metrics(&self) -> ScannerMetrics {
+        ScannerMetrics {
+            objects_scanned: self.objects_scanned.load(Ordering::Relaxed),
+            versions_scanned: self.versions_scanned.load(Ordering::Relaxed),
+            directories_scanned: self.directories_scanned.load(Ordering::Relaxed),
+            bucket_scans_started: self.bucket_scans_started.load(Ordering::Relaxed),
+            bucket_scans_finished: self.bucket_scans_finished.load(Ordering::Relaxed),
+            objects_with_issues: self.objects_with_issues.load(Ordering::Relaxed),
+            heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed),
+            heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed),
+            heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed),
+            healthy_objects: self.healthy_objects.load(Ordering::Relaxed),
+            corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed),
+            last_activity: Some(SystemTime::now()),
+            current_cycle: self.current_cycle.load(Ordering::Relaxed),
+            total_cycles: self.total_cycles.load(Ordering::Relaxed),
+            current_scan_duration: None,       // Will be set by scanner
+            avg_scan_duration: Duration::ZERO, // Will be calculated
+            objects_per_second: 0.0,           // Will be calculated
+            buckets_per_second: 0.0,           // Will be calculated
+            bucket_metrics: HashMap::new(),    // Will be populated by scanner
+            disk_metrics: HashMap::new(),      // Will be populated by scanner
         }
     }
 
-    /// Get the histogram as a map
-    pub fn to_map(&self) -> HashMap<String, u64> {
-        let mut result = HashMap::new();
-        for (idx, count) in self.counts.iter().enumerate() {
-            let interval = &OBJECTS_HISTOGRAM_INTERVALS[idx];
-            result.insert(interval.name.to_string(), *count);
-        }
-        result
-    }
+    /// Reset all metrics
+    pub fn reset(&self) {
+        self.objects_scanned.store(0, Ordering::Relaxed);
+        self.versions_scanned.store(0, Ordering::Relaxed);
+        self.directories_scanned.store(0, Ordering::Relaxed);
+        self.bucket_scans_started.store(0, Ordering::Relaxed);
+        self.bucket_scans_finished.store(0, Ordering::Relaxed);
+        self.objects_with_issues.store(0, Ordering::Relaxed);
+        self.heal_tasks_queued.store(0, Ordering::Relaxed);
+        self.heal_tasks_completed.store(0, Ordering::Relaxed);
+        self.heal_tasks_failed.store(0, Ordering::Relaxed);
+        self.current_cycle.store(0, Ordering::Relaxed);
+        self.total_cycles.store(0, Ordering::Relaxed);
+        self.healthy_objects.store(0, Ordering::Relaxed);
+        self.corrupted_objects.store(0, Ordering::Relaxed);
 
-    /// Merge another histogram into this one
-    pub fn merge(&mut self, other: &SizeHistogram) {
-        for (idx, count) in other.counts.iter().enumerate() {
-            self.counts[idx] += count;
-        }
-    }
-
-    /// Get total count
-    pub fn total_count(&self) -> u64 {
-        self.counts.iter().sum()
-    }
-
-    /// Reset the histogram
-    pub fn reset(&mut self) {
-        for count in &mut self.counts {
-            *count = 0;
-        }
+        info!("Scanner metrics reset");
     }
 }
 
-impl VersionsHistogram {
-    /// Create a new versions histogram
-    pub fn new() -> Self {
-        Self {
-            counts: vec![0; OBJECTS_VERSION_COUNT_INTERVALS.len()],
-        }
-    }
-
-    /// Add a version count to the histogram
-    pub fn add(&mut self, versions: u64) {
-        for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() {
-            if versions >= interval.start && versions <= interval.end {
-                self.counts[idx] += 1;
-                break;
-            }
-        }
-    }
-
-    /// Get the histogram as a map
-    pub fn to_map(&self) -> HashMap<String, u64> {
-        let mut result = HashMap::new();
-        for (idx, count) in self.counts.iter().enumerate() {
-            let interval = &OBJECTS_VERSION_COUNT_INTERVALS[idx];
-            result.insert(interval.name.to_string(), *count);
-        }
-        result
-    }
-
-    /// Merge another histogram into this one
-    pub fn merge(&mut self, other: &VersionsHistogram) {
-        for (idx, count) in other.counts.iter().enumerate() {
-            self.counts[idx] += count;
-        }
-    }
-
-    /// Get total count
-    pub fn total_count(&self) -> u64 {
-        self.counts.iter().sum()
-    }
-
-    /// Reset the histogram
-    pub fn reset(&mut self) {
-        for count in &mut self.counts {
-            *count = 0;
-        }
+impl Default for MetricsCollector {
+    fn default() -> Self {
+        Self::new()
     }
 }
 
@@ -211,67 +272,35 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_size_histogram() {
-        let mut histogram = SizeHistogram::new();
-
-        // Add some sizes
-        histogram.add(512); // LESS_THAN_1_KiB
-        histogram.add(1024); // 1_KiB_TO_1_MiB
-        histogram.add(1024 * 1024); // 1_MiB_TO_10_MiB
-        histogram.add(5 * 1024 * 1024); // 1_MiB_TO_10_MiB
-
-        let map = histogram.to_map();
-
-        assert_eq!(map.get("LESS_THAN_1_KiB"), Some(&1));
-        assert_eq!(map.get("1_KiB_TO_1_MiB"), Some(&1));
-        assert_eq!(map.get("1_MiB_TO_10_MiB"), Some(&2));
-        assert_eq!(map.get("10_MiB_TO_64_MiB"), Some(&0));
+    fn test_metrics_collector_creation() {
+        let collector = MetricsCollector::new();
+        let metrics = collector.get_metrics();
+        assert_eq!(metrics.objects_scanned, 0);
+        assert_eq!(metrics.versions_scanned, 0);
     }
 
     #[test]
-    fn test_versions_histogram() {
-        let mut histogram = VersionsHistogram::new();
+    fn test_metrics_increment() {
+        let collector = MetricsCollector::new();
 
-        // Add some version counts
-        histogram.add(1); // 1_VERSION
-        histogram.add(5); // 2_TO_10_VERSIONS
-        histogram.add(50); // 11_TO_100_VERSIONS
-        histogram.add(500); // 101_TO_1000_VERSIONS
+        collector.increment_objects_scanned(10);
+        collector.increment_versions_scanned(5);
+        collector.increment_objects_with_issues(2);
 
-        let map = histogram.to_map();
-
-        assert_eq!(map.get("1_VERSION"), Some(&1));
-        assert_eq!(map.get("2_TO_10_VERSIONS"), Some(&1));
-        assert_eq!(map.get("11_TO_100_VERSIONS"), Some(&1));
-        assert_eq!(map.get("101_TO_1000_VERSIONS"), Some(&1));
+        let metrics = collector.get_metrics();
+        assert_eq!(metrics.objects_scanned, 10);
+        assert_eq!(metrics.versions_scanned, 5);
+        assert_eq!(metrics.objects_with_issues, 2);
     }
 
     #[test]
-    fn test_histogram_merge() {
-        let mut histogram1 = SizeHistogram::new();
-        histogram1.add(1024);
-        histogram1.add(1024 * 1024);
+    fn test_metrics_reset() {
+        let collector = MetricsCollector::new();
 
-        let mut histogram2 = SizeHistogram::new();
-        histogram2.add(1024);
-        histogram2.add(5 * 1024 * 1024);
+        collector.increment_objects_scanned(10);
+        collector.reset();
 
-        histogram1.merge(&histogram2);
-
-        let map = histogram1.to_map();
-        assert_eq!(map.get("1_KiB_TO_1_MiB"), Some(&2)); // 1 from histogram1 + 1 from histogram2
-        assert_eq!(map.get("1_MiB_TO_10_MiB"), Some(&2)); // 1 from histogram1 + 1 from histogram2
-    }
-
-    #[test]
-    fn test_histogram_reset() {
-        let mut histogram = SizeHistogram::new();
-        histogram.add(1024);
-        histogram.add(1024 * 1024);
-
-        assert_eq!(histogram.total_count(), 2);
-
-        histogram.reset();
-        assert_eq!(histogram.total_count(), 0);
+        let metrics = collector.get_metrics();
+        assert_eq!(metrics.objects_scanned, 0);
     }
 }

crates/ahm/src/scanner/metrics.rs

@@ -42,6 +42,10 @@ pub struct ScannerMetrics {
     pub heal_tasks_completed: u64,
     /// Total heal tasks failed
     pub heal_tasks_failed: u64,
+    /// Total healthy objects found
+    pub healthy_objects: u64,
+    /// Total corrupted objects found
+    pub corrupted_objects: u64,
     /// Last scan activity time
     pub last_activity: Option<SystemTime>,
     /// Current scan cycle
@@ -122,6 +126,8 @@ pub struct MetricsCollector {
     heal_tasks_failed: AtomicU64,
     current_cycle: AtomicU64,
     total_cycles: AtomicU64,
+    healthy_objects: AtomicU64,
+    corrupted_objects: AtomicU64,
 }
 
 impl MetricsCollector {
@@ -139,6 +145,8 @@ impl MetricsCollector {
             heal_tasks_failed: AtomicU64::new(0),
             current_cycle: AtomicU64::new(0),
             total_cycles: AtomicU64::new(0),
+            healthy_objects: AtomicU64::new(0),
+            corrupted_objects: AtomicU64::new(0),
         }
     }
 
@@ -197,6 +205,16 @@ impl MetricsCollector {
         self.total_cycles.fetch_add(1, Ordering::Relaxed);
     }
 
+    /// Increment healthy objects count
+    pub fn increment_healthy_objects(&self) {
+        self.healthy_objects.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Increment corrupted objects count
+    pub fn increment_corrupted_objects(&self) {
+        self.corrupted_objects.fetch_add(1, Ordering::Relaxed);
+    }
+
     /// Get current metrics snapshot
     pub fn get_metrics(&self) -> ScannerMetrics {
         ScannerMetrics {
@@ -209,6 +227,8 @@ impl MetricsCollector {
             heal_tasks_queued: self.heal_tasks_queued.load(Ordering::Relaxed),
             heal_tasks_completed: self.heal_tasks_completed.load(Ordering::Relaxed),
             heal_tasks_failed: self.heal_tasks_failed.load(Ordering::Relaxed),
+            healthy_objects: self.healthy_objects.load(Ordering::Relaxed),
+            corrupted_objects: self.corrupted_objects.load(Ordering::Relaxed),
             last_activity: Some(SystemTime::now()),
             current_cycle: self.current_cycle.load(Ordering::Relaxed),
             total_cycles: self.total_cycles.load(Ordering::Relaxed),
@@ -234,6 +254,8 @@ impl MetricsCollector {
         self.heal_tasks_failed.store(0, Ordering::Relaxed);
         self.current_cycle.store(0, Ordering::Relaxed);
         self.total_cycles.store(0, Ordering::Relaxed);
+        self.healthy_objects.store(0, Ordering::Relaxed);
+        self.corrupted_objects.store(0, Ordering::Relaxed);
 
         info!("Scanner metrics reset");
     }

crates/ahm/src/scanner/mod.rs

@@ -13,13 +13,8 @@
 // limitations under the License.
 
 pub mod data_scanner;
-pub mod data_usage;
 pub mod histogram;
 pub mod metrics;
 
-// Re-export main types for convenience
 pub use data_scanner::Scanner;
-pub use data_usage::{
-    BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, load_data_usage_from_backend, store_data_usage_in_backend,
-};
 pub use metrics::ScannerMetrics;

crates/ahm/tests/heal_integration_test.rs

@@ -0,0 +1,410 @@
+use rustfs_ahm::heal::{
+    manager::{HealConfig, HealManager},
+    storage::{ECStoreHealStorage, HealStorageAPI},
+    task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
+};
+use rustfs_common::heal_channel::{HealOpts, HealScanMode};
+use rustfs_ecstore::{
+    disk::endpoint::Endpoint,
+    endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
+    store::ECStore,
+    store_api::{ObjectIO, ObjectOptions, PutObjReader, StorageAPI},
+};
+use serial_test::serial;
+use std::sync::Once;
+use std::sync::OnceLock;
+use std::{path::PathBuf, sync::Arc, time::Duration};
+use tokio::fs;
+use tracing::info;
+use walkdir::WalkDir;
+
+static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>)> = OnceLock::new();
+static INIT: Once = Once::new();
+
+fn init_tracing() {
+    INIT.call_once(|| {
+        let _ = tracing_subscriber::fmt::try_init();
+    });
+}
+
+/// Test helper: Create test environment with ECStore
+async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage>) {
+    init_tracing();
+
+    // Fast path: already initialized, just clone and return
+    if let Some((paths, ecstore, heal_storage)) = GLOBAL_ENV.get() {
+        return (paths.clone(), ecstore.clone(), heal_storage.clone());
+    }
+
+    // create temp dir as 4 disks with unique base dir
+    let test_base_dir = format!("/tmp/rustfs_ahm_heal_test_{}", uuid::Uuid::new_v4());
+    let temp_dir = std::path::PathBuf::from(&test_base_dir);
+    if temp_dir.exists() {
+        fs::remove_dir_all(&temp_dir).await.ok();
+    }
+    fs::create_dir_all(&temp_dir).await.unwrap();
+
+    // create 4 disk dirs
+    let disk_paths = vec![
+        temp_dir.join("disk1"),
+        temp_dir.join("disk2"),
+        temp_dir.join("disk3"),
+        temp_dir.join("disk4"),
+    ];
+
+    for disk_path in &disk_paths {
+        fs::create_dir_all(disk_path).await.unwrap();
+    }
+
+    // create EndpointServerPools
+    let mut endpoints = Vec::new();
+    for (i, disk_path) in disk_paths.iter().enumerate() {
+        let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+        // set correct index
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+        endpoints.push(endpoint);
+    }
+
+    let pool_endpoints = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: 4,
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "test".to_string(),
+        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+    };
+
+    let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
+
+    // format disks (only first time)
+    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
+
+    // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
+    let port = 9001; // for simplicity
+    let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+
+    // init bucket metadata system
+    let buckets_list = ecstore
+        .list_bucket(&rustfs_ecstore::store_api::BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+    rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
+
+    // Create heal storage layer
+    let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+
+    // Store in global once lock
+    let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone(), heal_storage.clone()));
+
+    (disk_paths, ecstore, heal_storage)
+}
+
+/// Test helper: Create a test bucket
+async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
+    (**ecstore)
+        .make_bucket(bucket_name, &Default::default())
+        .await
+        .expect("Failed to create test bucket");
+    info!("Created test bucket: {}", bucket_name);
+}
+
+/// Test helper: Upload test object
+async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
+    let mut reader = PutObjReader::from_vec(data.to_vec());
+    let object_info = (**ecstore)
+        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+        .await
+        .expect("Failed to upload test object");
+
+    info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn test_heal_object_basic() {
+    let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+
+    // Create test bucket and object
+    let bucket_name = "test-bucket";
+    let object_name = "test-object.txt";
+    let test_data = b"Hello, this is test data for healing!";
+
+    create_test_bucket(&ecstore, bucket_name).await;
+    upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+
+    // ─── 1️⃣ delete single data shard file ─────────────────────────────────────
+    let obj_dir = disk_paths[0].join(bucket_name).join(object_name);
+    // find part file at depth 2, e.g. .../<uuid>/part.1
+    let target_part = WalkDir::new(&obj_dir)
+        .min_depth(2)
+        .max_depth(2)
+        .into_iter()
+        .filter_map(Result::ok)
+        .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false))
+        .map(|e| e.into_path())
+        .expect("Failed to locate part file to delete");
+
+    std::fs::remove_file(&target_part).expect("failed to delete part file");
+    assert!(!target_part.exists());
+    println!("✅ Deleted shard part file: {target_part:?}");
+
+    // Create heal manager with faster interval
+    let cfg = HealConfig {
+        heal_interval: Duration::from_millis(1),
+        ..Default::default()
+    };
+    let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
+    heal_manager.start().await.unwrap();
+
+    // Submit heal request for the object
+    let heal_request = HealRequest::new(
+        HealType::Object {
+            bucket: bucket_name.to_string(),
+            object: object_name.to_string(),
+            version_id: None,
+        },
+        HealOptions {
+            dry_run: false,
+            recursive: false,
+            remove_corrupted: false,
+            recreate_missing: true,
+            scan_mode: HealScanMode::Normal,
+            update_parity: true,
+            timeout: Some(Duration::from_secs(300)),
+            pool_index: None,
+            set_index: None,
+        },
+        HealPriority::Normal,
+    );
+
+    let task_id = heal_manager
+        .submit_heal_request(heal_request)
+        .await
+        .expect("Failed to submit heal request");
+
+    info!("Submitted heal request with task ID: {}", task_id);
+
+    // Wait for task completion
+    tokio::time::sleep(tokio::time::Duration::from_secs(8)).await;
+
+    // Attempt to fetch task status (might be removed if finished)
+    match heal_manager.get_task_status(&task_id).await {
+        Ok(status) => info!("Task status: {:?}", status),
+        Err(e) => info!("Task status not found (likely completed): {}", e),
+    }
+
+    // ─── 2️⃣ verify each part file is restored ───────
+    assert!(target_part.exists());
+
+    info!("Heal object basic test passed");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn test_heal_bucket_basic() {
+    let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+
+    // Create test bucket
+    let bucket_name = "test-bucket-heal";
+    create_test_bucket(&ecstore, bucket_name).await;
+
+    // ─── 1️⃣ delete bucket dir on disk ──────────────
+    let broken_bucket_path = disk_paths[0].join(bucket_name);
+    assert!(broken_bucket_path.exists(), "bucket dir does not exist on disk");
+    std::fs::remove_dir_all(&broken_bucket_path).expect("failed to delete bucket dir on disk");
+    assert!(!broken_bucket_path.exists(), "bucket dir still exists after deletion");
+    println!("✅ Deleted bucket directory on disk: {broken_bucket_path:?}");
+
+    // Create heal manager with faster interval
+    let cfg = HealConfig {
+        heal_interval: Duration::from_millis(1),
+        ..Default::default()
+    };
+    let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
+    heal_manager.start().await.unwrap();
+
+    // Submit heal request for the bucket
+    let heal_request = HealRequest::new(
+        HealType::Bucket {
+            bucket: bucket_name.to_string(),
+        },
+        HealOptions {
+            dry_run: false,
+            recursive: true,
+            remove_corrupted: false,
+            recreate_missing: false,
+            scan_mode: HealScanMode::Normal,
+            update_parity: false,
+            timeout: Some(Duration::from_secs(300)),
+            pool_index: None,
+            set_index: None,
+        },
+        HealPriority::Normal,
+    );
+
+    let task_id = heal_manager
+        .submit_heal_request(heal_request)
+        .await
+        .expect("Failed to submit bucket heal request");
+
+    info!("Submitted bucket heal request with task ID: {}", task_id);
+
+    // Wait for task completion
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+    // Attempt to fetch task status (optional)
+    if let Ok(status) = heal_manager.get_task_status(&task_id).await {
+        if status == HealTaskStatus::Completed {
+            info!("Bucket heal task status: {:?}", status);
+        } else {
+            panic!("Bucket heal task status: {status:?}");
+        }
+    }
+
+    // ─── 3️⃣ Verify bucket directory is restored on every disk ───────
+    assert!(broken_bucket_path.exists(), "bucket dir does not exist on disk");
+
+    info!("Heal bucket basic test passed");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn test_heal_format_basic() {
+    let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;
+
+    // ─── 1️⃣ delete format.json on one disk ──────────────
+    let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
+    assert!(format_path.exists(), "format.json does not exist on disk");
+    std::fs::remove_file(&format_path).expect("failed to delete format.json on disk");
+    assert!(!format_path.exists(), "format.json still exists after deletion");
+    println!("✅ Deleted format.json on disk: {format_path:?}");
+
+    // Create heal manager with faster interval
+    let cfg = HealConfig {
+        heal_interval: Duration::from_secs(2),
+        ..Default::default()
+    };
+    let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
+    heal_manager.start().await.unwrap();
+
+    // Wait for task completion
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+    // ─── 2️⃣ verify format.json is restored ───────
+    assert!(format_path.exists(), "format.json does not exist on disk after heal");
+
+    info!("Heal format basic test passed");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn test_heal_format_with_data() {
+    let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
+
+    // Create test bucket and object
+    let bucket_name = "test-bucket";
+    let object_name = "test-object.txt";
+    let test_data = b"Hello, this is test data for healing!";
+
+    create_test_bucket(&ecstore, bucket_name).await;
+    upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+
+    let obj_dir = disk_paths[0].join(bucket_name).join(object_name);
+    let target_part = WalkDir::new(&obj_dir)
+        .min_depth(2)
+        .max_depth(2)
+        .into_iter()
+        .filter_map(Result::ok)
+        .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false))
+        .map(|e| e.into_path())
+        .expect("Failed to locate part file to delete");
+
+    // ─── 1️⃣ delete format.json on one disk ──────────────
+    let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
+    std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
+    std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
+    println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
+
+    // Create heal manager with faster interval
+    let cfg = HealConfig {
+        heal_interval: Duration::from_secs(2),
+        ..Default::default()
+    };
+    let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
+    heal_manager.start().await.unwrap();
+
+    // Wait for task completion
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+    // ─── 2️⃣ verify format.json is restored ───────
+    assert!(format_path.exists(), "format.json does not exist on disk after heal");
+    // ─── 3 verify each part file is restored ───────
+    assert!(target_part.exists());
+
+    info!("Heal format basic test passed");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn test_heal_storage_api_direct() {
+    let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
+
+    // Test direct heal storage API calls
+
+    // Test heal_format
+    let format_result = heal_storage.heal_format(true).await; // dry run
+    assert!(format_result.is_ok());
+    info!("Direct heal_format test passed");
+
+    // Test heal_bucket
+    let bucket_name = "test-bucket-direct";
+    create_test_bucket(&ecstore, bucket_name).await;
+
+    let heal_opts = HealOpts {
+        recursive: true,
+        dry_run: true,
+        remove: false,
+        recreate: false,
+        scan_mode: HealScanMode::Normal,
+        update_parity: false,
+        no_lock: false,
+        pool: None,
+        set: None,
+    };
+
+    let bucket_result = heal_storage.heal_bucket(bucket_name, &heal_opts).await;
+    assert!(bucket_result.is_ok());
+    info!("Direct heal_bucket test passed");
+
+    // Test heal_object
+    let object_name = "test-object-direct.txt";
+    let test_data = b"Test data for direct heal API";
+    upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+
+    let object_heal_opts = HealOpts {
+        recursive: false,
+        dry_run: true,
+        remove: false,
+        recreate: false,
+        scan_mode: HealScanMode::Normal,
+        update_parity: false,
+        no_lock: false,
+        pool: None,
+        set: None,
+    };
+
+    let object_result = heal_storage
+        .heal_object(bucket_name, object_name, None, &object_heal_opts)
+        .await;
+    assert!(object_result.is_ok());
+    info!("Direct heal_object test passed");
+
+    info!("Direct heal storage API test passed");
+}

crates/checksums/Cargo.toml

@@ -0,0 +1,45 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-checksums"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+homepage.workspace = true
+description = "Checksum calculation and verification callbacks for HTTP request and response bodies sent by service clients generated by RustFS, ensuring data integrity and authenticity."
+keywords = ["checksum-calculation", "verification", "integrity", "authenticity", "rustfs"]
+categories = ["web-programming", "development-tools", "network-programming"]
+documentation = "https://docs.rs/rustfs-signer/latest/rustfs_checksum/"
+
+[dependencies]
+bytes = { workspace = true }
+crc-fast = { workspace = true }
+hex = { workspace = true }
+http = { workspace = true }
+http-body = { workspace = true }
+base64-simd = { workspace = true }
+md-5 = { workspace = true }
+pin-project-lite = { workspace = true }
+sha1 = { workspace = true }
+sha2 = { workspace = true }
+tracing = { workspace = true }
+
+[dev-dependencies]
+bytes-utils = { workspace = true }
+pretty_assertions = { workspace = true }
+tracing-test = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt"] }

crates/checksums/README.md

@@ -0,0 +1,3 @@
+# rustfs-checksums
+
+Checksum calculation and verification callbacks for HTTP request and response bodies sent by service clients generated by RustFS object storage.

crates/checksums/src/base64.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(dead_code)]
+
+use base64_simd::STANDARD;
+use std::error::Error;
+
+#[derive(Debug)]
+pub(crate) struct DecodeError(base64_simd::Error);
+
+impl Error for DecodeError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        Some(&self.0)
+    }
+}
+
+impl std::fmt::Display for DecodeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "failed to decode base64")
+    }
+}
+
+pub(crate) fn decode(input: impl AsRef<str>) -> Result<Vec<u8>, DecodeError> {
+    STANDARD.decode_to_vec(input.as_ref()).map_err(DecodeError)
+}
+
+pub(crate) fn encode(input: impl AsRef<[u8]>) -> String {
+    STANDARD.encode_to_string(input.as_ref())
+}
+
+pub(crate) fn encoded_length(length: usize) -> usize {
+    STANDARD.encoded_length(length)
+}

crates/checksums/src/error.rs

@@ -0,0 +1,45 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug)]
+pub struct UnknownChecksumAlgorithmError {
+    checksum_algorithm: String,
+}
+
+impl UnknownChecksumAlgorithmError {
+    pub(crate) fn new(checksum_algorithm: impl Into<String>) -> Self {
+        Self {
+            checksum_algorithm: checksum_algorithm.into(),
+        }
+    }
+
+    pub fn checksum_algorithm(&self) -> &str {
+        &self.checksum_algorithm
+    }
+}
+
+impl fmt::Display for UnknownChecksumAlgorithmError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "sha1", "sha256", "md5")"#,
+            self.checksum_algorithm
+        )
+    }
+}
+
+impl Error for UnknownChecksumAlgorithmError {}

crates/checksums/src/http.rs

@@ -0,0 +1,196 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::base64;
+use http::header::{HeaderMap, HeaderValue};
+
+use crate::Crc64Nvme;
+use crate::{CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256};
+
+pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
+pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
+pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
+pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256";
+pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme";
+
+pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
+
+pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
+    [CRC_64_NVME_NAME, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME];
+
+pub trait HttpChecksum: Checksum + Send + Sync {
+    fn headers(self: Box<Self>) -> HeaderMap<HeaderValue> {
+        let mut header_map = HeaderMap::new();
+        header_map.insert(self.header_name(), self.header_value());
+
+        header_map
+    }
+
+    fn header_name(&self) -> &'static str;
+
+    fn header_value(self: Box<Self>) -> HeaderValue {
+        let hash = self.finalize();
+        HeaderValue::from_str(&base64::encode(&hash[..])).expect("base64 encoded bytes are always valid header values")
+    }
+
+    fn size(&self) -> u64 {
+        let trailer_name_size_in_bytes = self.header_name().len();
+        let base64_encoded_checksum_size_in_bytes = base64::encoded_length(Checksum::size(self) as usize);
+
+        let size = trailer_name_size_in_bytes + ":".len() + base64_encoded_checksum_size_in_bytes;
+
+        size as u64
+    }
+}
+
+impl HttpChecksum for Crc32 {
+    fn header_name(&self) -> &'static str {
+        CRC_32_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Crc32c {
+    fn header_name(&self) -> &'static str {
+        CRC_32_C_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Crc64Nvme {
+    fn header_name(&self) -> &'static str {
+        CRC_64_NVME_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Sha1 {
+    fn header_name(&self) -> &'static str {
+        SHA_1_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Sha256 {
+    fn header_name(&self) -> &'static str {
+        SHA_256_HEADER_NAME
+    }
+}
+
+impl HttpChecksum for Md5 {
+    fn header_name(&self) -> &'static str {
+        MD5_HEADER_NAME
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::base64;
+    use bytes::Bytes;
+
+    use crate::{CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, ChecksumAlgorithm, SHA_1_NAME, SHA_256_NAME};
+
+    use super::HttpChecksum;
+
+    #[test]
+    fn test_trailer_length_of_crc32_checksum_body() {
+        let checksum = CRC_32_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_size = 29;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_crc32_checksum_body() {
+        let checksum = CRC_32_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        // The CRC32 of an empty string is all zeroes
+        let expected_value = Bytes::from_static(b"\0\0\0\0");
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+
+    #[test]
+    fn test_trailer_length_of_crc32c_checksum_body() {
+        let checksum = CRC_32_C_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_size = 30;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_crc32c_checksum_body() {
+        let checksum = CRC_32_C_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        // The CRC32C of an empty string is all zeroes
+        let expected_value = Bytes::from_static(b"\0\0\0\0");
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+
+    #[test]
+    fn test_trailer_length_of_crc64nvme_checksum_body() {
+        let checksum = CRC_64_NVME_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_size = 37;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_crc64nvme_checksum_body() {
+        let checksum = CRC_64_NVME_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        // The CRC64NVME of an empty string is all zeroes
+        let expected_value = Bytes::from_static(b"\0\0\0\0\0\0\0\0");
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+
+    #[test]
+    fn test_trailer_length_of_sha1_checksum_body() {
+        let checksum = SHA_1_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_size = 48;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_sha1_checksum_body() {
+        let checksum = SHA_1_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        // The SHA1 of an empty string is da39a3ee5e6b4b0d3255bfef95601890afd80709
+        let expected_value = Bytes::from_static(&[
+            0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07,
+            0x09,
+        ]);
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+
+    #[test]
+    fn test_trailer_length_of_sha256_checksum_body() {
+        let checksum = SHA_256_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_size = 66;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_sha256_checksum_body() {
+        let checksum = SHA_256_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();
+        let expected_value = Bytes::from_static(&[
+            0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41,
+            0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
+        ]);
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+}

crates/checksums/src/lib.rs

@@ -0,0 +1,446 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![allow(clippy::derive_partial_eq_without_eq)]
+#![warn(
+    // missing_docs,
+    rustdoc::missing_crate_level_docs,
+    unreachable_pub,
+    rust_2018_idioms
+)]
+
+use crate::error::UnknownChecksumAlgorithmError;
+
+use bytes::Bytes;
+use std::{fmt::Debug, str::FromStr};
+
+mod base64;
+pub mod error;
+pub mod http;
+
+pub const CRC_32_NAME: &str = "crc32";
+pub const CRC_32_C_NAME: &str = "crc32c";
+pub const CRC_64_NVME_NAME: &str = "crc64nvme";
+pub const SHA_1_NAME: &str = "sha1";
+pub const SHA_256_NAME: &str = "sha256";
+pub const MD5_NAME: &str = "md5";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[non_exhaustive]
+pub enum ChecksumAlgorithm {
+    #[default]
+    Crc32,
+    Crc32c,
+    #[deprecated]
+    Md5,
+    Sha1,
+    Sha256,
+    Crc64Nvme,
+}
+
+impl FromStr for ChecksumAlgorithm {
+    type Err = UnknownChecksumAlgorithmError;
+
+    fn from_str(checksum_algorithm: &str) -> Result<Self, Self::Err> {
+        if checksum_algorithm.eq_ignore_ascii_case(CRC_32_NAME) {
+            Ok(Self::Crc32)
+        } else if checksum_algorithm.eq_ignore_ascii_case(CRC_32_C_NAME) {
+            Ok(Self::Crc32c)
+        } else if checksum_algorithm.eq_ignore_ascii_case(SHA_1_NAME) {
+            Ok(Self::Sha1)
+        } else if checksum_algorithm.eq_ignore_ascii_case(SHA_256_NAME) {
+            Ok(Self::Sha256)
+        } else if checksum_algorithm.eq_ignore_ascii_case(MD5_NAME) {
+            // MD5 is now an alias for the default Crc32 since it is deprecated
+            Ok(Self::Crc32)
+        } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) {
+            Ok(Self::Crc64Nvme)
+        } else {
+            Err(UnknownChecksumAlgorithmError::new(checksum_algorithm))
+        }
+    }
+}
+
+impl ChecksumAlgorithm {
+    pub fn into_impl(self) -> Box<dyn http::HttpChecksum> {
+        match self {
+            Self::Crc32 => Box::<Crc32>::default(),
+            Self::Crc32c => Box::<Crc32c>::default(),
+            Self::Crc64Nvme => Box::<Crc64Nvme>::default(),
+            #[allow(deprecated)]
+            Self::Md5 => Box::<Crc32>::default(),
+            Self::Sha1 => Box::<Sha1>::default(),
+            Self::Sha256 => Box::<Sha256>::default(),
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Crc32 => CRC_32_NAME,
+            Self::Crc32c => CRC_32_C_NAME,
+            Self::Crc64Nvme => CRC_64_NVME_NAME,
+            #[allow(deprecated)]
+            Self::Md5 => MD5_NAME,
+            Self::Sha1 => SHA_1_NAME,
+            Self::Sha256 => SHA_256_NAME,
+        }
+    }
+}
+
+pub trait Checksum: Send + Sync {
+    fn update(&mut self, bytes: &[u8]);
+    fn finalize(self: Box<Self>) -> Bytes;
+    fn size(&self) -> u64;
+}
+
+#[derive(Debug)]
+struct Crc32 {
+    hasher: crc_fast::Digest,
+}
+
+impl Default for Crc32 {
+    fn default() -> Self {
+        Self {
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc),
+        }
+    }
+}
+
+impl Crc32 {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        let checksum = self.hasher.finalize() as u32;
+
+        Bytes::copy_from_slice(checksum.to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        4
+    }
+}
+
+impl Checksum for Crc32 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[derive(Debug)]
+struct Crc32c {
+    hasher: crc_fast::Digest,
+}
+
+impl Default for Crc32c {
+    fn default() -> Self {
+        Self {
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi),
+        }
+    }
+}
+
+impl Crc32c {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        let checksum = self.hasher.finalize() as u32;
+
+        Bytes::copy_from_slice(checksum.to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        4
+    }
+}
+
+impl Checksum for Crc32c {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[derive(Debug)]
+struct Crc64Nvme {
+    hasher: crc_fast::Digest,
+}
+
+impl Default for Crc64Nvme {
+    fn default() -> Self {
+        Self {
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme),
+        }
+    }
+}
+
+impl Crc64Nvme {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        Bytes::copy_from_slice(self.hasher.finalize().to_be_bytes().as_slice())
+    }
+
+    fn size() -> u64 {
+        8
+    }
+}
+
+impl Checksum for Crc64Nvme {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[derive(Debug, Default)]
+struct Sha1 {
+    hasher: sha1::Sha1,
+}
+
+impl Sha1 {
+    fn update(&mut self, bytes: &[u8]) {
+        use sha1::Digest;
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        use sha1::Digest;
+        Bytes::copy_from_slice(self.hasher.finalize().as_slice())
+    }
+
+    fn size() -> u64 {
+        use sha1::Digest;
+        sha1::Sha1::output_size() as u64
+    }
+}
+
+impl Checksum for Sha1 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[derive(Debug, Default)]
+struct Sha256 {
+    hasher: sha2::Sha256,
+}
+
+impl Sha256 {
+    fn update(&mut self, bytes: &[u8]) {
+        use sha2::Digest;
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        use sha2::Digest;
+        Bytes::copy_from_slice(self.hasher.finalize().as_slice())
+    }
+
+    fn size() -> u64 {
+        use sha2::Digest;
+        sha2::Sha256::output_size() as u64
+    }
+}
+
+impl Checksum for Sha256 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes);
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[derive(Debug, Default)]
+struct Md5 {
+    hasher: md5::Md5,
+}
+
+impl Md5 {
+    fn update(&mut self, bytes: &[u8]) {
+        use md5::Digest;
+        self.hasher.update(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        use md5::Digest;
+        Bytes::copy_from_slice(self.hasher.finalize().as_slice())
+    }
+
+    fn size() -> u64 {
+        use md5::Digest;
+        md5::Md5::output_size() as u64
+    }
+}
+
+impl Checksum for Md5 {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        Crc32, Crc32c, Md5, Sha1, Sha256,
+        http::{CRC_32_C_HEADER_NAME, CRC_32_HEADER_NAME, MD5_HEADER_NAME, SHA_1_HEADER_NAME, SHA_256_HEADER_NAME},
+    };
+
+    use crate::ChecksumAlgorithm;
+    use crate::http::HttpChecksum;
+
+    use crate::base64;
+    use http::HeaderValue;
+    use pretty_assertions::assert_eq;
+    use std::fmt::Write;
+
+    const TEST_DATA: &str = r#"test data"#;
+
+    fn base64_encoded_checksum_to_hex_string(header_value: &HeaderValue) -> String {
+        let decoded_checksum = base64::decode(header_value.to_str().unwrap()).unwrap();
+        let decoded_checksum = decoded_checksum.into_iter().fold(String::new(), |mut acc, byte| {
+            write!(acc, "{byte:02X?}").expect("string will always be writeable");
+            acc
+        });
+
+        format!("0x{decoded_checksum}")
+    }
+
+    #[test]
+    fn test_crc32_checksum() {
+        let mut checksum = Crc32::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(CRC_32_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0xD308AEB2";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))]
+    #[test]
+    fn test_crc32c_checksum() {
+        let mut checksum = Crc32c::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(CRC_32_C_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0x3379B4CA";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[test]
+    fn test_crc64nvme_checksum() {
+        use crate::{Crc64Nvme, http::CRC_64_NVME_HEADER_NAME};
+        let mut checksum = Crc64Nvme::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(CRC_64_NVME_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0xAECAF3AF9C98A855";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[test]
+    fn test_sha1_checksum() {
+        let mut checksum = Sha1::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(SHA_1_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0xF48DD853820860816C75D54D0F584DC863327A7C";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[test]
+    fn test_sha256_checksum() {
+        let mut checksum = Sha256::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(SHA_256_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0x916F0027A575074CE72A331777C3478D6513F786A591BD892DA1A577BF2335F9";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[test]
+    fn test_md5_checksum() {
+        let mut checksum = Md5::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(MD5_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0xEB733A00C0C9D336E65691A37AB54293";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
+    #[test]
+    fn test_checksum_algorithm_returns_error_for_unknown() {
+        let error = "some invalid checksum algorithm"
+            .parse::<ChecksumAlgorithm>()
+            .expect_err("it should error");
+        assert_eq!("some invalid checksum algorithm", error.checksum_algorithm());
+    }
+}

crates/common/Cargo.toml

@@ -28,5 +28,15 @@ categories = ["web-programming", "development-tools", "data-structures"]
 workspace = true
 
 [dependencies]
-tokio.workspace = true
+lazy_static = { workspace = true}
+tokio = { workspace = true }
 tonic = { workspace = true }
+uuid = { workspace = true }
+chrono = { workspace = true }
+rustfs-madmin = { workspace = true }
+rustfs-filemeta = { workspace = true }
+serde = { workspace = true }
+path-clean = { workspace = true }
+rmp-serde = { workspace = true }
+async-trait = { workspace = true }
+s3s = { workspace = true }

crates/common/src/heal_channel.rs

@@ -0,0 +1,427 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::{self, Display},
+    sync::OnceLock,
+};
+use tokio::sync::mpsc;
+use uuid::Uuid;
+
+pub const HEAL_DELETE_DANGLING: bool = true;
+pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs";
+pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub enum HealItemType {
+    Metadata,
+    Bucket,
+    BucketMetadata,
+    Object,
+}
+
+impl HealItemType {
+    pub fn to_str(&self) -> &str {
+        match self {
+            HealItemType::Metadata => "metadata",
+            HealItemType::Bucket => "bucket",
+            HealItemType::BucketMetadata => "bucket-metadata",
+            HealItemType::Object => "object",
+        }
+    }
+}
+
+impl Display for HealItemType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_str())
+    }
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub enum DriveState {
+    Ok,
+    Offline,
+    Corrupt,
+    Missing,
+    PermissionDenied,
+    Faulty,
+    RootMount,
+    Unknown,
+    Unformatted, // only returned by disk
+}
+
+impl DriveState {
+    pub fn to_str(&self) -> &str {
+        match self {
+            DriveState::Ok => "ok",
+            DriveState::Offline => "offline",
+            DriveState::Corrupt => "corrupt",
+            DriveState::Missing => "missing",
+            DriveState::PermissionDenied => "permission-denied",
+            DriveState::Faulty => "faulty",
+            DriveState::RootMount => "root-mount",
+            DriveState::Unknown => "unknown",
+            DriveState::Unformatted => "unformatted",
+        }
+    }
+}
+
+impl Display for DriveState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.to_str())
+    }
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum HealScanMode {
+    Unknown,
+    Normal,
+    Deep,
+}
+
+impl Default for HealScanMode {
+    fn default() -> Self {
+        Self::Normal
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)]
+pub struct HealOpts {
+    pub recursive: bool,
+    #[serde(rename = "dryRun")]
+    pub dry_run: bool,
+    pub remove: bool,
+    pub recreate: bool,
+    #[serde(rename = "scanMode")]
+    pub scan_mode: HealScanMode,
+    #[serde(rename = "updateParity")]
+    pub update_parity: bool,
+    #[serde(rename = "nolock")]
+    pub no_lock: bool,
+    pub pool: Option<usize>,
+    pub set: Option<usize>,
+}
+
+/// Heal channel command type
+#[derive(Debug, Clone)]
+pub enum HealChannelCommand {
+    /// Start a new heal task
+    Start(HealChannelRequest),
+    /// Query heal task status
+    Query { heal_path: String, client_token: String },
+    /// Cancel heal task
+    Cancel { heal_path: String },
+}
+
+/// Heal request from admin to ahm
+#[derive(Debug, Clone, Default)]
+pub struct HealChannelRequest {
+    /// Unique request ID
+    pub id: String,
+    /// Disk ID for heal disk/erasure set task
+    pub disk: Option<String>,
+    /// Bucket name
+    pub bucket: String,
+    /// Object prefix (optional)
+    pub object_prefix: Option<String>,
+    /// Force start heal
+    pub force_start: bool,
+    /// Priority
+    pub priority: HealChannelPriority,
+    /// Pool index (optional)
+    pub pool_index: Option<usize>,
+    /// Set index (optional)
+    pub set_index: Option<usize>,
+    /// Scan mode (optional)
+    pub scan_mode: Option<HealScanMode>,
+    /// Whether to remove corrupted data
+    pub remove_corrupted: Option<bool>,
+    /// Whether to recreate missing data
+    pub recreate_missing: Option<bool>,
+    /// Whether to update parity
+    pub update_parity: Option<bool>,
+    /// Whether to recursively process
+    pub recursive: Option<bool>,
+    /// Whether to dry run
+    pub dry_run: Option<bool>,
+    /// Timeout in seconds (optional)
+    pub timeout_seconds: Option<u64>,
+}
+
+/// Heal response from ahm to admin
+#[derive(Debug, Clone)]
+pub struct HealChannelResponse {
+    /// Request ID
+    pub request_id: String,
+    /// Success status
+    pub success: bool,
+    /// Response data (if successful)
+    pub data: Option<Vec<u8>>,
+    /// Error message (if failed)
+    pub error: Option<String>,
+}
+
+/// Heal priority
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealChannelPriority {
+    /// Low priority
+    Low,
+    /// Normal priority
+    Normal,
+    /// High priority
+    High,
+    /// Critical priority
+    Critical,
+}
+
+impl Default for HealChannelPriority {
+    fn default() -> Self {
+        Self::Normal
+    }
+}
+
+/// Heal channel sender
+pub type HealChannelSender = mpsc::UnboundedSender<HealChannelCommand>;
+
+/// Heal channel receiver
+pub type HealChannelReceiver = mpsc::UnboundedReceiver<HealChannelCommand>;
+
+/// Global heal channel sender
+static GLOBAL_HEAL_CHANNEL_SENDER: OnceLock<HealChannelSender> = OnceLock::new();
+
+/// Initialize global heal channel
+pub fn init_heal_channel() -> HealChannelReceiver {
+    let (tx, rx) = mpsc::unbounded_channel();
+    GLOBAL_HEAL_CHANNEL_SENDER
+        .set(tx)
+        .expect("Heal channel sender already initialized");
+    rx
+}
+
+/// Get global heal channel sender
+pub fn get_heal_channel_sender() -> Option<&'static HealChannelSender> {
+    GLOBAL_HEAL_CHANNEL_SENDER.get()
+}
+
+/// Send heal command through global channel
+pub async fn send_heal_command(command: HealChannelCommand) -> Result<(), String> {
+    if let Some(sender) = get_heal_channel_sender() {
+        sender
+            .send(command)
+            .map_err(|e| format!("Failed to send heal command: {e}"))?;
+        Ok(())
+    } else {
+        Err("Heal channel not initialized".to_string())
+    }
+}
+
+/// Send heal start request
+pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
+    send_heal_command(HealChannelCommand::Start(request)).await
+}
+
+/// Send heal query request
+pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<(), String> {
+    send_heal_command(HealChannelCommand::Query { heal_path, client_token }).await
+}
+
+/// Send heal cancel request
+pub async fn cancel_heal_task(heal_path: String) -> Result<(), String> {
+    send_heal_command(HealChannelCommand::Cancel { heal_path }).await
+}
+
+/// Create a new heal request
+pub fn create_heal_request(
+    bucket: String,
+    object_prefix: Option<String>,
+    force_start: bool,
+    priority: Option<HealChannelPriority>,
+) -> HealChannelRequest {
+    HealChannelRequest {
+        id: Uuid::new_v4().to_string(),
+        bucket,
+        object_prefix,
+        force_start,
+        priority: priority.unwrap_or_default(),
+        pool_index: None,
+        set_index: None,
+        scan_mode: None,
+        remove_corrupted: None,
+        recreate_missing: None,
+        update_parity: None,
+        recursive: None,
+        dry_run: None,
+        timeout_seconds: None,
+        disk: None,
+    }
+}
+
+/// Create a new heal request with advanced options
+pub fn create_heal_request_with_options(
+    bucket: String,
+    object_prefix: Option<String>,
+    force_start: bool,
+    priority: Option<HealChannelPriority>,
+    pool_index: Option<usize>,
+    set_index: Option<usize>,
+) -> HealChannelRequest {
+    HealChannelRequest {
+        id: Uuid::new_v4().to_string(),
+        bucket,
+        object_prefix,
+        force_start,
+        priority: priority.unwrap_or_default(),
+        pool_index,
+        set_index,
+        ..Default::default()
+    }
+}
+
+/// Create a heal response
+pub fn create_heal_response(
+    request_id: String,
+    success: bool,
+    data: Option<Vec<u8>>,
+    error: Option<String>,
+) -> HealChannelResponse {
+    HealChannelResponse {
+        request_id,
+        success,
+        data,
+        error,
+    }
+}
+
+fn lc_get_prefix(rule: &LifecycleRule) -> String {
+    if let Some(p) = &rule.prefix {
+        return p.to_string();
+    } else if let Some(filter) = &rule.filter {
+        if let Some(p) = &filter.prefix {
+            return p.to_string();
+        } else if let Some(and) = &filter.and {
+            if let Some(p) = &and.prefix {
+                return p.to_string();
+            }
+        }
+    }
+
+    "".into()
+}
+
+pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool {
+    if config.rules.is_empty() {
+        return false;
+    }
+
+    for rule in config.rules.iter() {
+        if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) {
+            continue;
+        }
+        let rule_prefix = lc_get_prefix(rule);
+        if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix)
+        {
+            continue;
+        }
+
+        if let Some(e) = &rule.noncurrent_version_expiration {
+            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+                return true;
+            }
+            if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
+                return true;
+            }
+        }
+
+        if rule.noncurrent_version_transitions.is_some() {
+            return true;
+        }
+        if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) {
+            return true;
+        }
+
+        if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) {
+            return true;
+        }
+
+        if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) {
+            return true;
+        }
+
+        if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) {
+            return true;
+        }
+
+        if rule.transitions.is_some() {
+            return true;
+        }
+    }
+    false
+}
+
+pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
+    if config.rules.is_empty() {
+        return false;
+    }
+
+    for rule in config.rules.iter() {
+        if rule
+            .status
+            .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED))
+        {
+            continue;
+        }
+        if !prefix.is_empty() {
+            if let Some(filter) = &rule.filter {
+                if let Some(r_prefix) = &filter.prefix {
+                    if !r_prefix.is_empty() {
+                        // incoming prefix must be in rule prefix
+                        if !recursive && !prefix.starts_with(r_prefix) {
+                            continue;
+                        }
+                        // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix
+                        // does not match
+                        if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) {
+                            continue;
+                        }
+                    }
+                }
+            }
+        }
+        return true;
+    }
+    false
+}
+
+pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
+    let req = HealChannelRequest {
+        id: Uuid::new_v4().to_string(),
+        bucket: "".to_string(),
+        object_prefix: None,
+        disk: Some(set_disk_id),
+        force_start: false,
+        priority: priority.unwrap_or_default(),
+        pool_index: None,
+        set_index: None,
+        scan_mode: None,
+        remove_corrupted: None,
+        recreate_missing: None,
+        update_parity: None,
+        recursive: None,
+        dry_run: None,
+        timeout_seconds: None,
+    };
+    send_heal_request(req).await
+}

crates/common/src/lib.rs

@@ -14,8 +14,11 @@
 
 pub mod bucket_stats;
 // pub mod error;
+pub mod data_usage;
 pub mod globals;
+pub mod heal_channel;
 pub mod last_minute;
+pub mod metrics;
 
 // is ','
 pub static DEFAULT_DELIMITER: u8 = 44;

crates/common/src/metrics.rs

@@ -12,14 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::data_scanner::CurrentScannerCycle;
-use crate::bucket::lifecycle::lifecycle;
-use chrono::Utc;
+use chrono::{DateTime, Utc};
 use lazy_static::lazy_static;
-use rustfs_common::last_minute::{AccElem, LastMinuteLatency};
 use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
 use std::{
     collections::HashMap,
+    fmt::Display,
     pin::Pin,
     sync::{
         Arc,
@@ -29,12 +27,58 @@ use std::{
 };
 use tokio::sync::{Mutex, RwLock};
 
+use crate::last_minute::{AccElem, LastMinuteLatency};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum IlmAction {
+    NoneAction = 0,
+    DeleteAction,
+    DeleteVersionAction,
+    TransitionAction,
+    TransitionVersionAction,
+    DeleteRestoredAction,
+    DeleteRestoredVersionAction,
+    DeleteAllVersionsAction,
+    DelMarkerDeleteAllVersionsAction,
+    ActionCount,
+}
+
+impl IlmAction {
+    pub fn delete_restored(&self) -> bool {
+        *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_versioned(&self) -> bool {
+        *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_all(&self) -> bool {
+        *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+
+    pub fn delete(&self) -> bool {
+        if self.delete_restored() {
+            return true;
+        }
+        *self == Self::DeleteVersionAction
+            || *self == Self::DeleteAction
+            || *self == Self::DeleteAllVersionsAction
+            || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+}
+
+impl Display for IlmAction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{self:?}")
+    }
+}
+
 lazy_static! {
-    pub static ref globalScannerMetrics: Arc<ScannerMetrics> = Arc::new(ScannerMetrics::new());
+    pub static ref globalMetrics: Arc<Metrics> = Arc::new(Metrics::new());
 }
 
 #[derive(Clone, Debug, PartialEq, PartialOrd)]
-pub enum ScannerMetric {
+pub enum Metric {
     // START Realtime metrics, that only records
     // last minute latencies and total operation count.
     ReadMetadata = 0,
@@ -69,7 +113,7 @@ pub enum ScannerMetric {
     Last,
 }
 
-impl ScannerMetric {
+impl Metric {
     /// Convert to string representation for metrics
     pub fn as_str(self) -> &'static str {
         match self {
@@ -203,7 +247,7 @@ impl CurrentPathTracker {
 }
 
 /// Main scanner metrics structure
-pub struct ScannerMetrics {
+pub struct Metrics {
     // All fields must be accessed atomically and aligned.
     operations: Vec<AtomicU64>,
     latency: Vec<LockedLastMinuteLatency>,
@@ -213,94 +257,102 @@ pub struct ScannerMetrics {
     current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
 
     // Cycle information
-    cycle_info: Arc<RwLock<Option<CurrentScannerCycle>>>,
+    cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }
 
-impl ScannerMetrics {
-    pub fn new() -> Self {
-        let operations = (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect();
+// This is a placeholder. We'll need to define this struct.
+#[derive(Clone, Debug)]
+pub struct CurrentCycle {
+    pub current: u64,
+    pub cycle_completed: Vec<DateTime<Utc>>,
+    pub started: DateTime<Utc>,
+}
 
-        let latency = (0..ScannerMetric::LastRealtime as usize)
+impl Metrics {
+    pub fn new() -> Self {
+        let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
+
+        let latency = (0..Metric::LastRealtime as usize)
             .map(|_| LockedLastMinuteLatency::new())
             .collect();
 
         Self {
             operations,
             latency,
-            actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
-            actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
+            actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
+            actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize],
             current_paths: Arc::new(RwLock::new(HashMap::new())),
             cycle_info: Arc::new(RwLock::new(None)),
         }
     }
 
     /// Log scanner action with custom metadata - compatible with existing usage
-    pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap<String, String>) {
+    pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
         let metric = metric as usize;
         let start_time = SystemTime::now();
         move |_custom: &HashMap<String, String>| {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+            globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics (spawn async task for this)
-            if (metric) < ScannerMetric::LastRealtime as usize {
+            if (metric) < Metric::LastRealtime as usize {
                 let metric_index = metric;
                 tokio::spawn(async move {
-                    globalScannerMetrics.latency[metric_index].add(duration).await;
+                    globalMetrics.latency[metric_index].add(duration).await;
                 });
             }
 
             // Log trace metrics
-            if metric as u8 > ScannerMetric::StartTrace as u8 {
+            if metric as u8 > Metric::StartTrace as u8 {
                 //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
             }
         }
     }
 
     /// Time scanner action with size - returns function that takes size
-    pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) {
+    pub fn time_size(metric: Metric) -> impl Fn(u64) {
         let metric = metric as usize;
         let start_time = SystemTime::now();
         move |size: u64| {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+            globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics with size (spawn async task)
-            if (metric) < ScannerMetric::LastRealtime as usize {
+            if (metric) < Metric::LastRealtime as usize {
                 let metric_index = metric;
                 tokio::spawn(async move {
-                    globalScannerMetrics.latency[metric_index].add_size(duration, size).await;
+                    globalMetrics.latency[metric_index].add_size(duration, size).await;
                 });
             }
         }
     }
 
     /// Time a scanner action - returns a closure to call when done
-    pub fn time(metric: ScannerMetric) -> impl Fn() {
+    pub fn time(metric: Metric) -> impl Fn() {
         let metric = metric as usize;
         let start_time = SystemTime::now();
         move || {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+            globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics (spawn async task)
-            if (metric) < ScannerMetric::LastRealtime as usize {
+            if (metric) < Metric::LastRealtime as usize {
                 let metric_index = metric;
                 tokio::spawn(async move {
-                    globalScannerMetrics.latency[metric_index].add(duration).await;
+                    globalMetrics.latency[metric_index].add(duration).await;
                 });
             }
         }
     }
 
     /// Time N scanner actions - returns function that takes count, then returns completion function
-    pub fn time_n(metric: ScannerMetric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+    pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
         let metric = metric as usize;
         let start_time = SystemTime::now();
         Box::new(move |count: usize| {
@@ -308,22 +360,23 @@ impl ScannerMetrics {
                 let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
                 // Update operation count
-                globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed);
+                globalMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed);
 
                 // Update latency for realtime metrics (spawn async task)
-                if (metric) < ScannerMetric::LastRealtime as usize {
+                if (metric) < Metric::LastRealtime as usize {
                     let metric_index = metric;
                     tokio::spawn(async move {
-                        globalScannerMetrics.latency[metric_index].add(duration).await;
+                        globalMetrics.latency[metric_index].add(duration).await;
                     });
                 }
             })
         })
     }
 
-    pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+    /// Time ILM action with versions - returns function that takes versions, then returns completion function
+    pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
         let a_clone = a as usize;
-        if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
+        if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize {
             return Box::new(move |_: u64| Box::new(move || {}));
         }
         let start = SystemTime::now();
@@ -331,50 +384,50 @@ impl ScannerMetrics {
             Box::new(move || {
                 let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
                 tokio::spawn(async move {
-                    globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed);
-                    globalScannerMetrics.actions_latency[a_clone].add(duration).await;
+                    globalMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed);
+                    globalMetrics.actions_latency[a_clone].add(duration).await;
                 });
             })
         })
     }
 
     /// Increment time with specific duration
-    pub async fn inc_time(metric: ScannerMetric, duration: Duration) {
+    pub async fn inc_time(metric: Metric, duration: Duration) {
         let metric = metric as usize;
         // Update operation count
-        globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
+        globalMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
         // Update latency for realtime metrics
-        if (metric) < ScannerMetric::LastRealtime as usize {
-            globalScannerMetrics.latency[metric].add(duration).await;
+        if (metric) < Metric::LastRealtime as usize {
+            globalMetrics.latency[metric].add(duration).await;
         }
     }
 
     /// Get lifetime operation count for a metric
-    pub fn lifetime(&self, metric: ScannerMetric) -> u64 {
+    pub fn lifetime(&self, metric: Metric) -> u64 {
         let metric = metric as usize;
-        if (metric) >= ScannerMetric::Last as usize {
+        if (metric) >= Metric::Last as usize {
             return 0;
         }
         self.operations[metric].load(Ordering::Relaxed)
     }
 
     /// Get last minute statistics for a metric
-    pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem {
+    pub async fn last_minute(&self, metric: Metric) -> AccElem {
         let metric = metric as usize;
-        if (metric) >= ScannerMetric::LastRealtime as usize {
+        if (metric) >= Metric::LastRealtime as usize {
             return AccElem::default();
         }
         self.latency[metric].total().await
     }
 
     /// Set current cycle information
-    pub async fn set_cycle(&self, cycle: Option<CurrentScannerCycle>) {
+    pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
         *self.cycle_info.write().await = cycle;
     }
 
     /// Get current cycle information
-    pub async fn get_cycle(&self) -> Option<CurrentScannerCycle> {
+    pub async fn get_cycle(&self) -> Option<CurrentCycle> {
         self.cycle_info.read().await.clone()
     }
 
@@ -411,20 +464,20 @@ impl ScannerMetrics {
         metrics.active_paths = self.get_current_paths().await;
 
         // Lifetime operations
-        for i in 0..ScannerMetric::Last as usize {
+        for i in 0..Metric::Last as usize {
             let count = self.operations[i].load(Ordering::Relaxed);
             if count > 0 {
-                if let Some(metric) = ScannerMetric::from_index(i) {
+                if let Some(metric) = Metric::from_index(i) {
                     metrics.life_time_ops.insert(metric.as_str().to_string(), count);
                 }
             }
         }
 
         // Last minute statistics for realtime metrics
-        for i in 0..ScannerMetric::LastRealtime as usize {
+        for i in 0..Metric::LastRealtime as usize {
             let last_min = self.latency[i].total().await;
             if last_min.n > 0 {
-                if let Some(_metric) = ScannerMetric::from_index(i) {
+                if let Some(_metric) = Metric::from_index(i) {
                     // Convert to madmin TimedAction format if needed
                     // This would require implementing the conversion
                 }
@@ -448,11 +501,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
     let tracker_clone = Arc::clone(&tracker);
     let disk_clone = disk_name.clone();
     tokio::spawn(async move {
-        globalScannerMetrics
-            .current_paths
-            .write()
-            .await
-            .insert(disk_clone, tracker_clone);
+        globalMetrics.current_paths.write().await.insert(disk_clone, tracker_clone);
     });
 
     let update_fn = {
@@ -471,7 +520,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
         Arc::new(move || -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
             let disk_name = disk_name.clone();
             Box::pin(async move {
-                globalScannerMetrics.current_paths.write().await.remove(&disk_name);
+                globalMetrics.current_paths.write().await.remove(&disk_name);
             })
         })
     };
@@ -479,7 +528,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
     (update_fn, done_fn)
 }
 
-impl Default for ScannerMetrics {
+impl Default for Metrics {
     fn default() -> Self {
         Self::new()
     }

crates/config/Cargo.toml

@@ -26,9 +26,6 @@ categories = ["web-programming", "development-tools", "config"]
 
 [dependencies]
 const-str = { workspace = true, optional = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
-
 
 [lints]
 workspace = true

crates/config/src/constants/app.rs

@@ -15,9 +15,9 @@
 use const_str::concat;
 
 /// Application name
-/// Default value: RustFs
+/// Default value: RustFS
 /// Environment variable: RUSTFS_APP_NAME
-pub const APP_NAME: &str = "RustFs";
+pub const APP_NAME: &str = "RustFS";
 /// Application version
 /// Default value: 1.0.0
 /// Environment variable: RUSTFS_VERSION
@@ -71,6 +71,16 @@ pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
 /// Example: --secret-key rustfsadmin
 pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
 
+/// Default console enable
+/// This is the default value for the console server.
+/// It is used to enable or disable the console server.
+/// Default value: true
+/// Environment variable: RUSTFS_CONSOLE_ENABLE
+/// Command line argument: --console-enable
+/// Example: RUSTFS_CONSOLE_ENABLE=true
+/// Example: --console-enable true
+pub const DEFAULT_CONSOLE_ENABLE: bool = true;
+
 /// Default OBS configuration endpoint
 /// Environment variable: DEFAULT_OBS_ENDPOINT
 /// Command line argument: --obs-endpoint
@@ -108,34 +118,46 @@ pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);
 /// It is used to store the logs of the application.
 /// Default value: rustfs.log
 /// Environment variable: RUSTFS_OBSERVABILITY_LOG_FILENAME
-pub const DEFAULT_LOG_FILENAME: &str = "rustfs.log";
+pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
+
+/// Default OBS log filename for rustfs
+/// This is the default log filename for OBS.
+/// It is used to store the logs of the application.
+/// Default value: rustfs.log
+pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, ".log");
+
+/// Default sink file log file for rustfs
+/// This is the default sink file log file for rustfs.
+/// It is used to store the logs of the application.
+/// Default value: rustfs-sink.log
+pub const DEFAULT_SINK_FILE_LOG_FILE: &str = concat!(DEFAULT_LOG_FILENAME, "-sink.log");
 
 /// Default log directory for rustfs
 /// This is the default log directory for rustfs.
 /// It is used to store the logs of the application.
 /// Default value: logs
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_DIRECTORY
-pub const DEFAULT_LOG_DIR: &str = "deploy/logs";
+/// Environment variable: RUSTFS_LOG_DIRECTORY
+pub const DEFAULT_LOG_DIR: &str = "logs";
 
 /// Default log rotation size mb for rustfs
 /// This is the default log rotation size for rustfs.
 /// It is used to rotate the logs of the application.
 /// Default value: 100 MB
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_ROTATION_SIZE_MB
+/// Environment variable: RUSTFS_OBS_LOG_ROTATION_SIZE_MB
 pub const DEFAULT_LOG_ROTATION_SIZE_MB: u64 = 100;
 
 /// Default log rotation time for rustfs
 /// This is the default log rotation time for rustfs.
 /// It is used to rotate the logs of the application.
 /// Default value: hour, eg: day,hour,minute,second
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_ROTATION_TIME
+/// Environment variable: RUSTFS_OBS_LOG_ROTATION_TIME
 pub const DEFAULT_LOG_ROTATION_TIME: &str = "day";
 
 /// Default log keep files for rustfs
 /// This is the default log keep files for rustfs.
 /// It is used to keep the logs of the application.
 /// Default value: 30
-/// Environment variable: RUSTFS_OBSERVABILITY_LOG_KEEP_FILES
+/// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: u16 = 30;
 
 #[cfg(test)]
@@ -145,7 +167,7 @@ mod tests {
     #[test]
     fn test_app_basic_constants() {
         // Test application basic constants
-        assert_eq!(APP_NAME, "RustFs");
+        assert_eq!(APP_NAME, "RustFS");
         assert!(!APP_NAME.contains(' '), "App name should not contain spaces");
 
         assert_eq!(VERSION, "0.0.1");

crates/config/src/notify/mod.rs

@@ -27,7 +27,15 @@ pub const DEFAULT_TARGET: &str = "1";
 
 pub const NOTIFY_PREFIX: &str = "notify";
 
-pub const NOTIFY_ROUTE_PREFIX: &str = "notify_";
+pub const NOTIFY_ROUTE_PREFIX: &str = const_str::concat!(NOTIFY_PREFIX, "_");
+
+/// Standard config keys and values.
+pub const ENABLE_KEY: &str = "enable";
+pub const COMMENT_KEY: &str = "comment";
+
+/// Enable values
+pub const ENABLE_ON: &str = "on";
+pub const ENABLE_OFF: &str = "off";
 
 #[allow(dead_code)]
 pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];

crates/config/src/notify/mqtt.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::notify::{COMMENT_KEY, ENABLE_KEY};
+
 // MQTT Keys
 pub const MQTT_BROKER: &str = "broker";
 pub const MQTT_TOPIC: &str = "topic";
@@ -23,6 +25,21 @@ pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
 pub const MQTT_QUEUE_DIR: &str = "queue_dir";
 pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
 
+/// A list of all valid configuration keys for an MQTT target.
+pub const NOTIFY_MQTT_KEYS: &[&str] = &[
+    ENABLE_KEY, // "enable" is a common key
+    MQTT_BROKER,
+    MQTT_TOPIC,
+    MQTT_QOS,
+    MQTT_USERNAME,
+    MQTT_PASSWORD,
+    MQTT_RECONNECT_INTERVAL,
+    MQTT_KEEP_ALIVE_INTERVAL,
+    MQTT_QUEUE_DIR,
+    MQTT_QUEUE_LIMIT,
+    COMMENT_KEY,
+];
+
 // MQTT Environment Variables
 pub const ENV_MQTT_ENABLE: &str = "RUSTFS_NOTIFY_MQTT_ENABLE";
 pub const ENV_MQTT_BROKER: &str = "RUSTFS_NOTIFY_MQTT_BROKER";
@@ -34,3 +51,16 @@ pub const ENV_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNECT_INTE
 pub const ENV_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
 pub const ENV_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
 pub const ENV_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
+    ENV_MQTT_ENABLE,
+    ENV_MQTT_BROKER,
+    ENV_MQTT_TOPIC,
+    ENV_MQTT_QOS,
+    ENV_MQTT_USERNAME,
+    ENV_MQTT_PASSWORD,
+    ENV_MQTT_RECONNECT_INTERVAL,
+    ENV_MQTT_KEEP_ALIVE_INTERVAL,
+    ENV_MQTT_QUEUE_DIR,
+    ENV_MQTT_QUEUE_LIMIT,
+];

crates/config/src/notify/webhook.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::notify::{COMMENT_KEY, ENABLE_KEY};
+
 // Webhook Keys
 pub const WEBHOOK_ENDPOINT: &str = "endpoint";
 pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
@@ -20,6 +22,18 @@ pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
 pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
 pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
 
+/// A list of all valid configuration keys for a webhook target.
+pub const NOTIFY_WEBHOOK_KEYS: &[&str] = &[
+    ENABLE_KEY, // "enable" is a common key
+    WEBHOOK_ENDPOINT,
+    WEBHOOK_AUTH_TOKEN,
+    WEBHOOK_QUEUE_LIMIT,
+    WEBHOOK_QUEUE_DIR,
+    WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY,
+    COMMENT_KEY,
+];
+
 // Webhook Environment Variables
 pub const ENV_WEBHOOK_ENABLE: &str = "RUSTFS_NOTIFY_WEBHOOK_ENABLE";
 pub const ENV_WEBHOOK_ENDPOINT: &str = "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT";
@@ -28,3 +42,13 @@ pub const ENV_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT";
 pub const ENV_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
 pub const ENV_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
 pub const ENV_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
+
+pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
+    ENV_WEBHOOK_ENABLE,
+    ENV_WEBHOOK_ENDPOINT,
+    ENV_WEBHOOK_AUTH_TOKEN,
+    ENV_WEBHOOK_QUEUE_LIMIT,
+    ENV_WEBHOOK_QUEUE_DIR,
+    ENV_WEBHOOK_CLIENT_CERT,
+    ENV_WEBHOOK_CLIENT_KEY,
+];

crates/config/src/observability/config.rs

@@ -12,279 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::observability::logger::LoggerConfig;
-use crate::observability::otel::OtelConfig;
-use crate::observability::sink::SinkConfig;
-use serde::{Deserialize, Serialize};
+// Observability Keys
 
-/// Observability configuration
-#[derive(Debug, Deserialize, Serialize, Clone)]
-pub struct ObservabilityConfig {
-    pub otel: OtelConfig,
-    pub sinks: Vec<SinkConfig>,
-    pub logger: Option<LoggerConfig>,
-}
+pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
+pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
+pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
+pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
+pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
+pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
+pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
+pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
+pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
+pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
+pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
+pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
 
-impl ObservabilityConfig {
-    pub fn new() -> Self {
-        Self {
-            otel: OtelConfig::new(),
-            sinks: vec![SinkConfig::new()],
-            logger: Some(LoggerConfig::new()),
-        }
-    }
-}
+pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
 
-impl Default for ObservabilityConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_observability_config_new() {
-        let config = ObservabilityConfig::new();
-
-        // Verify OTEL config is initialized
-        assert!(config.otel.use_stdout.is_some(), "OTEL use_stdout should be configured");
-        assert!(config.otel.sample_ratio.is_some(), "OTEL sample_ratio should be configured");
-        assert!(config.otel.meter_interval.is_some(), "OTEL meter_interval should be configured");
-        assert!(config.otel.service_name.is_some(), "OTEL service_name should be configured");
-        assert!(config.otel.service_version.is_some(), "OTEL service_version should be configured");
-        assert!(config.otel.environment.is_some(), "OTEL environment should be configured");
-        assert!(config.otel.logger_level.is_some(), "OTEL logger_level should be configured");
-
-        // Verify sinks are initialized
-        assert!(!config.sinks.is_empty(), "Sinks should not be empty");
-        assert_eq!(config.sinks.len(), 1, "Should have exactly one default sink");
-
-        // Verify logger is initialized
-        assert!(config.logger.is_some(), "Logger should be configured");
-    }
-
-    #[test]
-    fn test_observability_config_default() {
-        let config = ObservabilityConfig::default();
-        let new_config = ObservabilityConfig::new();
-
-        // Default should be equivalent to new()
-        assert_eq!(config.sinks.len(), new_config.sinks.len());
-        assert_eq!(config.logger.is_some(), new_config.logger.is_some());
-
-        // OTEL configs should be equivalent
-        assert_eq!(config.otel.use_stdout, new_config.otel.use_stdout);
-        assert_eq!(config.otel.sample_ratio, new_config.otel.sample_ratio);
-        assert_eq!(config.otel.meter_interval, new_config.otel.meter_interval);
-        assert_eq!(config.otel.service_name, new_config.otel.service_name);
-        assert_eq!(config.otel.service_version, new_config.otel.service_version);
-        assert_eq!(config.otel.environment, new_config.otel.environment);
-        assert_eq!(config.otel.logger_level, new_config.otel.logger_level);
-    }
-
-    #[test]
-    fn test_observability_config_otel_defaults() {
-        let config = ObservabilityConfig::new();
-
-        // Test OTEL default values
-        if let Some(_use_stdout) = config.otel.use_stdout {
-            // Test boolean values - any boolean value is valid
-        }
-
-        if let Some(sample_ratio) = config.otel.sample_ratio {
-            assert!((0.0..=1.0).contains(&sample_ratio), "Sample ratio should be between 0.0 and 1.0");
-        }
-
-        if let Some(meter_interval) = config.otel.meter_interval {
-            assert!(meter_interval > 0, "Meter interval should be positive");
-            assert!(meter_interval <= 3600, "Meter interval should be reasonable (≤ 1 hour)");
-        }
-
-        if let Some(service_name) = &config.otel.service_name {
-            assert!(!service_name.is_empty(), "Service name should not be empty");
-            assert!(!service_name.contains(' '), "Service name should not contain spaces");
-        }
-
-        if let Some(service_version) = &config.otel.service_version {
-            assert!(!service_version.is_empty(), "Service version should not be empty");
-        }
-
-        if let Some(environment) = &config.otel.environment {
-            assert!(!environment.is_empty(), "Environment should not be empty");
-            assert!(
-                ["development", "staging", "production", "test"].contains(&environment.as_str()),
-                "Environment should be a standard environment name"
-            );
-        }
-
-        if let Some(logger_level) = &config.otel.logger_level {
-            assert!(
-                ["trace", "debug", "info", "warn", "error"].contains(&logger_level.as_str()),
-                "Logger level should be a valid tracing level"
-            );
-        }
-    }
-
-    #[test]
-    fn test_observability_config_sinks() {
-        let config = ObservabilityConfig::new();
-
-        // Test default sink configuration
-        assert_eq!(config.sinks.len(), 1, "Should have exactly one default sink");
-
-        let _default_sink = &config.sinks[0];
-        // Test that the sink has valid configuration
-        // Note: We can't test specific values without knowing SinkConfig implementation
-        // but we can test that it's properly initialized
-
-        // Test that we can add more sinks
-        let mut config_mut = config.clone();
-        config_mut.sinks.push(SinkConfig::new());
-        assert_eq!(config_mut.sinks.len(), 2, "Should be able to add more sinks");
-    }
-
-    #[test]
-    fn test_observability_config_logger() {
-        let config = ObservabilityConfig::new();
-
-        // Test logger configuration
-        assert!(config.logger.is_some(), "Logger should be configured by default");
-
-        if let Some(_logger) = &config.logger {
-            // Test that logger has valid configuration
-            // Note: We can't test specific values without knowing LoggerConfig implementation
-            // but we can test that it's properly initialized
-        }
-
-        // Test that logger can be disabled
-        let mut config_mut = config.clone();
-        config_mut.logger = None;
-        assert!(config_mut.logger.is_none(), "Logger should be able to be disabled");
-    }
-
-    #[test]
-    fn test_observability_config_serialization() {
-        let config = ObservabilityConfig::new();
-
-        // Test serialization to JSON
-        let json_result = serde_json::to_string(&config);
-        assert!(json_result.is_ok(), "Config should be serializable to JSON");
-
-        let json_str = json_result.unwrap();
-        assert!(!json_str.is_empty(), "Serialized JSON should not be empty");
-        assert!(json_str.contains("otel"), "JSON should contain otel configuration");
-        assert!(json_str.contains("sinks"), "JSON should contain sinks configuration");
-        assert!(json_str.contains("logger"), "JSON should contain logger configuration");
-
-        // Test deserialization from JSON
-        let deserialized_result: Result<ObservabilityConfig, _> = serde_json::from_str(&json_str);
-        assert!(deserialized_result.is_ok(), "Config should be deserializable from JSON");
-
-        let deserialized_config = deserialized_result.unwrap();
-        assert_eq!(deserialized_config.sinks.len(), config.sinks.len());
-        assert_eq!(deserialized_config.logger.is_some(), config.logger.is_some());
-    }
-
-    #[test]
-    fn test_observability_config_debug_format() {
-        let config = ObservabilityConfig::new();
-
-        let debug_str = format!("{config:?}");
-        assert!(!debug_str.is_empty(), "Debug output should not be empty");
-        assert!(debug_str.contains("ObservabilityConfig"), "Debug output should contain struct name");
-        assert!(debug_str.contains("otel"), "Debug output should contain otel field");
-        assert!(debug_str.contains("sinks"), "Debug output should contain sinks field");
-        assert!(debug_str.contains("logger"), "Debug output should contain logger field");
-    }
-
-    #[test]
-    fn test_observability_config_clone() {
-        let config = ObservabilityConfig::new();
-        let cloned_config = config.clone();
-
-        // Test that clone creates an independent copy
-        assert_eq!(cloned_config.sinks.len(), config.sinks.len());
-        assert_eq!(cloned_config.logger.is_some(), config.logger.is_some());
-        assert_eq!(cloned_config.otel.endpoint, config.otel.endpoint);
-        assert_eq!(cloned_config.otel.use_stdout, config.otel.use_stdout);
-        assert_eq!(cloned_config.otel.sample_ratio, config.otel.sample_ratio);
-        assert_eq!(cloned_config.otel.meter_interval, config.otel.meter_interval);
-        assert_eq!(cloned_config.otel.service_name, config.otel.service_name);
-        assert_eq!(cloned_config.otel.service_version, config.otel.service_version);
-        assert_eq!(cloned_config.otel.environment, config.otel.environment);
-        assert_eq!(cloned_config.otel.logger_level, config.otel.logger_level);
-    }
-
-    #[test]
-    fn test_observability_config_modification() {
-        let mut config = ObservabilityConfig::new();
-
-        // Test modifying OTEL endpoint
-        let original_endpoint = config.otel.endpoint.clone();
-        config.otel.endpoint = "http://localhost:4317".to_string();
-        assert_ne!(config.otel.endpoint, original_endpoint);
-        assert_eq!(config.otel.endpoint, "http://localhost:4317");
-
-        // Test modifying sinks
-        let original_sinks_len = config.sinks.len();
-        config.sinks.push(SinkConfig::new());
-        assert_eq!(config.sinks.len(), original_sinks_len + 1);
-
-        // Test disabling logger
-        config.logger = None;
-        assert!(config.logger.is_none());
-    }
-
-    #[test]
-    fn test_observability_config_edge_cases() {
-        // Test with empty sinks
-        let mut config = ObservabilityConfig::new();
-        config.sinks.clear();
-        assert!(config.sinks.is_empty(), "Sinks should be empty after clearing");
-
-        // Test serialization with empty sinks
-        let json_result = serde_json::to_string(&config);
-        assert!(json_result.is_ok(), "Config with empty sinks should be serializable");
-
-        // Test with no logger
-        config.logger = None;
-        let json_result = serde_json::to_string(&config);
-        assert!(json_result.is_ok(), "Config with no logger should be serializable");
-    }
-
-    #[test]
-    fn test_observability_config_memory_efficiency() {
-        let config = ObservabilityConfig::new();
-
-        // Test that config doesn't use excessive memory
-        let config_size = std::mem::size_of_val(&config);
-        assert!(config_size < 5000, "Config should not use excessive memory");
-
-        // Test that endpoint string is not excessively long
-        assert!(config.otel.endpoint.len() < 1000, "Endpoint should not be excessively long");
-
-        // Test that collections are reasonably sized
-        assert!(config.sinks.len() < 100, "Sinks collection should be reasonably sized");
-    }
-
-    #[test]
-    fn test_observability_config_consistency() {
-        // Create multiple configs and ensure they're consistent
-        let config1 = ObservabilityConfig::new();
-        let config2 = ObservabilityConfig::new();
-
-        // Both configs should have the same default structure
-        assert_eq!(config1.sinks.len(), config2.sinks.len());
-        assert_eq!(config1.logger.is_some(), config2.logger.is_some());
-        assert_eq!(config1.otel.use_stdout, config2.otel.use_stdout);
-        assert_eq!(config1.otel.sample_ratio, config2.otel.sample_ratio);
-        assert_eq!(config1.otel.meter_interval, config2.otel.meter_interval);
-        assert_eq!(config1.otel.service_name, config2.otel.service_name);
-        assert_eq!(config1.otel.service_version, config2.otel.service_version);
-        assert_eq!(config1.otel.environment, config2.otel.environment);
-        assert_eq!(config1.otel.logger_level, config2.otel.logger_level);
-    }
-}
+// Default values for observability configuration
+pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;

crates/config/src/observability/file.rs

@@ -12,62 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Serialize};
-use std::env;
+// RUSTFS_SINKS_FILE_PATH
+pub const ENV_SINKS_FILE_PATH: &str = "RUSTFS_SINKS_FILE_PATH";
+// RUSTFS_SINKS_FILE_BUFFER_SIZE
+pub const ENV_SINKS_FILE_BUFFER_SIZE: &str = "RUSTFS_SINKS_FILE_BUFFER_SIZE";
+// RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS
+pub const ENV_SINKS_FILE_FLUSH_INTERVAL_MS: &str = "RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS";
+// RUSTFS_SINKS_FILE_FLUSH_THRESHOLD
+pub const ENV_SINKS_FILE_FLUSH_THRESHOLD: &str = "RUSTFS_SINKS_FILE_FLUSH_THRESHOLD";
 
-/// File sink configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct FileSink {
-    pub path: String,
-    #[serde(default = "default_buffer_size")]
-    pub buffer_size: Option<usize>,
-    #[serde(default = "default_flush_interval_ms")]
-    pub flush_interval_ms: Option<u64>,
-    #[serde(default = "default_flush_threshold")]
-    pub flush_threshold: Option<usize>,
-}
+pub const DEFAULT_SINKS_FILE_BUFFER_SIZE: usize = 8192;
 
-impl FileSink {
-    pub fn new() -> Self {
-        Self {
-            path: env::var("RUSTFS_SINKS_FILE_PATH")
-                .ok()
-                .filter(|s| !s.trim().is_empty())
-                .unwrap_or_else(default_path),
-            buffer_size: default_buffer_size(),
-            flush_interval_ms: default_flush_interval_ms(),
-            flush_threshold: default_flush_threshold(),
-        }
-    }
-}
+pub const DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS: u64 = 1000;
 
-impl Default for FileSink {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-fn default_buffer_size() -> Option<usize> {
-    Some(8192)
-}
-fn default_flush_interval_ms() -> Option<u64> {
-    Some(1000)
-}
-fn default_flush_threshold() -> Option<usize> {
-    Some(100)
-}
-
-fn default_path() -> String {
-    let temp_dir = env::temp_dir().join("rustfs");
-
-    if let Err(e) = std::fs::create_dir_all(&temp_dir) {
-        eprintln!("Failed to create log directory: {e}");
-        return "rustfs/rustfs.log".to_string();
-    }
-
-    temp_dir
-        .join("rustfs.log")
-        .to_str()
-        .unwrap_or("rustfs/rustfs.log")
-        .to_string()
-}
+pub const DEFAULT_SINKS_FILE_FLUSH_THRESHOLD: usize = 100;

crates/config/src/observability/kafka.rs

@@ -12,39 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Serialize};
+// RUSTFS_SINKS_KAFKA_BROKERS
+pub const ENV_SINKS_KAFKA_BROKERS: &str = "RUSTFS_SINKS_KAFKA_BROKERS";
+pub const ENV_SINKS_KAFKA_TOPIC: &str = "RUSTFS_SINKS_KAFKA_TOPIC";
+// batch_size
+pub const ENV_SINKS_KAFKA_BATCH_SIZE: &str = "RUSTFS_SINKS_KAFKA_BATCH_SIZE";
+// batch_timeout_ms
+pub const ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS: &str = "RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS";
 
-/// Kafka sink configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct KafkaSink {
-    pub brokers: String,
-    pub topic: String,
-    #[serde(default = "default_batch_size")]
-    pub batch_size: Option<usize>,
-    #[serde(default = "default_batch_timeout_ms")]
-    pub batch_timeout_ms: Option<u64>,
-}
-
-impl KafkaSink {
-    pub fn new() -> Self {
-        Self {
-            brokers: "localhost:9092".to_string(),
-            topic: "rustfs".to_string(),
-            batch_size: default_batch_size(),
-            batch_timeout_ms: default_batch_timeout_ms(),
-        }
-    }
-}
-
-impl Default for KafkaSink {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-fn default_batch_size() -> Option<usize> {
-    Some(100)
-}
-fn default_batch_timeout_ms() -> Option<u64> {
-    Some(1000)
-}
+// brokers
+pub const DEFAULT_SINKS_KAFKA_BROKERS: &str = "localhost:9092";
+pub const DEFAULT_SINKS_KAFKA_TOPIC: &str = "rustfs-sinks";
+pub const DEFAULT_SINKS_KAFKA_BATCH_SIZE: usize = 100;
+pub const DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS: u64 = 1000;

crates/config/src/observability/mod.rs

@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub(crate) mod config;
-pub(crate) mod file;
-pub(crate) mod kafka;
-pub(crate) mod logger;
-pub(crate) mod otel;
-pub(crate) mod sink;
-pub(crate) mod webhook;
+mod config;
+mod file;
+mod kafka;
+mod webhook;
+
+pub use config::*;
+pub use file::*;
+pub use kafka::*;
+pub use webhook::*;

crates/config/src/observability/otel.rs

@@ -1,83 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
-use crate::{APP_NAME, DEFAULT_LOG_LEVEL};
-use serde::{Deserialize, Serialize};
-use std::env;
-
-/// OpenTelemetry configuration
-#[derive(Debug, Deserialize, Serialize, Clone)]
-pub struct OtelConfig {
-    pub endpoint: String,                    // Endpoint for metric collection
-    pub use_stdout: Option<bool>,            // Output to stdout
-    pub sample_ratio: Option<f64>,           // Trace sampling ratio
-    pub meter_interval: Option<u64>,         // Metric collection interval
-    pub service_name: Option<String>,        // Service name
-    pub service_version: Option<String>,     // Service version
-    pub environment: Option<String>,         // Environment
-    pub logger_level: Option<String>,        // Logger level
-    pub local_logging_enabled: Option<bool>, // Local logging enabled
-}
-
-impl OtelConfig {
-    pub fn new() -> Self {
-        extract_otel_config_from_env()
-    }
-}
-
-impl Default for OtelConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-// Helper function: Extract observable configuration from environment variables
-fn extract_otel_config_from_env() -> OtelConfig {
-    OtelConfig {
-        endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
-        use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(USE_STDOUT)),
-        sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(SAMPLE_RATIO)),
-        meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(METER_INTERVAL)),
-        service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(APP_NAME.to_string())),
-        service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(SERVICE_VERSION.to_string())),
-        environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(ENVIRONMENT.to_string())),
-        logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(DEFAULT_LOG_LEVEL.to_string())),
-        local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .or(Some(false)),
-    }
-}

crates/config/src/observability/sink.rs

@@ -1,39 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::observability::file::FileSink;
-use crate::observability::kafka::KafkaSink;
-use crate::observability::webhook::WebhookSink;
-use serde::{Deserialize, Serialize};
-
-/// Sink configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "type")]
-pub enum SinkConfig {
-    Kafka(KafkaSink),
-    Webhook(WebhookSink),
-    File(FileSink),
-}
-
-impl SinkConfig {
-    pub fn new() -> Self {
-        Self::File(FileSink::new())
-    }
-}
-
-impl Default for SinkConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}

crates/config/src/observability/webhook.rs

@@ -12,42 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+// RUSTFS_SINKS_WEBHOOK_ENDPOINT
+pub const ENV_SINKS_WEBHOOK_ENDPOINT: &str = "RUSTFS_SINKS_WEBHOOK_ENDPOINT";
+// RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN
+pub const ENV_SINKS_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN";
+// max_retries
+pub const ENV_SINKS_WEBHOOK_MAX_RETRIES: &str = "RUSTFS_SINKS_WEBHOOK_MAX_RETRIES";
+// retry_delay_ms
+pub const ENV_SINKS_WEBHOOK_RETRY_DELAY_MS: &str = "RUSTFS_SINKS_WEBHOOK_RETRY_DELAY_MS";
 
-/// Webhook sink configuration
-#[derive(Debug, Deserialize, Serialize, Clone)]
-pub struct WebhookSink {
-    pub endpoint: String,
-    pub auth_token: String,
-    pub headers: Option<HashMap<String, String>>,
-    #[serde(default = "default_max_retries")]
-    pub max_retries: Option<usize>,
-    #[serde(default = "default_retry_delay_ms")]
-    pub retry_delay_ms: Option<u64>,
-}
-
-impl WebhookSink {
-    pub fn new() -> Self {
-        Self {
-            endpoint: "".to_string(),
-            auth_token: "".to_string(),
-            headers: Some(HashMap::new()),
-            max_retries: default_max_retries(),
-            retry_delay_ms: default_retry_delay_ms(),
-        }
-    }
-}
-
-impl Default for WebhookSink {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-fn default_max_retries() -> Option<usize> {
-    Some(3)
-}
-fn default_retry_delay_ms() -> Option<u64> {
-    Some(100)
-}
+// Default values for webhook sink configuration
+pub const DEFAULT_SINKS_WEBHOOK_ENDPOINT: &str = "http://localhost:8080";
+pub const DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN: &str = "";
+pub const DEFAULT_SINKS_WEBHOOK_MAX_RETRIES: usize = 3;
+pub const DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS: u64 = 100;

crates/crypto/src/encdec/decrypt.rs

@@ -33,11 +33,11 @@ pub fn decrypt_data(password: &[u8], data: &[u8]) -> Result<Vec<u8>, crate::Erro
     match id {
         ID::Argon2idChaCHa20Poly1305 => {
             let key = id.get_key(password, salt)?;
-            decryp(ChaCha20Poly1305::new_from_slice(&key)?, nonce, data)
+            decrypt(ChaCha20Poly1305::new_from_slice(&key)?, nonce, data)
         }
         _ => {
             let key = id.get_key(password, salt)?;
-            decryp(Aes256Gcm::new_from_slice(&key)?, nonce, data)
+            decrypt(Aes256Gcm::new_from_slice(&key)?, nonce, data)
         }
     }
 }
@@ -135,7 +135,7 @@ pub fn decrypt_data(password: &[u8], data: &[u8]) -> Result<Vec<u8>, crate::Erro
 
 #[cfg(any(test, feature = "crypto"))]
 #[inline]
-fn decryp<T: aes_gcm::aead::Aead>(stream: T, nonce: &[u8], data: &[u8]) -> Result<Vec<u8>, crate::Error> {
+fn decrypt<T: aes_gcm::aead::Aead>(stream: T, nonce: &[u8], data: &[u8]) -> Result<Vec<u8>, crate::Error> {
     use crate::error::Error;
     stream
         .decrypt(aes_gcm::Nonce::from_slice(nonce), data)

crates/e2e_test/Cargo.toml

@@ -38,3 +38,7 @@ url.workspace = true
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
+serial_test = { workspace = true }
+aws-sdk-s3.workspace = true
+aws-config = { workspace = true }
+async-trait = { workspace = true }

crates/e2e_test/src/reliant/lock.rs

@@ -13,28 +13,45 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_lock::{
-    drwmutex::Options,
-    lock_args::LockArgs,
-    namespace_lock::{NsLockMap, new_nslock},
-    new_lock_api,
-};
+use async_trait::async_trait;
+use rustfs_ecstore::{disk::endpoint::Endpoint, lock_utils::create_unique_clients};
+use rustfs_lock::client::{LockClient, local::LocalClient};
+use rustfs_lock::types::{LockInfo, LockResponse, LockStats};
+use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType};
+use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager};
 use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
+use serial_test::serial;
 use std::{error::Error, sync::Arc, time::Duration};
-use tokio::sync::RwLock;
+use tokio::time::sleep;
 use tonic::Request;
+use url::Url;
 
 const CLUSTER_ADDR: &str = "http://localhost:9000";
 
+fn get_cluster_endpoints() -> Vec<Endpoint> {
+    vec![Endpoint {
+        url: Url::parse(CLUSTER_ADDR).unwrap(),
+        is_local: false,
+        pool_idx: 0,
+        set_idx: 0,
+        disk_idx: 0,
+    }]
+}
+
 #[tokio::test]
+#[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
 async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
-    let args = LockArgs {
-        uid: "1111".to_string(),
-        resources: vec!["dandan".to_string()],
+    let args = LockRequest {
+        lock_id: LockId::new_deterministic("dandan"),
+        resource: "dandan".to_string(),
+        lock_type: LockType::Exclusive,
         owner: "dd".to_string(),
-        source: "".to_string(),
-        quorum: 3,
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
     };
     let args = serde_json::to_string(&args)?;
 
@@ -58,31 +75,616 @@ async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
-    let url = url::Url::parse("http://127.0.0.1:9000/data")?;
-    let locker = new_lock_api(false, Some(url));
-    let ns_mutex = Arc::new(RwLock::new(NsLockMap::new(true)));
-    let ns = new_nslock(
-        Arc::clone(&ns_mutex),
-        "local".to_string(),
-        "dandan".to_string(),
-        vec!["foo".to_string()],
-        vec![locker],
-    )
-    .await;
-    assert!(
-        ns.0.write()
-            .await
-            .get_lock(&Options {
-                timeout: Duration::from_secs(5),
-                retry_interval: Duration::from_secs(1),
-            })
-            .await
-            .unwrap()
-    );
+/// Mock client that simulates remote node failures
+#[derive(Debug)]
+struct FailingMockClient {
+    local_client: Arc<dyn LockClient>,
+    should_fail_acquire: bool,
+    should_fail_release: bool,
+}
+
+impl FailingMockClient {
+    fn new(should_fail_acquire: bool, should_fail_release: bool) -> Self {
+        Self {
+            local_client: Arc::new(LocalClient::new()),
+            should_fail_acquire,
+            should_fail_release,
+        }
+    }
+}
+
+#[async_trait]
+impl LockClient for FailingMockClient {
+    async fn acquire_exclusive(&self, request: &LockRequest) -> rustfs_lock::error::Result<LockResponse> {
+        if self.should_fail_acquire {
+            // Simulate network timeout or remote node failure
+            return Ok(LockResponse::failure("Simulated remote node failure", Duration::from_millis(100)));
+        }
+        self.local_client.acquire_exclusive(request).await
+    }
+
+    async fn acquire_shared(&self, request: &LockRequest) -> rustfs_lock::error::Result<LockResponse> {
+        if self.should_fail_acquire {
+            return Ok(LockResponse::failure("Simulated remote node failure", Duration::from_millis(100)));
+        }
+        self.local_client.acquire_shared(request).await
+    }
+
+    async fn release(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
+        if self.should_fail_release {
+            return Err(rustfs_lock::error::LockError::internal("Simulated release failure"));
+        }
+        self.local_client.release(lock_id).await
+    }
+
+    async fn refresh(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
+        self.local_client.refresh(lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
+        self.local_client.force_release(lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &LockId) -> rustfs_lock::error::Result<Option<LockInfo>> {
+        self.local_client.check_status(lock_id).await
+    }
+
+    async fn get_stats(&self) -> rustfs_lock::error::Result<LockStats> {
+        self.local_client.get_stats().await
+    }
+
+    async fn close(&self) -> rustfs_lock::error::Result<()> {
+        self.local_client.close().await
+    }
+
+    async fn is_online(&self) -> bool {
+        if self.should_fail_acquire {
+            return false; // Simulate offline node
+        }
+        true // Simulate online node
+    }
+
+    async fn is_local(&self) -> bool {
+        false // Simulate remote client
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn test_transactional_lock_with_remote_failure() -> Result<(), Box<dyn Error>> {
+    println!("🧪 Testing transactional lock with simulated remote node failure");
+
+    // Create a two-node cluster: one local (success) + one remote (failure)
+    let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
+    let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
+
+    let clients = vec![local_client, failing_remote_client];
+    let ns_lock = NamespaceLock::with_clients("test_transactional".to_string(), clients);
+
+    let resource = "critical_resource".to_string();
+
+    // Test single lock operation with 2PC
+    println!("📝 Testing single lock with remote failure...");
+    let request = LockRequest::new(&resource, LockType::Exclusive, "test_owner").with_ttl(Duration::from_secs(30));
+
+    let response = ns_lock.acquire_lock(&request).await?;
+
+    // Should fail because quorum (2/2) is not met due to remote failure
+    assert!(!response.success, "Lock should fail due to remote node failure");
+    println!("✅ Single lock correctly failed due to remote node failure");
+
+    // Verify no locks are left behind on the local node
+    let local_client_direct = LocalClient::new();
+    let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(&resource));
+    let lock_status = local_client_direct.check_status(&lock_id).await?;
+    assert!(lock_status.is_none(), "No lock should remain on local node after rollback");
+    println!("✅ Verified rollback: no locks left on local node");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_transactional_batch_lock_with_mixed_failures() -> Result<(), Box<dyn Error>> {
+    println!("🧪 Testing transactional batch lock with mixed node failures");
+
+    // Create a cluster with different failure patterns
+    let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
+    let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
+
+    let clients = vec![local_client, failing_remote_client];
+    let ns_lock = NamespaceLock::with_clients("test_batch_transactional".to_string(), clients);
+
+    let resources = vec!["resource_1".to_string(), "resource_2".to_string(), "resource_3".to_string()];
+
+    println!("📝 Testing batch lock with remote failure...");
+    let result = ns_lock
+        .lock_batch(&resources, "batch_owner", Duration::from_millis(100), Duration::from_secs(30))
+        .await?;
+
+    // Should fail because remote node cannot acquire locks
+    assert!(!result, "Batch lock should fail due to remote node failure");
+    println!("✅ Batch lock correctly failed due to remote node failure");
+
+    // Verify no locks are left behind on any resource
+    let local_client_direct = LocalClient::new();
+    for resource in &resources {
+        let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(resource));
+        let lock_status = local_client_direct.check_status(&lock_id).await?;
+        assert!(lock_status.is_none(), "No lock should remain for resource: {resource}");
+    }
+    println!("✅ Verified rollback: no locks left on any resource");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_transactional_lock_with_quorum_success() -> Result<(), Box<dyn Error>> {
+    println!("🧪 Testing transactional lock with quorum success");
+
+    // Create a three-node cluster where 2 succeed and 1 fails (quorum = 2 automatically)
+    let local_client1: Arc<dyn LockClient> = Arc::new(LocalClient::new());
+    let local_client2: Arc<dyn LockClient> = Arc::new(LocalClient::new());
+    let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
+
+    let clients = vec![local_client1, local_client2, failing_remote_client];
+    let ns_lock = NamespaceLock::with_clients("test_quorum".to_string(), clients);
+
+    let resource = "quorum_resource".to_string();
+
+    println!("📝 Testing lock with automatic quorum=2, 2 success + 1 failure...");
+    let request = LockRequest::new(&resource, LockType::Exclusive, "quorum_owner").with_ttl(Duration::from_secs(30));
+
+    let response = ns_lock.acquire_lock(&request).await?;
+
+    // Should fail because we require all nodes to succeed for consistency
+    // (even though quorum is met, the implementation requires all nodes for consistency)
+    assert!(!response.success, "Lock should fail due to consistency requirement");
+    println!("✅ Lock correctly failed due to consistency requirement (partial success rolled back)");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_transactional_lock_rollback_on_release_failure() -> Result<(), Box<dyn Error>> {
+    println!("🧪 Testing rollback behavior when release fails");
+
+    // Create clients where acquire succeeds but release fails
+    let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
+    let failing_release_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(false, true));
+
+    let clients = vec![local_client, failing_release_client];
+    let ns_lock = NamespaceLock::with_clients("test_release_failure".to_string(), clients);
+
+    let resource = "release_test_resource".to_string();
+
+    println!("📝 Testing lock acquisition with release failure handling...");
+    let request = LockRequest::new(&resource, LockType::Exclusive, "test_owner").with_ttl(Duration::from_secs(30));
+
+    // This should fail because both LocalClient instances share the same global lock map
+    // The first client (LocalClient) will acquire the lock, but the second client
+    // (FailingMockClient's internal LocalClient) will fail to acquire the same resource
+    let response = ns_lock.acquire_lock(&request).await?;
+
+    // The operation should fail due to lock contention between the two LocalClient instances
+    assert!(
+        !response.success,
+        "Lock should fail due to lock contention between LocalClient instances sharing global lock map"
+    );
+    println!("✅ Lock correctly failed due to lock contention (both clients use same global lock map)");
+
+    // Verify no locks are left behind after rollback
+    let local_client_direct = LocalClient::new();
+    let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(&resource));
+    let lock_status = local_client_direct.check_status(&lock_id).await?;
+    assert!(lock_status.is_none(), "No lock should remain after rollback");
+    println!("✅ Verified rollback: no locks left after failed acquisition");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
+
+    let resources = vec!["foo".to_string()];
+    let result = ns_lock
+        .lock_batch(&resources, "dandan", Duration::from_secs(5), Duration::from_secs(10))
+        .await;
+    match &result {
+        Ok(success) => println!("Lock result: {success}"),
+        Err(e) => println!("Lock error: {e}"),
+    }
+    let result = result?;
+    assert!(result, "Lock should succeed, but got: {result}");
+
+    ns_lock.unlock_batch(&resources, "dandan").await?;
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_concurrent_lock_attempts() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
+    let resource = vec!["concurrent_resource".to_string()];
+
+    // First lock should succeed
+    println!("Attempting first lock...");
+    let result1 = ns_lock
+        .lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    println!("First lock result: {result1}");
+    assert!(result1, "First lock should succeed");
+
+    // Second lock should fail (resource already locked)
+    println!("Attempting second lock...");
+    let result2 = ns_lock
+        .lock_batch(&resource, "owner2", Duration::from_secs(1), Duration::from_secs(10))
+        .await?;
+    println!("Second lock result: {result2}");
+    assert!(!result2, "Second lock should fail");
+
+    // Unlock by first owner
+    println!("Unlocking first lock...");
+    ns_lock.unlock_batch(&resource, "owner1").await?;
+    println!("First lock unlocked");
+
+    // Now second owner should be able to lock
+    println!("Attempting third lock...");
+    let result3 = ns_lock
+        .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    println!("Third lock result: {result3}");
+    assert!(result3, "Lock should succeed after unlock");
+
+    // Clean up
+    println!("Cleaning up...");
+    ns_lock.unlock_batch(&resource, "owner2").await?;
+    println!("Test completed");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_read_write_lock_compatibility() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock = NamespaceLock::with_clients("test_rw".to_string(), clients);
+    let resource = vec!["rw_resource".to_string()];
+
+    // First read lock should succeed
+    let result1 = ns_lock
+        .rlock_batch(&resource, "reader1", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result1, "First read lock should succeed");
+
+    // Second read lock should also succeed (read locks are compatible)
+    let result2 = ns_lock
+        .rlock_batch(&resource, "reader2", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result2, "Second read lock should succeed");
+
+    // Write lock should fail (read locks are held)
+    let result3 = ns_lock
+        .lock_batch(&resource, "writer1", Duration::from_secs(1), Duration::from_secs(10))
+        .await?;
+    assert!(!result3, "Write lock should fail when read locks are held");
+
+    // Release read locks
+    ns_lock.runlock_batch(&resource, "reader1").await?;
+    ns_lock.runlock_batch(&resource, "reader2").await?;
+
+    // Now write lock should succeed
+    let result4 = ns_lock
+        .lock_batch(&resource, "writer1", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result4, "Write lock should succeed after read locks released");
+
+    // Clean up
+    ns_lock.unlock_batch(&resource, "writer1").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_lock_timeout() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock = NamespaceLock::with_clients("test_timeout".to_string(), clients);
+    let resource = vec!["timeout_resource".to_string()];
+
+    // First lock with short timeout
+    let result1 = ns_lock
+        .lock_batch(&resource, "owner1", Duration::from_secs(2), Duration::from_secs(1))
+        .await?;
+    assert!(result1, "First lock should succeed");
+
+    // Wait for lock to expire
+    sleep(Duration::from_secs(5)).await;
+
+    // Second lock should succeed after timeout
+    let result2 = ns_lock
+        .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(1))
+        .await?;
+    assert!(result2, "Lock should succeed after timeout");
+
+    // Clean up
+    ns_lock.unlock_batch(&resource, "owner2").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_batch_lock_operations() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock = NamespaceLock::with_clients("test_batch".to_string(), clients);
+    let resources = vec![
+        "batch_resource1".to_string(),
+        "batch_resource2".to_string(),
+        "batch_resource3".to_string(),
+    ];
+
+    // Lock all resources
+    let result = ns_lock
+        .lock_batch(&resources, "batch_owner", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result, "Batch lock should succeed");
+
+    // Try to lock one of the resources with different owner - should fail
+    let single_resource = vec!["batch_resource2".to_string()];
+    let result2 = ns_lock
+        .lock_batch(&single_resource, "other_owner", Duration::from_secs(1), Duration::from_secs(10))
+        .await?;
+    assert!(!result2, "Lock should fail for already locked resource");
+
+    // Unlock all resources
+    ns_lock.unlock_batch(&resources, "batch_owner").await?;
+
+    // Now should be able to lock single resource
+    let result3 = ns_lock
+        .lock_batch(&single_resource, "other_owner", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result3, "Lock should succeed after batch unlock");
+
+    // Clean up
+    ns_lock.unlock_batch(&single_resource, "other_owner").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_multiple_namespaces() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock1 = NamespaceLock::with_clients("namespace1".to_string(), clients.clone());
+    let ns_lock2 = NamespaceLock::with_clients("namespace2".to_string(), clients);
+    let resource = vec!["shared_resource".to_string()];
+
+    // Lock same resource in different namespaces - both should succeed
+    let result1 = ns_lock1
+        .lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result1, "Lock in namespace1 should succeed");
+
+    let result2 = ns_lock2
+        .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
+        .await?;
+    assert!(result2, "Lock in namespace2 should succeed");
+
+    // Clean up
+    ns_lock1.unlock_batch(&resource, "owner1").await?;
+    ns_lock2.unlock_batch(&resource, "owner2").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_rpc_read_lock() -> Result<(), Box<dyn Error>> {
+    let args = LockRequest {
+        lock_id: LockId::new_deterministic("read_resource"),
+        resource: "read_resource".to_string(),
+        lock_type: LockType::Shared,
+        owner: "reader1".to_string(),
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
+    };
+    let args_str = serde_json::to_string(&args)?;
+
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+
+    // First read lock
+    let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
+    let response = client.r_lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not get read lock: {error_info}");
+    }
+
+    // Second read lock with different owner should also succeed
+    let args2 = LockRequest {
+        lock_id: LockId::new_deterministic("read_resource"),
+        resource: "read_resource".to_string(),
+        lock_type: LockType::Shared,
+        owner: "reader2".to_string(),
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
+    };
+    let args2_str = serde_json::to_string(&args2)?;
+    let request2 = Request::new(GenerallyLockRequest { args: args2_str });
+    let response2 = client.r_lock(request2).await?.into_inner();
+    if let Some(error_info) = response2.error_info {
+        panic!("can not get second read lock: {error_info}");
+    }
+
+    // Unlock both
+    let request = Request::new(GenerallyLockRequest { args: args_str });
+    let response = client.r_un_lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not unlock read lock: {error_info}");
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_lock_refresh() -> Result<(), Box<dyn Error>> {
+    let args = LockRequest {
+        lock_id: LockId::new_deterministic("refresh_resource"),
+        resource: "refresh_resource".to_string(),
+        lock_type: LockType::Exclusive,
+        owner: "refresh_owner".to_string(),
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
+    };
+    let args_str = serde_json::to_string(&args)?;
+
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+
+    // Acquire lock
+    let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
+    let response = client.lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not get lock: {error_info}");
+    }
+
+    // Refresh lock
+    let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
+    let response = client.refresh(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not refresh lock: {error_info}");
+    }
+    assert!(response.success, "Lock refresh should succeed");
+
+    // Unlock
+    let request = Request::new(GenerallyLockRequest { args: args_str });
+    let response = client.un_lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not unlock: {error_info}");
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_force_unlock() -> Result<(), Box<dyn Error>> {
+    let args = LockRequest {
+        lock_id: LockId::new_deterministic("force_resource"),
+        resource: "force_resource".to_string(),
+        lock_type: LockType::Exclusive,
+        owner: "force_owner".to_string(),
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
+    };
+    let args_str = serde_json::to_string(&args)?;
+
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+
+    // Acquire lock
+    let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
+    let response = client.lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not get lock: {error_info}");
+    }
+
+    // Force unlock (even by different owner)
+    let force_args = LockRequest {
+        lock_id: LockId::new_deterministic("force_resource"),
+        resource: "force_resource".to_string(),
+        lock_type: LockType::Exclusive,
+        owner: "admin".to_string(),
+        acquire_timeout: Duration::from_secs(30),
+        ttl: Duration::from_secs(30),
+        metadata: LockMetadata::default(),
+        priority: LockPriority::Normal,
+        deadlock_detection: false,
+    };
+    let force_args_str = serde_json::to_string(&force_args)?;
+    let request = Request::new(GenerallyLockRequest { args: force_args_str });
+    let response = client.force_un_lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not force unlock: {error_info}");
+    }
+    assert!(response.success, "Force unlock should succeed");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_global_lock_map_sharing() -> Result<(), Box<dyn Error>> {
+    let endpoints = get_cluster_endpoints();
+    let clients = create_unique_clients(&endpoints).await?;
+    let ns_lock1 = NamespaceLock::with_clients("global_test".to_string(), clients.clone());
+    let ns_lock2 = NamespaceLock::with_clients("global_test".to_string(), clients);
+
+    let resource = vec!["global_test_resource".to_string()];
+
+    // First instance acquires lock
+    println!("First lock map attempting to acquire lock...");
+    let result1 = ns_lock1
+        .lock_batch(&resource, "owner1", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
+        .await?;
+    println!("First lock result: {result1}");
+    assert!(result1, "First lock should succeed");
+
+    // Second instance should fail to acquire the same lock
+    println!("Second lock map attempting to acquire lock...");
+    let result2 = ns_lock2
+        .lock_batch(&resource, "owner2", std::time::Duration::from_secs(1), std::time::Duration::from_secs(10))
+        .await?;
+    println!("Second lock result: {result2}");
+    assert!(!result2, "Second lock should fail because resource is already locked");
+
+    // Release lock from first instance
+    println!("First lock map releasing lock...");
+    ns_lock1.unlock_batch(&resource, "owner1").await?;
+
+    // Now second instance should be able to acquire lock
+    println!("Second lock map attempting to acquire lock again...");
+    let result3 = ns_lock2
+        .lock_batch(&resource, "owner2", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
+        .await?;
+    println!("Third lock result: {result3}");
+    assert!(result3, "Lock should succeed after first lock is released");
+
+    // Clean up
+    ns_lock2.unlock_batch(&resource, "owner2").await?;
 
-    ns.0.write().await.un_lock().await.unwrap();
     Ok(())
 }

crates/e2e_test/src/reliant/mod.rs

@@ -14,3 +14,4 @@
 
 mod lock;
 mod node_interact_test;
+mod sql;

crates/e2e_test/src/reliant/sql.rs

@@ -0,0 +1,402 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use aws_config::meta::region::RegionProviderChain;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::types::{
+    CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
+};
+use bytes::Bytes;
+use serial_test::serial;
+use std::error::Error;
+
+const ENDPOINT: &str = "http://localhost:9000";
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-sql-bucket";
+const CSV_OBJECT: &str = "test-data.csv";
+const JSON_OBJECT: &str = "test-data.json";
+
+async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
+    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
+    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
+        .region(region_provider)
+        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
+        .endpoint_url(ENDPOINT)
+        .load()
+        .await;
+
+    let client = Client::from_conf(
+        aws_sdk_s3::Config::from(&shared_config)
+            .to_builder()
+            .force_path_style(true) // Important for S3-compatible services
+            .build(),
+    );
+
+    Ok(client)
+}
+
+async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
+    match client.create_bucket().bucket(BUCKET).send().await {
+        Ok(_) => {}
+        Err(e) => {
+            let error_str = e.to_string();
+            if !error_str.contains("BucketAlreadyOwnedByYou") && !error_str.contains("BucketAlreadyExists") {
+                return Err(e.into());
+            }
+        }
+    }
+    Ok(())
+}
+
+async fn upload_test_csv(client: &Client) -> Result<(), Box<dyn Error>> {
+    let csv_data = "name,age,city\nAlice,30,New York\nBob,25,Los Angeles\nCharlie,35,Chicago\nDiana,28,Boston";
+
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .body(Bytes::from(csv_data.as_bytes()).into())
+        .send()
+        .await?;
+
+    Ok(())
+}
+
+async fn upload_test_json(client: &Client) -> Result<(), Box<dyn Error>> {
+    let json_data = r#"{"name":"Alice","age":30,"city":"New York"}
+{"name":"Bob","age":25,"city":"Los Angeles"}
+{"name":"Charlie","age":35,"city":"Chicago"}
+{"name":"Diana","age":28,"city":"Boston"}"#;
+
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(JSON_OBJECT)
+        .body(Bytes::from(json_data.as_bytes()).into())
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn process_select_response(
+    mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
+) -> Result<String, Box<dyn Error>> {
+    let mut total_data = Vec::new();
+
+    while let Ok(Some(event)) = event_stream.payload.recv().await {
+        match event {
+            aws_sdk_s3::types::SelectObjectContentEventStream::Records(records_event) => {
+                if let Some(payload) = records_event.payload {
+                    let data = payload.into_inner();
+                    total_data.extend_from_slice(&data);
+                }
+            }
+            aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
+                break;
+            }
+            _ => {
+                // Handle other event types (Stats, Progress, Cont, etc.)
+            }
+        }
+    }
+
+    Ok(String::from_utf8(total_data)?)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_csv_basic() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_csv(&client).await?;
+
+    // Construct SelectObjectContent request - basic query
+    let sql = "SELECT * FROM S3Object WHERE age > 28";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await?;
+
+    let result_str = process_select_response(response).await?;
+
+    println!("CSV Select result: {result_str}");
+
+    // Verify results contain records with age > 28
+    assert!(result_str.contains("Alice,30,New York"));
+    assert!(result_str.contains("Charlie,35,Chicago"));
+    assert!(!result_str.contains("Bob,25,Los Angeles"));
+    assert!(!result_str.contains("Diana,28,Boston"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_csv_aggregation() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_csv(&client).await?;
+
+    // Construct aggregation query - use simpler approach
+    let sql = "SELECT name, age FROM S3Object WHERE age >= 25";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await?;
+
+    let result_str = process_select_response(response).await?;
+
+    println!("CSV Aggregation result: {result_str}");
+
+    // Verify query results - should include records with age >= 25
+    assert!(result_str.contains("Alice"));
+    assert!(result_str.contains("Bob"));
+    assert!(result_str.contains("Charlie"));
+    assert!(result_str.contains("Diana"));
+    assert!(result_str.contains("30"));
+    assert!(result_str.contains("25"));
+    assert!(result_str.contains("35"));
+    assert!(result_str.contains("28"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_json_basic() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_json(&client).await?;
+
+    // Construct JSON query
+    let sql = "SELECT s.name, s.age FROM S3Object s WHERE s.age > 28";
+
+    let json_input = JsonInput::builder().set_type(Some(JsonType::Document)).build();
+
+    let input_serialization = InputSerialization::builder().json(json_input).build();
+
+    let json_output = JsonOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().json(json_output).build();
+
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(JSON_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await?;
+
+    let result_str = process_select_response(response).await?;
+
+    println!("JSON Select result: {result_str}");
+
+    // Verify JSON query results
+    assert!(result_str.contains("Alice"));
+    assert!(result_str.contains("Charlie"));
+    assert!(result_str.contains("30"));
+    assert!(result_str.contains("35"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_csv(&client).await?;
+
+    // Test LIMIT clause
+    let sql = "SELECT * FROM S3Object LIMIT 2";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await?;
+
+    let result_str = process_select_response(response).await?;
+
+    println!("CSV Limit result: {result_str}");
+
+    // Verify only first 2 records are returned
+    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
+    assert_eq!(lines.len(), 2, "Should return exactly 2 records");
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_csv(&client).await?;
+
+    // Test ORDER BY clause
+    let sql = "SELECT name, age FROM S3Object ORDER BY age DESC LIMIT 2";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    let response = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await?;
+
+    let result_str = process_select_response(response).await?;
+
+    println!("CSV Order By result: {result_str}");
+
+    // Verify ordered by age descending
+    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
+    assert!(lines.len() >= 2, "Should return at least 2 records");
+
+    // Check if contains highest age records
+    assert!(result_str.contains("Charlie,35"));
+    assert!(result_str.contains("Alice,30"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_error_handling() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+    upload_test_csv(&client).await?;
+
+    // Test invalid SQL query
+    let sql = "SELECT * FROM S3Object WHERE invalid_column > 10";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    // This query should fail because invalid_column doesn't exist
+    let result = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key(CSV_OBJECT)
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await;
+
+    // Verify query fails (expected behavior)
+    assert!(result.is_err(), "Query with invalid column should fail");
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_select_object_content_nonexistent_object() -> Result<(), Box<dyn Error>> {
+    let client = create_aws_s3_client().await?;
+    setup_test_bucket(&client).await?;
+
+    // Test query on nonexistent object
+    let sql = "SELECT * FROM S3Object";
+
+    let csv_input = CsvInput::builder().file_header_info(FileHeaderInfo::Use).build();
+
+    let input_serialization = InputSerialization::builder().csv(csv_input).build();
+
+    let csv_output = CsvOutput::builder().build();
+    let output_serialization = OutputSerialization::builder().csv(csv_output).build();
+
+    let result = client
+        .select_object_content()
+        .bucket(BUCKET)
+        .key("nonexistent.csv")
+        .expression(sql)
+        .expression_type(ExpressionType::Sql)
+        .input_serialization(input_serialization)
+        .output_serialization(output_serialization)
+        .send()
+        .await;
+
+    // Verify query fails (expected behavior)
+    assert!(result.is_err(), "Query on nonexistent object should fail");
+
+    Ok(())
+}

crates/ecstore/Cargo.toml

@@ -50,7 +50,7 @@ serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
 serde_json.workspace = true
-serde-xml-rs.workspace = true
+quick-xml.workspace = true
 s3s.workspace = true
 http.workspace = true
 url.workspace = true
@@ -66,6 +66,7 @@ rmp-serde.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat"] }
 base64 = { workspace = true }
 hmac = { workspace = true }
+sha1 = { workspace = true }
 sha2 = { workspace = true }
 hex-simd = { workspace = true }
 path-clean = { workspace = true }
@@ -98,6 +99,7 @@ rustfs-filemeta.workspace = true
 rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-rio.workspace = true
 rustfs-signer.workspace = true
+rustfs-checksums.workspace = true
 futures-util.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]

crates/ecstore/src/admin_server_info.rs

@@ -12,23 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend};
 use crate::error::{Error, Result};
 use crate::{
     disk::endpoint::Endpoint,
     global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints},
-    heal::{
-        data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend},
-        data_usage_cache::DataUsageCache,
-        heal_commands::{DRIVE_STATE_OK, DRIVE_STATE_UNFORMATTED},
-    },
     new_object_layer_fn,
     notification_sys::get_global_notification_sys,
     store_api::StorageAPI,
 };
-use rustfs_common::{
-    // error::{Error, Result},
-    globals::GLOBAL_Local_Node_Name,
-};
+
+use crate::data_usage::load_data_usage_cache;
+use rustfs_common::{globals::GLOBAL_Local_Node_Name, heal_channel::DriveState};
 use rustfs_madmin::{
     BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
 };
@@ -253,7 +248,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
 
         warn!("load_data_usage_from_backend end {:?}", after3 - after2);
 
-        let backen_info = store.clone().backend_info().await;
+        let backend_info = store.clone().backend_info().await;
 
         let after4 = OffsetDateTime::now_utc();
 
@@ -272,10 +267,10 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
             backend_type: rustfs_madmin::BackendType::ErasureType,
             online_disks: online_disks.sum(),
             offline_disks: offline_disks.sum(),
-            standard_sc_parity: backen_info.standard_sc_parity,
-            rr_sc_parity: backen_info.rr_sc_parity,
-            total_sets: backen_info.total_sets,
-            drives_per_set: backen_info.drives_per_set,
+            standard_sc_parity: backend_info.standard_sc_parity,
+            rr_sc_parity: backend_info.rr_sc_parity,
+            total_sets: backend_info.total_sets,
+            drives_per_set: backend_info.drives_per_set,
         };
         if get_pools {
             pools = get_pools_info(&all_disks).await.unwrap_or_default();
@@ -318,7 +313,7 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
     for disk in disks_info {
         let ep = &disk.endpoint;
         let state = &disk.state;
-        if *state != DRIVE_STATE_OK && *state != DRIVE_STATE_UNFORMATTED {
+        if *state != DriveState::Ok.to_string() && *state != DriveState::Unformatted.to_string() {
             *offline_disks.get_mut(ep).unwrap() += 1;
             continue;
         }
@@ -359,13 +354,13 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
 
         if erasure_set.id == 0 {
             erasure_set.id = d.set_index;
-            if let Ok(cache) = DataUsageCache::load(
+            if let Ok(cache) = load_data_usage_cache(
                 &store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
                 DATA_USAGE_CACHE_NAME,
             )
             .await
             {
-                let data_usage_info = cache.dui(DATA_USAGE_ROOT, &[]);
+                let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::<String>::new());
                 erasure_set.objects_count = data_usage_info.objects_total_count;
                 erasure_set.versions_count = data_usage_info.versions_total_count;
                 erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -22,6 +22,10 @@ use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
 use futures::Future;
 use http::HeaderMap;
 use lazy_static::lazy_static;
+use rustfs_common::data_usage::TierStats;
+use rustfs_common::heal_channel::rep_has_active_rules;
+use rustfs_common::metrics::{IlmAction, Metrics};
+use rustfs_utils::path::encode_dir_object;
 use s3s::Body;
 use sha2::{Digest, Sha256};
 use std::any::Any;
@@ -31,6 +35,7 @@ use std::io::Write;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::{Arc, Mutex};
+use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
@@ -41,9 +46,10 @@ use xxhash_rust::xxh64;
 //use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger};
 //use rustfs_notify::{initialize, notification_system};
 use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
-use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
+use super::lifecycle::{self, ExpirationOptions, Lifecycle, TransitionOptions};
 use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
 use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
+use crate::bucket::object_lock::objectlock_sys::enforce_retention_for_deletion;
 use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
 use crate::client::object_api_utils::new_getobjectreader;
 use crate::error::Error;
@@ -52,16 +58,11 @@ use crate::event::name::EventName;
 use crate::event_notification::{EventArgs, send_event};
 use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
-use crate::heal::{
-    data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
-    data_scanner_metric::ScannerMetrics,
-    data_usage_cache::TierStats,
-};
 use crate::store::ECStore;
 use crate::store_api::StorageAPI;
 use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
 use crate::tier::warm_backend::WarmBackendGetOpts;
-use s3s::dto::BucketLifecycleConfiguration;
+use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
 
 pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
 pub type TraceFn =
@@ -345,8 +346,12 @@ impl ExpiryState {
     }
 
     pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
+        //let cancel_token =
+        //    get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?;
+
         loop {
             select! {
+                //_ = cancel_token.cancelled() => {
                 _ = tokio::signal::ctrl_c() => {
                     info!("got ctrl+c, exits");
                     break;
@@ -631,7 +636,7 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
     if !lc.is_none() {
         let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
         match event.action {
-            lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+            IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
                 if oi.delete_marker || oi.is_dir {
                     return;
                 }
@@ -728,7 +733,7 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
 }
 
 pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
-    let time_ilm = ScannerMetrics::time_ilm(lae.event.action);
+    let time_ilm = Metrics::time_ilm(lae.event.action);
 
     let opts = ObjectOptions {
         transition: TransitionOptions {
@@ -810,8 +815,8 @@ impl LifecycleOps for ObjectInfo {
             num_versions: self.num_versions,
             delete_marker: self.delete_marker,
             successor_mod_time: self.successor_mod_time,
-            //restore_ongoing:    self.restore_ongoing,
-            //restore_expires:    self.restore_expires,
+            restore_ongoing: self.restore_ongoing,
+            restore_expires: self.restore_expires,
             transition_status: self.transitioned_object.status.clone(),
             ..Default::default()
         }
@@ -842,3 +847,161 @@ pub struct RestoreObjectRequest {
 }
 
 const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
+
+pub async fn eval_action_from_lifecycle(
+    lc: &BucketLifecycleConfiguration,
+    lr: Option<DefaultRetention>,
+    rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>,
+    oi: &ObjectInfo,
+) -> lifecycle::Event {
+    let event = lc.eval(&oi.to_lifecycle_opts()).await;
+    //if serverDebugLog {
+    info!("lifecycle: Secondary scan: {}", event.action);
+    //}
+
+    let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
+
+    match event.action {
+        lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+            if lock_enabled {
+                return lifecycle::Event::default();
+            }
+        }
+        lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => {
+            if oi.version_id.is_none() {
+                return lifecycle::Event::default();
+            }
+            if lock_enabled && enforce_retention_for_deletion(oi) {
+                //if serverDebugLog {
+                if oi.version_id.is_some() {
+                    info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err"));
+                } else {
+                    info!("lifecycle: {} is locked, not deleting", oi.name);
+                }
+                //}
+                return lifecycle::Event::default();
+            }
+            if let Some(rcfg) = rcfg {
+                if rep_has_active_rules(&rcfg.0, &oi.name, true) {
+                    return lifecycle::Event::default();
+                }
+            }
+        }
+        _ => (),
+    }
+
+    event
+}
+
+async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    if oi.delete_marker || oi.is_dir {
+        return false;
+    }
+    GLOBAL_TransitionState.queue_transition_task(oi, event, src).await;
+    true
+}
+
+pub async fn apply_expiry_on_transitioned_object(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    src: &LcEventSrc,
+) -> bool {
+    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
+        return false;
+    }
+    // let _ = time_ilm(1);
+
+    true
+}
+
+pub async fn apply_expiry_on_non_transitioned_objects(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    _src: &LcEventSrc,
+) -> bool {
+    let mut opts = ObjectOptions {
+        expiration: ExpirationOptions { expire: true },
+        ..Default::default()
+    };
+
+    if lc_event.action.delete_versioned() {
+        opts.version_id = Some(oi.version_id.expect("err").to_string());
+    }
+
+    opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await;
+    opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await;
+
+    if lc_event.action.delete_all() {
+        opts.delete_prefix = true;
+        opts.delete_prefix_object = true;
+    }
+
+    // let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+
+    let mut dobj = api
+        .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts)
+        .await
+        .unwrap();
+    if dobj.name.is_empty() {
+        dobj = oi.clone();
+    }
+
+    //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
+    //tags["version-id"] = dobj.version_id;
+
+    let mut event_name = EventName::ObjectRemovedDelete;
+    if oi.delete_marker {
+        event_name = EventName::ObjectRemovedDeleteMarkerCreated;
+    }
+    match lc_event.action {
+        lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
+        lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete,
+        _ => (),
+    }
+    send_event(EventArgs {
+        event_name: event_name.as_ref().to_string(),
+        bucket_name: dobj.bucket.clone(),
+        object: dobj,
+        user_agent: "Internal: [ILM-Expiry]".to_string(),
+        host: GLOBAL_LocalNodeName.to_string(),
+        ..Default::default()
+    });
+
+    if lc_event.action != lifecycle::IlmAction::NoneAction {
+        // let mut num_versions = 1_u64;
+        // if lc_event.action.delete_all() {
+        //     num_versions = oi.num_versions as u64;
+        // }
+        // let _ = time_ilm(num_versions);
+    }
+
+    true
+}
+
+async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    let mut expiry_state = GLOBAL_ExpiryState.write().await;
+    expiry_state.enqueue_by_days(oi, event, src).await;
+    true
+}
+
+pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    let mut success = false;
+    match event.action {
+        lifecycle::IlmAction::DeleteVersionAction
+        | lifecycle::IlmAction::DeleteAction
+        | lifecycle::IlmAction::DeleteRestoredAction
+        | lifecycle::IlmAction::DeleteRestoredVersionAction
+        | lifecycle::IlmAction::DeleteAllVersionsAction
+        | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+            success = apply_expiry_rule(event, src, oi).await;
+        }
+        lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+            success = apply_transition_rule(event, src, oi).await;
+        }
+        _ => (),
+    }
+    success
+}

crates/ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -43,49 +43,7 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
 const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
     "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket";
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum IlmAction {
-    NoneAction = 0,
-    DeleteAction,
-    DeleteVersionAction,
-    TransitionAction,
-    TransitionVersionAction,
-    DeleteRestoredAction,
-    DeleteRestoredVersionAction,
-    DeleteAllVersionsAction,
-    DelMarkerDeleteAllVersionsAction,
-    ActionCount,
-}
-
-impl IlmAction {
-    pub fn delete_restored(&self) -> bool {
-        *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
-    }
-
-    pub fn delete_versioned(&self) -> bool {
-        *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
-    }
-
-    pub fn delete_all(&self) -> bool {
-        *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
-    }
-
-    pub fn delete(&self) -> bool {
-        if self.delete_restored() {
-            return true;
-        }
-        *self == Self::DeleteVersionAction
-            || *self == Self::DeleteAction
-            || *self == Self::DeleteAllVersionsAction
-            || *self == Self::DelMarkerDeleteAllVersionsAction
-    }
-}
-
-impl Display for IlmAction {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
+pub use rustfs_common::metrics::IlmAction;
 
 #[async_trait::async_trait]
 pub trait RuleValidate {
@@ -174,7 +132,7 @@ pub trait Lifecycle {
     async fn has_transition(&self) -> bool;
     fn has_expiry(&self) -> bool;
     async fn has_active_rules(&self, prefix: &str) -> bool;
-    async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error>;
+    async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
     async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
     async fn eval(&self, obj: &ObjectOpts) -> Event;
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event;
@@ -255,7 +213,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
         false
     }
 
-    async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error> {
+    async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error> {
         if self.rules.len() > 1000 {
             return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES));
         }
@@ -265,13 +223,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
 
         for r in &self.rules {
             r.validate()?;
-            if let Some(expiration) = r.expiration.as_ref() {
-                if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
-                    if lr_retention && (expired_object_delete_marker) {
-                        return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+            /*if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
+                if let Some(expiration) = r.expiration.as_ref() {
+                    if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                        if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (expired_object_delete_marker) {
+                            return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+                        }
                     }
-                }
-            }
+                        }
+            }*/
         }
         for (i, _) in self.rules.iter().enumerate() {
             if i == self.rules.len() - 1 {
@@ -642,7 +602,7 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
     }
     let t = mod_time
         .to_offset(offset!(-0:00:00))
-        .saturating_add(Duration::days(0 /*days as i64*/)); //debug
+        .saturating_add(Duration::days(days as i64));
     let mut hour = 3600;
     if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
         if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -25,7 +25,7 @@ use std::ops::Sub;
 use time::OffsetDateTime;
 use tracing::{error, warn};
 
-use crate::heal::data_usage_cache::TierStats;
+use rustfs_common::data_usage::TierStats;
 
 pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
 

crates/ecstore/src/bucket/metadata_sys.rs

@@ -18,9 +18,9 @@ use crate::bucket::utils::{deserialize, is_meta_bucketname};
 use crate::cmd::bucket_targets;
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
-use crate::heal::heal_commands::HealOpts;
 use crate::store::ECStore;
 use futures::future::join_all;
+use rustfs_common::heal_channel::HealOpts;
 use rustfs_policy::policy::BucketPolicy;
 use s3s::dto::{
     BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,

crates/ecstore/src/cache_value/cache.rs

@@ -1,137 +0,0 @@
-#![allow(unsafe_code)] // TODO: audit unsafe code
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
-use std::{
-    fmt::Debug,
-    future::Future,
-    pin::Pin,
-    ptr,
-    sync::{
-        Arc,
-        atomic::{AtomicPtr, AtomicU64, Ordering},
-    },
-    time::{Duration, SystemTime, UNIX_EPOCH},
-};
-
-use tokio::{spawn, sync::Mutex};
-
-use std::io::Result;
-
-pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>;
-
-#[derive(Clone, Debug, Default)]
-pub struct Opts {
-    return_last_good: bool,
-    no_wait: bool,
-}
-
-pub struct Cache<T: Clone + Debug + Send> {
-    update_fn: UpdateFn<T>,
-    ttl: Duration,
-    opts: Opts,
-    val: AtomicPtr<T>,
-    last_update_ms: AtomicU64,
-    updating: Arc<Mutex<bool>>,
-}
-
-impl<T: Clone + Debug + Send + 'static> Cache<T> {
-    pub fn new(update_fn: UpdateFn<T>, ttl: Duration, opts: Opts) -> Self {
-        let val = AtomicPtr::new(ptr::null_mut());
-        Self {
-            update_fn,
-            ttl,
-            opts,
-            val,
-            last_update_ms: AtomicU64::new(0),
-            updating: Arc::new(Mutex::new(false)),
-        }
-    }
-
-    pub async fn get(self: Arc<Self>) -> Result<T> {
-        let v_ptr = self.val.load(Ordering::SeqCst);
-        let v = if v_ptr.is_null() {
-            None
-        } else {
-            Some(unsafe { (*v_ptr).clone() })
-        };
-
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .expect("Time went backwards")
-            .as_secs();
-        if now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() {
-            if let Some(v) = v {
-                return Ok(v);
-            }
-        }
-
-        if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 {
-            if self.updating.try_lock().is_ok() {
-                let this = Arc::clone(&self);
-                spawn(async move {
-                    let _ = this.update().await;
-                });
-            }
-
-            return Ok(v.unwrap());
-        }
-
-        let _ = self.updating.lock().await;
-
-        if let Ok(duration) =
-            SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst)))
-        {
-            if duration < self.ttl {
-                return Ok(v.unwrap());
-            }
-        }
-
-        match self.update().await {
-            Ok(_) => {
-                let v_ptr = self.val.load(Ordering::SeqCst);
-                let v = if v_ptr.is_null() {
-                    None
-                } else {
-                    Some(unsafe { (*v_ptr).clone() })
-                };
-                Ok(v.unwrap())
-            }
-            Err(err) => Err(err),
-        }
-    }
-
-    async fn update(&self) -> Result<()> {
-        match (self.update_fn)().await {
-            Ok(val) => {
-                self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst);
-                let now = SystemTime::now()
-                    .duration_since(UNIX_EPOCH)
-                    .expect("Time went backwards")
-                    .as_secs();
-                self.last_update_ms.store(now, Ordering::SeqCst);
-                Ok(())
-            }
-            Err(err) => {
-                let v_ptr = self.val.load(Ordering::SeqCst);
-                if self.opts.return_last_good && !v_ptr.is_null() {
-                    return Ok(());
-                }
-
-                Err(err)
-            }
-        }
-    }
-}

crates/ecstore/src/cache_value/metacache_set.rs

@@ -31,7 +31,7 @@ pub struct ListPathRawOptions {
     pub fallback_disks: Vec<Option<DiskStore>>,
     pub bucket: String,
     pub path: String,
-    pub recursice: bool,
+    pub recursive: bool,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -52,7 +52,7 @@ impl Clone for ListPathRawOptions {
             fallback_disks: self.fallback_disks.clone(),
             bucket: self.bucket.clone(),
             path: self.path.clone(),
-            recursice: self.recursice,
+            recursive: self.recursive,
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -85,7 +85,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
             let wakl_opts = WalkDirOptions {
                 bucket: opts_clone.bucket.clone(),
                 base_dir: opts_clone.path.clone(),
-                recursive: opts_clone.recursice,
+                recursive: opts_clone.recursive,
                 report_notfound: opts_clone.report_not_found,
                 filter_prefix: opts_clone.filter_prefix.clone(),
                 forward_to: opts_clone.forward_to.clone(),
@@ -133,7 +133,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
                         WalkDirOptions {
                             bucket: opts_clone.bucket.clone(),
                             base_dir: opts_clone.path.clone(),
-                            recursive: opts_clone.recursice,
+                            recursive: opts_clone.recursive,
                             report_notfound: opts_clone.report_not_found,
                             filter_prefix: opts_clone.filter_prefix.clone(),
                             forward_to: opts_clone.forward_to.clone(),

crates/ecstore/src/cache_value/mod.rs

@@ -12,5 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// pub mod cache;
+use std::sync::Arc;
+
+use lazy_static::lazy_static;
+use tokio_util::sync::CancellationToken;
+
 pub mod metacache_set;
+
+lazy_static! {
+    pub static ref LIST_PATH_RAW_CANCEL_TOKEN: Arc<CancellationToken> = Arc::new(CancellationToken::new());
+}

crates/ecstore/src/checksum.rs

@@ -20,7 +20,7 @@
 #![allow(clippy::all)]
 
 use lazy_static::lazy_static;
-use rustfs_utils::HashAlgorithm;
+use rustfs_checksums::ChecksumAlgorithm;
 use std::collections::HashMap;
 
 use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
@@ -103,15 +103,34 @@ impl ChecksumMode {
     }
 
     pub fn can_composite(&self) -> bool {
-        todo!();
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            2_u8 => true,
+            4_u8 => true,
+            8_u8 => true,
+            16_u8 => true,
+            _ => false,
+        }
     }
 
     pub fn can_merge_crc(&self) -> bool {
-        todo!();
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            8_u8 => true,
+            16_u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
     }
 
     pub fn full_object_requested(&self) -> bool {
-        todo!();
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            //C_ChecksumFullObjectCRC32 as u8 => true,
+            //C_ChecksumFullObjectCRC32C as u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
     }
 
     pub fn key_capitalized(&self) -> String {
@@ -123,33 +142,35 @@ impl ChecksumMode {
         if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
             4
         } else if u == ChecksumMode::ChecksumSHA1 as u8 {
-            4 //sha1.size
+            use sha1::Digest;
+            sha1::Sha1::output_size() as usize
         } else if u == ChecksumMode::ChecksumSHA256 as u8 {
-            4 //sha256.size
+            use sha2::Digest;
+            sha2::Sha256::output_size() as usize
         } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
-            4 //crc64.size
+            8
         } else {
             0
         }
     }
 
-    pub fn hasher(&self) -> Result<HashAlgorithm, std::io::Error> {
+    pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
         match /*C_ChecksumMask & **/self {
-            /*ChecksumMode::ChecksumCRC32 => {
-                return Ok(Box::new(crc32fast::Hasher::new()));
-            }*/
-            /*ChecksumMode::ChecksumCRC32C => {
-                return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli))));
+            ChecksumMode::ChecksumCRC32 => {
+                return Ok(ChecksumAlgorithm::Crc32.into_impl());
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return Ok(ChecksumAlgorithm::Crc32c.into_impl());
             }
             ChecksumMode::ChecksumSHA1 => {
-                return Ok(Box::new(sha1::new()));
-            }*/
-            ChecksumMode::ChecksumSHA256 => {
-                return Ok(HashAlgorithm::SHA256);
+                return Ok(ChecksumAlgorithm::Sha1.into_impl());
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return Ok(ChecksumAlgorithm::Sha256.into_impl());
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
             }
-            /*ChecksumMode::ChecksumCRC64NVME => {
-                return Ok(Box::new(crc64nvme.New());
-            }*/
             _ => return Err(std::io::Error::other("unsupported checksum type")),
         }
     }
@@ -170,7 +191,8 @@ impl ChecksumMode {
             return Ok("".to_string());
         }
         let mut h = self.hasher()?;
-        let hash = h.hash_encode(b);
+        h.update(b);
+        let hash = h.finalize();
         Ok(base64_encode(hash.as_ref()))
     }
 
@@ -227,7 +249,8 @@ impl ChecksumMode {
         let c = self.base();
         let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
         let mut h = self.hasher()?;
-        let hash = h.hash_encode(crc_bytes.as_ref());
+        h.update(crc_bytes.as_ref());
+        let hash = h.finalize();
         Ok(Checksum {
             checksum_type: self.clone(),
             r: hash.as_ref().to_vec(),

crates/ecstore/src/chunk_stream.rs

@@ -41,7 +41,7 @@
 //                 pin_mut!(body);
 //                 // 上一次没用完的数据
 //                 let mut prev_bytes = Bytes::new();
-//                 let mut readed_size = 0;
+//                 let mut read_size = 0;
 
 //                 loop {
 //                     let data: Vec<Bytes> = {
@@ -51,9 +51,9 @@
 //                             Some(Err(e)) => return Err(e),
 //                             Some(Ok((data, remaining_bytes))) => {
 //                                 // debug!(
-//                                 //     "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ",
+//                                 //     "content_length:{},read_size:{}, read_data data:{}, remaining_bytes: {} ",
 //                                 //     content_length,
-//                                 //     readed_size,
+//                                 //     read_size,
 //                                 //     data.len(),
 //                                 //     remaining_bytes.len()
 //                                 // );
@@ -65,15 +65,15 @@
 //                     };
 
 //                     for bytes in data {
-//                         readed_size += bytes.len();
-//                         // debug!("readed_size {}, content_length {}", readed_size, content_length,);
+//                         read_size += bytes.len();
+//                         // debug!("read_size {}, content_length {}", read_size, content_length,);
 //                         y.yield_ok(bytes).await;
 //                     }
 
-//                     if readed_size + prev_bytes.len() >= content_length {
+//                     if read_size + prev_bytes.len() >= content_length {
 //                         // debug!(
-//                         //     "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}",
-//                         //     readed_size,
+//                         //     "读完了 read_size:{} + prev_bytes.len({}) == content_length {}",
+//                         //     read_size,
 //                         //     prev_bytes.len(),
 //                         //     content_length,
 //                         // );

crates/ecstore/src/client/api_bucket_policy.rs

@@ -63,7 +63,7 @@ impl TransitionClient {
         //defer closeResponse(resp)
         //if resp != nil {
         if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
         }
         //}
         Ok(())
@@ -98,7 +98,7 @@ impl TransitionClient {
         //defer closeResponse(resp)
 
         if resp.status() != StatusCode::NO_CONTENT {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
         }
 
         Ok(())

crates/ecstore/src/client/api_error_response.rs

@@ -95,13 +95,13 @@ pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
 }
 
 pub fn http_resp_to_error_response(
-    resp: http::Response<Body>,
+    resp: &http::Response<Body>,
     b: Vec<u8>,
     bucket_name: &str,
     object_name: &str,
 ) -> ErrorResponse {
     let err_body = String::from_utf8(b).unwrap();
-    let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
+    let err_resp_ = quick_xml::de::from_str::<ErrorResponse>(&err_body);
     let mut err_resp = ErrorResponse::default();
     if err_resp_.is_err() {
         match resp.status() {

crates/ecstore/src/client/api_get_object_acl.rs

@@ -87,11 +87,11 @@ impl TransitionClient {
 
         if resp.status() != http::StatusCode::OK {
             let b = resp.body().bytes().expect("err").to_vec();
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, b, bucket_name, object_name)));
+            return Err(std::io::Error::other(http_resp_to_error_response(&resp, b, bucket_name, object_name)));
         }
 
         let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
-        let mut res = match serde_xml_rs::from_str::<AccessControlPolicy>(&String::from_utf8(b).unwrap()) {
+        let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(b).unwrap()) {
             Ok(result) => result,
             Err(err) => {
                 return Err(std::io::Error::other(err.to_string()));

crates/ecstore/src/client/api_get_object_attributes.rs

@@ -144,7 +144,7 @@ impl ObjectAttributes {
         self.version_id = h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap().to_string();
 
         let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
-        let mut response = match serde_xml_rs::from_str::<ObjectAttributesResponse>(&String::from_utf8(b).unwrap()) {
+        let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&String::from_utf8(b).unwrap()) {
             Ok(result) => result,
             Err(err) => {
                 return Err(std::io::Error::other(err.to_string()));
@@ -226,7 +226,7 @@ impl TransitionClient {
         if resp.status() != http::StatusCode::OK {
             let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
             let err_body = String::from_utf8(b).unwrap();
-            let mut er = match serde_xml_rs::from_str::<AccessControlPolicy>(&err_body) {
+            let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
                 Ok(result) => result,
                 Err(err) => {
                     return Err(std::io::Error::other(err.to_string()));

crates/ecstore/src/client/api_list.rs

@@ -98,12 +98,12 @@ impl TransitionClient {
             )
             .await?;
         if resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
         }
 
         //let mut list_bucket_result = ListBucketV2Result::default();
         let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
-        let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
+        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
             Ok(result) => result,
             Err(err) => {
                 return Err(std::io::Error::other(err.to_string()));

crates/ecstore/src/client/api_put_object.rs

@@ -85,7 +85,7 @@ pub struct PutObjectOptions {
     pub expires: OffsetDateTime,
     pub mode: ObjectLockRetentionMode,
     pub retain_until_date: OffsetDateTime,
-    //pub server_side_encryption: encrypt.ServerSide,
+    //pub server_side_encryption: encrypt::ServerSide,
     pub num_threads: u64,
     pub storage_class: String,
     pub website_redirect_location: String,
@@ -135,7 +135,7 @@ impl Default for PutObjectOptions {
 
 #[allow(dead_code)]
 impl PutObjectOptions {
-    fn set_matche_tag(&mut self, etag: &str) {
+    fn set_match_etag(&mut self, etag: &str) {
         if etag == "*" {
             self.custom_header
                 .insert("If-Match", HeaderValue::from_str("*").expect("err"));
@@ -145,7 +145,7 @@ impl PutObjectOptions {
         }
     }
 
-    fn set_matche_tag_except(&mut self, etag: &str) {
+    fn set_match_etag_except(&mut self, etag: &str) {
         if etag == "*" {
             self.custom_header
                 .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
@@ -181,7 +181,7 @@ impl PutObjectOptions {
             header.insert(
                 "Expires",
                 HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            ); //rustfs invalid heade
+            ); //rustfs invalid header
         }
 
         if self.mode.as_str() != "" {
@@ -366,7 +366,8 @@ impl TransitionClient {
                 md5_base64 = base64_encode(hash.as_ref());
             } else {
                 let mut crc = opts.auto_checksum.hasher()?;
-                let csum = crc.hash_encode(&buf[..length]);
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
                     custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));

crates/ecstore/src/client/api_put_object_common.rs

@@ -12,7 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -19,20 +18,14 @@
 #![allow(clippy::all)]
 
 use bytes::Bytes;
-use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use http::{HeaderMap, HeaderName, StatusCode};
 use s3s::S3ErrorCode;
-use std::io::Read;
-use std::{collections::HashMap, sync::Arc};
-use time::{OffsetDateTime, format_description};
-use tokio_util::sync::CancellationToken;
+use std::collections::HashMap;
+use time::OffsetDateTime;
 use tracing::warn;
-use tracing::{error, info};
-use url::form_urlencoded::Serializer;
 use uuid::Uuid;
 
-use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
-use s3s::{Body, dto::StreamingBlob};
-//use crate::disk::{Reader, BufferReader};
+use crate::checksum::ChecksumMode;
 use crate::client::{
     api_error_response::{
         err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
@@ -42,15 +35,11 @@ use crate::client::{
     api_s3_datatypes::{
         CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
     },
-    constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
+    constants::{ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
     transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
 };
-use crate::{
-    checksum::ChecksumMode,
-    disk::DiskAPI,
-    store_api::{GetObjectReader, StorageAPI},
-};
 use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
 
 impl TransitionClient {
     pub async fn put_object_multipart(
@@ -133,7 +122,8 @@ impl TransitionClient {
             //}
             if hash_sums.len() == 0 {
                 let mut crc = opts.auto_checksum.hasher()?;
-                let csum = crc.hash_encode(&buf[..length]);
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
                     custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
@@ -236,7 +226,12 @@ impl TransitionClient {
         let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
         //if resp.is_none() {
         if resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                &resp,
+                vec![],
+                bucket_name,
+                object_name,
+            )));
         }
         //}
         let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
@@ -293,7 +288,7 @@ impl TransitionClient {
         let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
         if resp.status() != StatusCode::OK {
             return Err(std::io::Error::other(http_resp_to_error_response(
-                resp,
+                &resp,
                 vec![],
                 &p.bucket_name.clone(),
                 &p.object_name,

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -156,7 +156,8 @@ impl TransitionClient {
                 md5_base64 = base64_encode(hash.as_ref());
             } else {
                 let mut crc = opts.auto_checksum.hasher()?;
-                let csum = crc.hash_encode(&buf[..length]);
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
                     custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
@@ -303,7 +304,8 @@ impl TransitionClient {
             let mut custom_header = HeaderMap::new();
             if !opts.send_content_md5 {
                 let mut crc = opts.auto_checksum.hasher()?;
-                let csum = crc.hash_encode(&buf[..length]);
+                crc.update(&buf[..length]);
+                let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
                     custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
@@ -477,7 +479,12 @@ impl TransitionClient {
         let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
 
         if resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                &resp,
+                vec![],
+                bucket_name,
+                object_name,
+            )));
         }
 
         let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {

crates/ecstore/src/client/api_remove.rs

@@ -425,7 +425,12 @@ impl TransitionClient {
                     };
                 }
                 _ => {
-                    return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+                    return Err(std::io::Error::other(http_resp_to_error_response(
+                        &resp,
+                        vec![],
+                        bucket_name,
+                        object_name,
+                    )));
                 }
             }
             return Err(std::io::Error::other(error_response));

crates/ecstore/src/client/api_restore.rs

@@ -125,7 +125,7 @@ impl TransitionClient {
         version_id: &str,
         restore_req: &RestoreRequest,
     ) -> Result<(), std::io::Error> {
-        let restore_request = match serde_xml_rs::to_string(restore_req) {
+        let restore_request = match quick_xml::se::to_string(restore_req) {
             Ok(buf) => buf,
             Err(e) => {
                 return Err(std::io::Error::other(e));
@@ -165,7 +165,7 @@ impl TransitionClient {
 
         let b = resp.body().bytes().expect("err").to_vec();
         if resp.status() != http::StatusCode::ACCEPTED && resp.status() != http::StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(resp, b, bucket_name, "")));
+            return Err(std::io::Error::other(http_resp_to_error_response(&resp, b, bucket_name, "")));
         }
         Ok(())
     }

crates/ecstore/src/client/api_s3_datatypes.rs

@@ -279,7 +279,7 @@ pub struct CompleteMultipartUpload {
 impl CompleteMultipartUpload {
     pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
         //let buf = serde_json::to_string(self)?;
-        let buf = match serde_xml_rs::to_string(self) {
+        let buf = match quick_xml::se::to_string(self) {
             Ok(buf) => buf,
             Err(e) => {
                 return Err(std::io::Error::other(e));
@@ -329,7 +329,7 @@ pub struct DeleteMultiObjects {
 impl DeleteMultiObjects {
     pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
         //let buf = serde_json::to_string(self)?;
-        let buf = match serde_xml_rs::to_string(self) {
+        let buf = match quick_xml::se::to_string(self) {
             Ok(buf) => buf,
             Err(e) => {
                 return Err(std::io::Error::other(e));

crates/ecstore/src/client/api_stat.rs

@@ -59,7 +59,7 @@ impl TransitionClient {
 
         if let Ok(resp) = resp {
             let b = resp.body().bytes().expect("err").to_vec();
-            let resperr = http_resp_to_error_response(resp, b, bucket_name, "");
+            let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
             /*if to_error_response(resperr).code == "NoSuchBucket" {
                 return Ok(false);
             }

crates/ecstore/src/client/bucket_cache.rs

@@ -177,7 +177,7 @@ impl TransitionClient {
 async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
     //if resp != nil {
     if resp.status() != StatusCode::OK {
-        let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
+        let err_resp = http_resp_to_error_response(&resp, vec![], bucket_name, "");
         match err_resp.code {
                 S3ErrorCode::NotImplemented => {
                     match err_resp.server.as_str() {
@@ -208,7 +208,7 @@ async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket
     //}
 
     let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
-    let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
+    let Document(location_constraint) = quick_xml::de::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
 
     let mut location = location_constraint;
     if location == "" {

crates/ecstore/src/client/object_handlers_common.rs

@@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::store::ECStore;
 use crate::store_api::{ObjectOptions, ObjectToDelete};
-use rustfs_lock::local_locker::MAX_DELETE_LIST;
+use rustfs_lock::MAX_DELETE_LIST;
 
 pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
     let mut remaining = to_del;

crates/ecstore/src/client/transition_api.rs

@@ -19,7 +19,7 @@
 #![allow(clippy::all)]
 
 use bytes::Bytes;
-use futures::Future;
+use futures::{Future, StreamExt};
 use http::{HeaderMap, HeaderName};
 use http::{
     HeaderValue, Response, StatusCode,
@@ -65,7 +65,9 @@ use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
 use rustfs_rio::HashReader;
 use rustfs_utils::{
     net::get_endpoint_url,
-    retry::{MAX_RETRY, new_retry_timer},
+    retry::{
+        DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable,
+    },
 };
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
@@ -186,6 +188,7 @@ impl TransitionClient {
 
         clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4;
 
+        clnt.max_retries = MAX_RETRY;
         if opts.max_retries > 0 {
             clnt.max_retries = opts.max_retries;
         }
@@ -313,12 +316,9 @@ impl TransitionClient {
         }
         //}
 
-        //let mut retry_timer = RetryTimer::new();
-        //while let Some(v) = retry_timer.next().await {
-        for _ in [1; 1]
-        /*new_retry_timer(req_retry, default_retry_unit, default_retry_cap, max_jitter)*/
-        {
-            let req = self.new_request(method, metadata).await?;
+        let mut retry_timer = RetryTimer::new(req_retry, DEFAULT_RETRY_UNIT, DEFAULT_RETRY_CAP, MAX_JITTER, self.random);
+        while let Some(v) = retry_timer.next().await {
+            let req = self.new_request(&method, metadata).await?;
 
             resp = self.doit(req).await?;
 
@@ -329,7 +329,7 @@ impl TransitionClient {
             }
 
             let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
-            let err_response = http_resp_to_error_response(resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
+            let err_response = http_resp_to_error_response(&resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
 
             if self.region == "" {
                 match err_response.code {
@@ -360,6 +360,14 @@ impl TransitionClient {
                 }
             }
 
+            if is_s3code_retryable(err_response.code.as_str()) {
+                continue;
+            }
+
+            if is_http_status_retryable(&resp.status()) {
+                continue;
+            }
+
             break;
         }
 
@@ -368,7 +376,7 @@ impl TransitionClient {
 
     async fn new_request(
         &self,
-        method: http::Method,
+        method: &http::Method,
         metadata: &mut RequestMetadata,
     ) -> Result<http::Request<Body>, std::io::Error> {
         let location = metadata.bucket_location.clone();

crates/ecstore/src/cmd/bucket_replication.rs

@@ -2014,6 +2014,8 @@ impl ReplicateObjectInfo {
             version_id: Uuid::try_parse(&self.version_id).ok(),
             delete_marker: self.delete_marker,
             transitioned_object: TransitionedObject::default(),
+            restore_ongoing: false,
+            restore_expires: Some(OffsetDateTime::now_utc()),
             user_tags: self.user_tags.clone(),
             parts: Vec::new(),
             is_latest: true,
@@ -2422,7 +2424,7 @@ impl ReplicateObjectInfo {
 // let mut arns = Vec::new();
 // let mut tgts_map = std::collections::HashSet::new();
 // for rule in cfg.rules {
-//     if rule.status.as_str() == "Disabe" {
+//     if rule.status.as_str() == "Disable" {
 //         continue;
 //     }
 

crates/ecstore/src/cmd/bucket_targets.rs

@@ -95,7 +95,7 @@ impl ArnTarget {
         Self {
             client: TargetClient {
                 bucket,
-                storage_class: "STANDRD".to_string(),
+                storage_class: "STANDARD".to_string(),
                 disable_proxy: false,
                 health_check_duration: Duration::from_secs(100),
                 endpoint,
@@ -361,7 +361,7 @@ impl BucketTargetSys {
     //     // Mocked implementation for obtaining a remote client
     //     let tcli = TargetClient {
     //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDRD".to_string(),
+    //         storage_class: "STANDARD".to_string(),
     //         disable_proxy: false,
     //         health_check_duration: Duration::from_secs(100),
     //         endpoint: _tgt.endpoint.clone(),
@@ -379,7 +379,7 @@ impl BucketTargetSys {
     //     // Mocked implementation for obtaining a remote client
     //     let tcli = TargetClient {
     //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDRD".to_string(),
+    //         storage_class: "STANDARD".to_string(),
     //         disable_proxy: false,
     //         health_check_duration: Duration::from_secs(100),
     //         endpoint: _tgt.endpoint.clone(),
@@ -403,7 +403,7 @@ impl BucketTargetSys {
         match store.get_bucket_info(_bucket, &store_api::BucketOptions::default()).await {
             Ok(info) => {
                 println!("Bucket Info: {info:?}");
-                info.versionning
+                info.versioning
             }
             Err(err) => {
                 eprintln!("Error: {err:?}");
@@ -431,7 +431,7 @@ impl BucketTargetSys {
         // {
         //     Ok(info) => {
         //         println!("Bucket Info: {:?}", info);
-        //         info.versionning
+        //         info.versioning
         //     }
         //     Err(err) => {
         //         eprintln!("Error: {:?}", err);
@@ -475,8 +475,7 @@ impl BucketTargetSys {
         {
             Ok(info) => {
                 println!("Bucket Info: {info:?}");
-                if !info.versionning {
-                    println!("2222222222 {}", info.versionning);
+                if !info.versioning {
                     return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.to_string()));
                 }
             }

crates/ecstore/src/config/com.rs

@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::{Config, GLOBAL_StorageClass, storageclass};
+use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result};
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use lazy_static::lazy_static;
 use rustfs_config::DEFAULT_DELIMITER;
 use rustfs_utils::path::SLASH_SEPARATOR;
 use std::collections::HashSet;
 use std::sync::Arc;
+use std::sync::LazyLock;
 use tracing::{error, warn};
 
 pub const CONFIG_PREFIX: &str = "config";
@@ -29,14 +29,13 @@ const CONFIG_FILE: &str = "config.json";
 
 pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
 
-lazy_static! {
-    static ref CONFIG_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, CONFIG_PREFIX);
-    static ref SubSystemsDynamic: HashSet<String> = {
-        let mut h = HashSet::new();
-        h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
-        h
-    };
-}
+static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
+
+static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
+    let mut h = HashSet::new();
+    h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
+    h
+});
 pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
     Ok(data)
@@ -197,7 +196,7 @@ pub async fn lookup_configs<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) {
 }
 
 async fn apply_dynamic_config<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) -> Result<()> {
-    for key in SubSystemsDynamic.iter() {
+    for key in SUB_SYSTEMS_DYNAMIC.iter() {
         apply_dynamic_config_for_sub_sys(cfg, api.clone(), key).await?;
     }
 
@@ -212,9 +211,9 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
         for (i, count) in set_drive_counts.iter().enumerate() {
             match storageclass::lookup_config(&kvs, *count) {
                 Ok(res) => {
-                    if i == 0 && GLOBAL_StorageClass.get().is_none() {
-                        if let Err(r) = GLOBAL_StorageClass.set(res) {
-                            error!("GLOBAL_StorageClass.set failed {:?}", r);
+                    if i == 0 && GLOBAL_STORAGE_CLASS.get().is_none() {
+                        if let Err(r) = GLOBAL_STORAGE_CLASS.set(res) {
+                            error!("GLOBAL_STORAGE_CLASS.set failed {:?}", r);
                         }
                     }
                 }

crates/ecstore/src/config/mod.rs

@@ -21,26 +21,17 @@ pub mod storageclass;
 use crate::error::Result;
 use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
-use lazy_static::lazy_static;
 use rustfs_config::DEFAULT_DELIMITER;
+use rustfs_config::notify::{COMMENT_KEY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::sync::LazyLock;
 use std::sync::{Arc, OnceLock};
 
-lazy_static! {
-    pub static ref GLOBAL_StorageClass: OnceLock<storageclass::Config> = OnceLock::new();
-    pub static ref DefaultKVS: OnceLock<HashMap<String, KVS>> = OnceLock::new();
-    pub static ref GLOBAL_ServerConfig: OnceLock<Config> = OnceLock::new();
-    pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
-}
-
-/// Standard config keys and values.
-pub const ENABLE_KEY: &str = "enable";
-pub const COMMENT_KEY: &str = "comment";
-
-/// Enable values
-pub const ENABLE_ON: &str = "on";
-pub const ENABLE_OFF: &str = "off";
+pub static GLOBAL_STORAGE_CLASS: LazyLock<OnceLock<storageclass::Config>> = LazyLock::new(OnceLock::new);
+pub static DEFAULT_KVS: LazyLock<OnceLock<HashMap<String, KVS>>> = LazyLock::new(OnceLock::new);
+pub static GLOBAL_SERVER_CONFIG: LazyLock<OnceLock<Config>> = LazyLock::new(OnceLock::new);
+pub static GLOBAL_CONFIG_SYS: LazyLock<ConfigSys> = LazyLock::new(ConfigSys::new);
 
 pub const ENV_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 pub const ENV_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
@@ -66,7 +57,7 @@ impl ConfigSys {
 
         lookup_configs(&mut cfg, api).await;
 
-        let _ = GLOBAL_ServerConfig.set(cfg);
+        let _ = GLOBAL_SERVER_CONFIG.set(cfg);
 
         Ok(())
     }
@@ -131,6 +122,28 @@ impl KVS {
 
         keys
     }
+
+    /// Insert or update a pair of key/values in KVS
+    pub fn insert(&mut self, key: String, value: String) {
+        for kv in self.0.iter_mut() {
+            if kv.key == key {
+                kv.value = value.clone();
+                return;
+            }
+        }
+        self.0.push(KV {
+            key,
+            value,
+            hidden_if_empty: false,
+        });
+    }
+
+    /// Merge all entries from another KVS to the current instance
+    pub fn extend(&mut self, other: KVS) {
+        for KV { key, value, .. } in other.0.into_iter() {
+            self.insert(key, value);
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -159,7 +172,7 @@ impl Config {
     }
 
     pub fn set_defaults(&mut self) {
-        if let Some(defaults) = DefaultKVS.get() {
+        if let Some(defaults) = DEFAULT_KVS.get() {
             for (k, v) in defaults.iter() {
                 if !self.0.contains_key(k) {
                     let mut default = HashMap::new();
@@ -198,20 +211,17 @@ pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
         p.insert(k, v);
     }
 
-    let _ = DefaultKVS.set(p);
+    let _ = DEFAULT_KVS.set(p);
 }
 
 pub fn init() {
     let mut kvs = HashMap::new();
     // Load storageclass default configuration
-    kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone());
+    kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DEFAULT_KVS.clone());
     // New: Loading default configurations for notify_webhook and notify_mqtt
     // Referring subsystem names through constants to improve the readability and maintainability of the code
-    kvs.insert(
-        rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_owned(),
-        notify::DefaultWebhookKVS.clone(),
-    );
-    kvs.insert(rustfs_config::notify::NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DefaultMqttKVS.clone());
+    kvs.insert(NOTIFY_WEBHOOK_SUB_SYS.to_owned(), notify::DEFAULT_WEBHOOK_KVS.clone());
+    kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_MQTT_KVS.clone());
 
     // Register all default configurations
     register_default_kvs(kvs)

crates/ecstore/src/config/notify.rs

@@ -12,40 +12,120 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::config::{ENABLE_KEY, ENABLE_OFF, KV, KVS};
-use lazy_static::lazy_static;
+use crate::config::{KV, KVS};
 use rustfs_config::notify::{
-    DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
-    MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    COMMENT_KEY, DEFAULT_DIR, DEFAULT_LIMIT, ENABLE_KEY, ENABLE_OFF, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
+    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
 };
+use std::sync::LazyLock;
 
-lazy_static! {
-    /// The default configuration collection of webhooks，
-    /// Use lazy_static! to ensure that these configurations are initialized only once during the program life cycle, enabling high-performance lazy loading.
-    pub static ref DefaultWebhookKVS: KVS = KVS(vec![
-        KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
-        KV { key: WEBHOOK_ENDPOINT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+/// The default configuration collection of webhooks，
+/// Initialized only once during the program life cycle, enabling high-performance lazy loading.
+pub static DEFAULT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: ENABLE_OFF.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_ENDPOINT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
         // Sensitive information such as authentication tokens is hidden when the value is empty, enhancing security
-        KV { key: WEBHOOK_AUTH_TOKEN.to_owned(), value: "".to_owned(), hidden_if_empty: true },
-        KV { key: WEBHOOK_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
-        KV { key: WEBHOOK_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
-        KV { key: WEBHOOK_CLIENT_CERT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
-        KV { key: WEBHOOK_CLIENT_KEY.to_owned(), value: "".to_owned(), hidden_if_empty: false },
-    ]);
+        KV {
+            key: WEBHOOK_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: WEBHOOK_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_QUEUE_DIR.to_owned(),
+            value: DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
 
-    /// MQTT's default configuration collection
-    pub static ref DefaultMqttKVS: KVS = KVS(vec![
-        KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_BROKER.to_owned(), value: "".to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_TOPIC.to_owned(), value: "".to_owned(), hidden_if_empty: false },
+/// MQTT's default configuration collection
+pub static DEFAULT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: ENABLE_OFF.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
         // Sensitive information such as passwords are hidden when the value is empty
-        KV { key: MQTT_PASSWORD.to_owned(), value: "".to_owned(), hidden_if_empty: true },
-        KV { key: MQTT_USERNAME.to_owned(), value: "".to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_QOS.to_owned(), value: "0".to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_KEEP_ALIVE_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_RECONNECT_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
-        KV { key: MQTT_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
-    ]);
-}
+        KV {
+            key: MQTT_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: MQTT_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_QOS.to_owned(),
+            value: "0".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_KEEP_ALIVE_INTERVAL.to_owned(),
+            value: "0s".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_RECONNECT_INTERVAL.to_owned(),
+            value: "0s".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_QUEUE_DIR.to_owned(),
+            value: DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/storageclass.rs

@@ -15,9 +15,9 @@
 use super::KVS;
 use crate::config::KV;
 use crate::error::{Error, Result};
-use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 use std::env;
+use std::sync::LazyLock;
 use tracing::warn;
 
 /// Default parity count for a given drive count
@@ -62,34 +62,32 @@ pub const DEFAULT_RRS_PARITY: usize = 1;
 
 pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
 
-lazy_static! {
-    pub static ref DefaultKVS: KVS = {
-        let kvs = vec![
-            KV {
-                key: CLASS_STANDARD.to_owned(),
-                value: "".to_owned(),
-                hidden_if_empty: false,
-            },
-            KV {
-                key: CLASS_RRS.to_owned(),
-                value: "EC:1".to_owned(),
-                hidden_if_empty: false,
-            },
-            KV {
-                key: OPTIMIZE.to_owned(),
-                value: "availability".to_owned(),
-                hidden_if_empty: false,
-            },
-            KV {
-                key: INLINE_BLOCK.to_owned(),
-                value: "".to_owned(),
-                hidden_if_empty: true,
-            },
-        ];
+pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    let kvs = vec![
+        KV {
+            key: CLASS_STANDARD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: CLASS_RRS.to_owned(),
+            value: "EC:1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: OPTIMIZE.to_owned(),
+            value: "availability".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: INLINE_BLOCK.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+    ];
 
-        KVS(kvs)
-    };
-}
+    KVS(kvs)
+});
 
 // StorageClass - holds storage class information
 #[derive(Serialize, Deserialize, Debug, Default)]

crates/ecstore/src/data_usage.rs

@@ -0,0 +1,297 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore};
+use rustfs_common::data_usage::{BucketTargetUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, SizeSummary};
+use rustfs_utils::path::SLASH_SEPARATOR;
+use tracing::{error, warn};
+
+use crate::error::Error;
+
+// Data usage storage constants
+pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
+const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
+const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
+pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
+
+// Data usage storage paths
+lazy_static::lazy_static! {
+    pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
+        crate::disk::RUSTFS_META_BUCKET,
+        SLASH_SEPARATOR,
+        crate::disk::BUCKET_META_PREFIX
+    );
+    pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
+        crate::disk::BUCKET_META_PREFIX,
+        SLASH_SEPARATOR,
+        DATA_USAGE_OBJ_NAME
+    );
+    pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
+        crate::disk::BUCKET_META_PREFIX,
+        SLASH_SEPARATOR,
+        DATA_USAGE_BLOOM_NAME
+    );
+}
+
+/// Store data usage info to backend storage
+pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
+    let data =
+        serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
+
+    // Save to backend using the same mechanism as original code
+    crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
+        .await
+        .map_err(Error::other)?;
+
+    Ok(())
+}
+
+/// Load data usage info from backend storage
+pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    let buf: Vec<u8> = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await {
+        Ok(data) => data,
+        Err(e) => {
+            error!("Failed to read data usage info from backend: {}", e);
+            if e == crate::error::Error::ConfigNotFound {
+                return Ok(DataUsageInfo::default());
+            }
+            return Err(Error::other(e));
+        }
+    };
+
+    let mut data_usage_info: DataUsageInfo =
+        serde_json::from_slice(&buf).map_err(|e| Error::other(format!("Failed to deserialize data usage info: {e}")))?;
+
+    warn!("Loaded data usage info from backend {:?}", &data_usage_info);
+
+    // Handle backward compatibility like original code
+    if data_usage_info.buckets_usage.is_empty() {
+        data_usage_info.buckets_usage = data_usage_info
+            .bucket_sizes
+            .iter()
+            .map(|(bucket, &size)| {
+                (
+                    bucket.clone(),
+                    rustfs_common::data_usage::BucketUsageInfo {
+                        size,
+                        ..Default::default()
+                    },
+                )
+            })
+            .collect();
+    }
+
+    if data_usage_info.bucket_sizes.is_empty() {
+        data_usage_info.bucket_sizes = data_usage_info
+            .buckets_usage
+            .iter()
+            .map(|(bucket, bui)| (bucket.clone(), bui.size))
+            .collect();
+    }
+
+    for (bucket, bui) in &data_usage_info.buckets_usage {
+        if bui.replicated_size_v1 > 0
+            || bui.replication_failed_count_v1 > 0
+            || bui.replication_failed_size_v1 > 0
+            || bui.replication_pending_count_v1 > 0
+        {
+            if let Ok((cfg, _)) = get_replication_config(bucket).await {
+                if !cfg.role.is_empty() {
+                    data_usage_info.replication_info.insert(
+                        cfg.role.clone(),
+                        BucketTargetUsageInfo {
+                            replication_failed_size: bui.replication_failed_size_v1,
+                            replication_failed_count: bui.replication_failed_count_v1,
+                            replicated_size: bui.replicated_size_v1,
+                            replication_pending_count: bui.replication_pending_count_v1,
+                            replication_pending_size: bui.replication_pending_size_v1,
+                            ..Default::default()
+                        },
+                    );
+                }
+            }
+        }
+    }
+
+    Ok(data_usage_info)
+}
+
+/// Create a data usage cache entry from size summary
+pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry {
+    let mut entry = DataUsageEntry::default();
+    entry.add_sizes(summary);
+    entry
+}
+
+/// Convert data usage cache to DataUsageInfo
+pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[crate::store_api::BucketInfo]) -> DataUsageInfo {
+    let e = match cache.find(path) {
+        Some(e) => e,
+        None => return DataUsageInfo::default(),
+    };
+    let flat = cache.flatten(&e);
+
+    let mut buckets_usage = HashMap::new();
+    for bucket in buckets.iter() {
+        let e = match cache.find(&bucket.name) {
+            Some(e) => e,
+            None => continue,
+        };
+        let flat = cache.flatten(&e);
+        let mut bui = rustfs_common::data_usage::BucketUsageInfo {
+            size: flat.size as u64,
+            versions_count: flat.versions as u64,
+            objects_count: flat.objects as u64,
+            delete_markers_count: flat.delete_markers as u64,
+            object_size_histogram: flat.obj_sizes.to_map(),
+            object_versions_histogram: flat.obj_versions.to_map(),
+            ..Default::default()
+        };
+
+        if let Some(rs) = &flat.replication_stats {
+            bui.replica_size = rs.replica_size;
+            bui.replica_count = rs.replica_count;
+
+            for (arn, stat) in rs.targets.iter() {
+                bui.replication_info.insert(
+                    arn.clone(),
+                    BucketTargetUsageInfo {
+                        replication_pending_size: stat.pending_size,
+                        replicated_size: stat.replicated_size,
+                        replication_failed_size: stat.failed_size,
+                        replication_pending_count: stat.pending_count,
+                        replication_failed_count: stat.failed_count,
+                        replicated_count: stat.replicated_count,
+                        ..Default::default()
+                    },
+                );
+            }
+        }
+        buckets_usage.insert(bucket.name.clone(), bui);
+    }
+
+    DataUsageInfo {
+        last_update: cache.info.last_update,
+        objects_total_count: flat.objects as u64,
+        versions_total_count: flat.versions as u64,
+        delete_markers_total_count: flat.delete_markers as u64,
+        objects_total_size: flat.size as u64,
+        buckets_count: e.children.len() as u64,
+        buckets_usage,
+        ..Default::default()
+    }
+}
+
+// Helper functions for DataUsageCache operations
+pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
+    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+    use crate::store_api::{ObjectIO, ObjectOptions};
+    use http::HeaderMap;
+    use rand::Rng;
+    use std::path::Path;
+    use std::time::Duration;
+    use tokio::time::sleep;
+
+    let mut d = DataUsageCache::default();
+    let mut retries = 0;
+    while retries < 5 {
+        let path = Path::new(BUCKET_META_PREFIX).join(name);
+        match store
+            .get_object_reader(
+                RUSTFS_META_BUCKET,
+                path.to_str().unwrap(),
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+        {
+            Ok(mut reader) => {
+                if let Ok(info) = DataUsageCache::unmarshal(&reader.read_all().await?) {
+                    d = info
+                }
+                break;
+            }
+            Err(err) => match err {
+                crate::error::Error::FileNotFound | crate::error::Error::VolumeNotFound => {
+                    match store
+                        .get_object_reader(
+                            RUSTFS_META_BUCKET,
+                            name,
+                            None,
+                            HeaderMap::new(),
+                            &ObjectOptions {
+                                no_lock: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                    {
+                        Ok(mut reader) => {
+                            if let Ok(info) = DataUsageCache::unmarshal(&reader.read_all().await?) {
+                                d = info
+                            }
+                            break;
+                        }
+                        Err(_) => match err {
+                            crate::error::Error::FileNotFound | crate::error::Error::VolumeNotFound => {
+                                break;
+                            }
+                            _ => {}
+                        },
+                    }
+                }
+                _ => {
+                    break;
+                }
+            },
+        }
+        retries += 1;
+        let dur = {
+            let mut rng = rand::rng();
+            rng.random_range(0..1_000)
+        };
+        sleep(Duration::from_millis(dur)).await;
+    }
+    Ok(d)
+}
+
+pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> {
+    use crate::config::com::save_config;
+    use crate::disk::BUCKET_META_PREFIX;
+    use crate::new_object_layer_fn;
+    use std::path::Path;
+
+    let Some(store) = new_object_layer_fn() else {
+        return Err(crate::error::Error::other("errServerNotInitialized"));
+    };
+    let buf = cache.marshal_msg().map_err(crate::error::Error::other)?;
+    let buf_clone = buf.clone();
+
+    let store_clone = store.clone();
+
+    let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string();
+
+    let name_clone = name.clone();
+    tokio::spawn(async move {
+        let _ = save_config(store_clone, &format!("{}{}", &name_clone, ".bkp"), buf_clone).await;
+    });
+    save_config(store, &name, buf).await?;
+    Ok(())
+}

crates/ecstore/src/disk/local.rs

@@ -21,9 +21,6 @@ use super::{
 };
 use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 
-use crate::bucket::metadata_sys::{self};
-use crate::bucket::versioning::VersioningApi;
-use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::disk::error::FileAccessDeniedWithContext;
 use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
 use crate::disk::fs::{
@@ -36,16 +33,6 @@ use crate::disk::{
 };
 use crate::disk::{FileWriter, STORAGE_FORMAT_FILE};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
-use crate::heal::data_scanner::{
-    ScannerItem, ShouldSleepFn, SizeSummary, lc_has_active_rules, rep_has_active_rules, scan_data_folder,
-};
-use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
-use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
-use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
-use crate::heal::heal_commands::{HealScanMode, HealingTracker};
-use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
-use crate::new_object_layer_fn;
-use crate::store_api::{ObjectInfo, StorageAPI};
 use rustfs_utils::path::{
     GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
     path_join, path_join_buf,
@@ -55,19 +42,18 @@ use tokio::time::interval;
 use crate::erasure_coding::bitrot_verify;
 use bytes::Bytes;
 use path_absolutize::Absolutize;
-use rustfs_common::defer;
 use rustfs_filemeta::{
     Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
     get_file_info, read_xl_meta_no_data,
 };
 use rustfs_utils::HashAlgorithm;
 use rustfs_utils::os::get_info;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::fmt::Debug;
 use std::io::SeekFrom;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicU32, Ordering};
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
@@ -76,7 +62,6 @@ use time::OffsetDateTime;
 use tokio::fs::{self, File};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
 use tokio::sync::RwLock;
-use tokio::sync::mpsc::Sender;
 use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 
@@ -563,7 +548,7 @@ impl LocalDisk {
     }
 
     async fn read_metadata(&self, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO: suport timeout
+        // TODO: support timeout
         let (data, _) = self.read_metadata_with_dmtime(file_path.as_ref()).await?;
         Ok(data)
     }
@@ -595,7 +580,7 @@ impl LocalDisk {
     }
 
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO: timeout suport
+        // TODO: timeout support
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data)
     }
@@ -750,7 +735,7 @@ impl LocalDisk {
 
         let mut f = {
             if sync {
-                // TODO: suport sync
+                // TODO: support sync
                 self.open_file(file_path, flags, skip_parent).await?
             } else {
                 self.open_file(file_path, flags, skip_parent).await?
@@ -1705,6 +1690,15 @@ impl DiskAPI for LocalDisk {
                 };
                 out.write_obj(&meta).await?;
                 objs_returned += 1;
+            } else {
+                let fpath =
+                    self.get_object_path(&opts.bucket, path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str())?;
+
+                if let Ok(meta) = tokio::fs::metadata(fpath).await
+                    && meta.is_file()
+                {
+                    return Err(DiskError::FileNotFound);
+                }
             }
         }
 
@@ -2268,184 +2262,6 @@ impl DiskAPI for LocalDisk {
 
         Ok(info)
     }
-
-    #[tracing::instrument(level = "info", skip_all)]
-    async fn ns_scanner(
-        &self,
-        cache: &DataUsageCache,
-        updates: Sender<DataUsageEntry>,
-        scan_mode: HealScanMode,
-        we_sleep: ShouldSleepFn,
-    ) -> Result<DataUsageCache> {
-        self.scanning.fetch_add(1, Ordering::SeqCst);
-        defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) });
-
-        // must before metadata_sys
-        let Some(store) = new_object_layer_fn() else {
-            return Err(Error::other("errServerNotInitialized"));
-        };
-
-        let mut cache = cache.clone();
-        // Check if the current bucket has a configured lifecycle policy
-        if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await {
-            if lc_has_active_rules(&lc, "") {
-                cache.info.lifecycle = Some(lc);
-            }
-        }
-
-        // Check if the current bucket has replication configuration
-        if let Ok((rcfg, _)) = metadata_sys::get_replication_config(&cache.info.name).await {
-            if rep_has_active_rules(&rcfg, "", true) {
-                // TODO: globalBucketTargetSys
-            }
-        }
-
-        let vcfg = BucketVersioningSys::get(&cache.info.name).await.ok();
-
-        let loc = self.get_disk_location();
-        // TODO: 这里需要处理错误
-        let disks = store
-            .get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap())
-            .await
-            .map_err(|e| Error::other(e.to_string()))?;
-        let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?);
-        let disk_clone = disk.clone();
-        cache.info.updates = Some(updates.clone());
-        let mut data_usage_info = scan_data_folder(
-            &disks,
-            disk,
-            &cache,
-            Box::new(move |item: &ScannerItem| {
-                let mut item = item.clone();
-                let disk = disk_clone.clone();
-                let vcfg = vcfg.clone();
-                Box::pin(async move {
-                    if !item.path.ends_with(&format!("{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}")) {
-                        return Err(Error::other(ERR_SKIP_FILE).into());
-                    }
-                    let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
-                    let mut res = HashMap::new();
-                    let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata);
-                    let buf = match disk.read_metadata(item.path.clone()).await {
-                        Ok(buf) => buf,
-                        Err(err) => {
-                            res.insert("err".to_string(), err.to_string());
-                            stop_fn(&res);
-                            return Err(Error::other(ERR_SKIP_FILE).into());
-                        }
-                    };
-                    done_sz(buf.len() as u64);
-                    res.insert("metasize".to_string(), buf.len().to_string());
-                    item.transform_meda_dir();
-                    let meta_cache = MetaCacheEntry {
-                        name: item.object_path().to_string_lossy().to_string(),
-                        metadata: buf,
-                        ..Default::default()
-                    };
-                    let fivs = match meta_cache.file_info_versions(&item.bucket) {
-                        Ok(fivs) => fivs,
-                        Err(err) => {
-                            res.insert("err".to_string(), err.to_string());
-                            stop_fn(&res);
-                            return Err(Error::other(ERR_SKIP_FILE).into());
-                        }
-                    };
-                    let mut size_s = SizeSummary::default();
-                    let done = ScannerMetrics::time(ScannerMetric::ApplyAll);
-                    let obj_infos = match item.apply_versions_actions(&fivs.versions).await {
-                        Ok(obj_infos) => obj_infos,
-                        Err(err) => {
-                            res.insert("err".to_string(), err.to_string());
-                            stop_fn(&res);
-                            return Err(Error::other(ERR_SKIP_FILE).into());
-                        }
-                    };
-
-                    let versioned = if let Some(vcfg) = vcfg.as_ref() {
-                        vcfg.versioned(item.object_path().to_str().unwrap_or_default())
-                    } else {
-                        false
-                    };
-
-                    let mut obj_deleted = false;
-                    for info in obj_infos.iter() {
-                        let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);
-                        let sz: i64;
-                        (obj_deleted, sz) = item.apply_actions(info, &mut size_s).await;
-                        done();
-
-                        if obj_deleted {
-                            break;
-                        }
-
-                        let actual_sz = match info.get_actual_size() {
-                            Ok(size) => size,
-                            Err(_) => continue,
-                        };
-
-                        if info.delete_marker {
-                            size_s.delete_markers += 1;
-                        }
-
-                        if info.version_id.is_some() && sz == actual_sz {
-                            size_s.versions += 1;
-                        }
-
-                        size_s.total_size += sz as usize;
-
-                        if info.delete_marker {
-                            continue;
-                        }
-                    }
-
-                    for free_version in fivs.free_versions.iter() {
-                        let _obj_info = ObjectInfo::from_file_info(
-                            free_version,
-                            &item.bucket,
-                            &item.object_path().to_string_lossy(),
-                            versioned,
-                        );
-                        let done = ScannerMetrics::time(ScannerMetric::TierObjSweep);
-                        done();
-                    }
-
-                    // todo: global trace
-                    if obj_deleted {
-                        return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into());
-                    }
-                    done();
-                    Ok(size_s)
-                })
-            }),
-            scan_mode,
-            we_sleep,
-        )
-        .await?;
-        data_usage_info.info.last_update = Some(SystemTime::now());
-        debug!("ns_scanner completed: {data_usage_info:?}");
-        Ok(data_usage_info)
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn healing(&self) -> Option<HealingTracker> {
-        let healing_file = path_join(&[
-            self.path(),
-            PathBuf::from(RUSTFS_META_BUCKET),
-            PathBuf::from(BUCKET_META_PREFIX),
-            PathBuf::from(HEALING_TRACKER_FILENAME),
-        ]);
-        let b = match fs::read(healing_file).await {
-            Ok(b) => b,
-            Err(_) => return None,
-        };
-        if b.is_empty() {
-            return None;
-        }
-        match HealingTracker::unmarshal_msg(&b) {
-            Ok(h) => Some(h),
-            Err(_) => Some(HealingTracker::default()),
-        }
-    }
 }
 
 async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInfo, bool)> {

crates/ecstore/src/disk/mod.rs

@@ -30,11 +30,6 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
-use crate::heal::{
-    data_scanner::ShouldSleepFn,
-    data_usage_cache::{DataUsageCache, DataUsageEntry},
-    heal_commands::{HealScanMode, HealingTracker},
-};
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
 use endpoint::Endpoint;
@@ -46,10 +41,7 @@ use rustfs_madmin::info_commands::DiskMetrics;
 use serde::{Deserialize, Serialize};
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
 use time::OffsetDateTime;
-use tokio::{
-    io::{AsyncRead, AsyncWrite},
-    sync::mpsc::Sender,
-};
+use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
 
 pub type DiskStore = Arc<Disk>;
@@ -406,28 +398,6 @@ impl DiskAPI for Disk {
             Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await,
         }
     }
-
-    #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))]
-    async fn ns_scanner(
-        &self,
-        cache: &DataUsageCache,
-        updates: Sender<DataUsageEntry>,
-        scan_mode: HealScanMode,
-        we_sleep: ShouldSleepFn,
-    ) -> Result<DataUsageCache> {
-        match self {
-            Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
-            Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode, we_sleep).await,
-        }
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn healing(&self) -> Option<HealingTracker> {
-        match self {
-            Disk::Local(local_disk) => local_disk.healing().await,
-            Disk::Remote(remote_disk) => remote_disk.healing().await,
-        }
-    }
 }
 
 pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
@@ -527,14 +497,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()>;
     async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
-    async fn ns_scanner(
-        &self,
-        cache: &DataUsageCache,
-        updates: Sender<DataUsageEntry>,
-        scan_mode: HealScanMode,
-        we_sleep: ShouldSleepFn,
-    ) -> Result<DataUsageCache>;
-    async fn healing(&self) -> Option<HealingTracker>;
 }
 
 #[derive(Debug, Default, Serialize, Deserialize)]

crates/ecstore/src/disk/os.rs

@@ -49,7 +49,8 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
     let mut count = 0usize;
     for c in path_name.chars() {
         match c {
-            '/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset
+            '/' => count = 0,
+            '\\' if cfg!(target_os = "windows") => count = 0, // Reset
             _ => {
                 count += 1;
                 if count > 255 {

crates/ecstore/src/erasure.rs

@@ -308,7 +308,7 @@ impl Erasure {
             // ec encode, 结果会写进 data_buffer
             let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(shard_size).collect();
 
-            // partiy 数量大于 0 才 ec
+            // parity 数量大于 0 才 ec
             if self.parity_shards > 0 {
                 self.encoder.as_ref().unwrap().encode(data_slices).map_err(Error::other)?;
             }

crates/ecstore/src/error.rs

@@ -183,6 +183,9 @@ pub enum StorageError {
 
     #[error("Io error: {0}")]
     Io(std::io::Error),
+
+    #[error("Lock error: {0}")]
+    Lock(#[from] rustfs_lock::LockError),
 }
 
 impl StorageError {
@@ -409,6 +412,7 @@ impl Clone for StorageError {
             StorageError::FirstDiskWait => StorageError::FirstDiskWait,
             StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
             StorageError::NoHealRequired => StorageError::NoHealRequired,
+            StorageError::Lock(e) => StorageError::Lock(e.clone()),
         }
     }
 }
@@ -471,6 +475,7 @@ impl StorageError {
             StorageError::ConfigNotFound => 0x35,
             StorageError::TooManyOpenFiles => 0x36,
             StorageError::NoHealRequired => 0x37,
+            StorageError::Lock(_) => 0x38,
         }
     }
 
@@ -535,6 +540,7 @@ impl StorageError {
             0x35 => Some(StorageError::ConfigNotFound),
             0x36 => Some(StorageError::TooManyOpenFiles),
             0x37 => Some(StorageError::NoHealRequired),
+            0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
             _ => None,
         }
     }

crates/ecstore/src/global.rs

@@ -12,13 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::heal::mrf::MRFState;
 use crate::{
     bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
     disk::DiskStore,
     endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
     event_notification::EventNotifier,
-    heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
     store::ECStore,
     tier::tier::TierConfigMgr,
 };
@@ -38,8 +36,6 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;
 
-pub const DEFAULT_PORT: u16 = 9000;
-
 lazy_static! {
 static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
 pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
@@ -51,14 +47,10 @@ pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStor
 pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
 pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
 pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
-pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
-pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
 pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
 pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
 pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
 //pub static ref GLOBAL_RemoteTargetTransport
-pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
-pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
 static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
 pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
 pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();

crates/ecstore/src/heal/background_heal_ops.rs

@@ -1,512 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use futures::future::join_all;
-use rustfs_madmin::heal_commands::HealResultItem;
-use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
-use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
-use tokio::{
-    spawn,
-    sync::{
-        RwLock,
-        mpsc::{self, Receiver, Sender},
-    },
-    time::interval,
-};
-use tokio_util::sync::CancellationToken;
-use tracing::{error, info};
-use uuid::Uuid;
-
-use super::{
-    heal_commands::HealOpts,
-    heal_ops::{HealSequence, new_bg_heal_sequence},
-};
-use crate::error::{Error, Result};
-use crate::global::{GLOBAL_MRFState, get_background_services_cancel_token};
-use crate::heal::error::ERR_RETRY_HEALING;
-use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode};
-use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource};
-use crate::{
-    config::RUSTFS_CONFIG_PREFIX,
-    disk::{BUCKET_META_PREFIX, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError},
-    global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
-    heal::{
-        data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
-        data_usage_cache::DataUsageCache,
-        heal_commands::{init_healing_tracker, load_healing_tracker},
-        heal_ops::NOP_HEAL,
-    },
-    new_object_layer_fn,
-    store::get_disk_via_endpoint,
-    store_api::{BucketInfo, BucketOptions, StorageAPI},
-};
-
-pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
-
-pub async fn init_auto_heal() {
-    info!("Initializing auto heal background task");
-
-    let Some(cancel_token) = get_background_services_cancel_token() else {
-        error!("Background services cancel token not initialized");
-        return;
-    };
-
-    init_background_healing().await;
-    let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string());
-    if v == "on" {
-        info!("start monitor local disks and heal");
-        GLOBAL_BackgroundHealState
-            .push_heal_local_disks(&get_local_disks_to_heal().await)
-            .await;
-
-        let cancel_clone = cancel_token.clone();
-        spawn(async move {
-            monitor_local_disks_and_heal(cancel_clone).await;
-        });
-    }
-
-    let cancel_clone = cancel_token.clone();
-    spawn(async move {
-        GLOBAL_MRFState.heal_routine_with_cancel(cancel_clone).await;
-    });
-}
-
-async fn init_background_healing() {
-    let bg_seq = Arc::new(new_bg_heal_sequence());
-    for _ in 0..GLOBAL_BackgroundHealRoutine.workers {
-        let bg_seq_clone = bg_seq.clone();
-        spawn(async {
-            GLOBAL_BackgroundHealRoutine.add_worker(bg_seq_clone).await;
-        });
-    }
-    let _ = GLOBAL_BackgroundHealState.launch_new_heal_sequence(bg_seq).await;
-}
-
-pub async fn get_local_disks_to_heal() -> Vec<Endpoint> {
-    let mut disks_to_heal = Vec::new();
-    for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
-        if let Some(disk) = disk {
-            if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
-                if err == DiskError::UnformattedDisk {
-                    info!("get_local_disks_to_heal, disk is unformatted: {}", err);
-                    disks_to_heal.push(disk.endpoint());
-                }
-            }
-            let h = disk.healing().await;
-            if let Some(h) = h {
-                if !h.finished {
-                    info!("get_local_disks_to_heal, disk healing not finished");
-                    disks_to_heal.push(disk.endpoint());
-                }
-            }
-        }
-    }
-
-    // todo
-    // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n {
-
-    // }
-    disks_to_heal
-}
-
-async fn monitor_local_disks_and_heal(cancel_token: CancellationToken) {
-    info!("Auto heal monitor started");
-    let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL);
-
-    loop {
-        tokio::select! {
-            _ = cancel_token.cancelled() => {
-                info!("Auto heal monitor received shutdown signal, exiting gracefully");
-                break;
-            }
-            _ = interval.tick() => {
-                let heal_disks = GLOBAL_BackgroundHealState.get_heal_local_disk_endpoints().await;
-                if heal_disks.is_empty() {
-                    info!("heal local disks is empty");
-                    interval.reset();
-                    continue;
-                }
-
-                info!("heal local disks: {:?}", heal_disks);
-
-                let store = new_object_layer_fn().expect("errServerNotInitialized");
-                if let (_result, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
-                    error!("heal local disk format error: {}", err);
-                    if err == Error::NoHealRequired {
-                    } else {
-                        info!("heal format err: {}", err.to_string());
-                        interval.reset();
-                        continue;
-                    }
-                }
-
-                let mut futures = Vec::new();
-                for disk in heal_disks.into_ref().iter() {
-                    let disk_clone = disk.clone();
-                    let cancel_clone = cancel_token.clone();
-                    futures.push(async move {
-                        let disk_for_cancel = disk_clone.clone();
-                        tokio::select! {
-                            _ = cancel_clone.cancelled() => {
-                                info!("Disk healing task cancelled for disk: {}", disk_for_cancel);
-                            }
-                            _ = async {
-                                GLOBAL_BackgroundHealState
-                                    .set_disk_healing_status(disk_clone.clone(), true)
-                                    .await;
-                                if heal_fresh_disk(&disk_clone).await.is_err() {
-                                    info!("heal_fresh_disk is err");
-                                    GLOBAL_BackgroundHealState
-                                        .set_disk_healing_status(disk_clone.clone(), false)
-                                        .await;
-                                }
-                                GLOBAL_BackgroundHealState.pop_heal_local_disks(&[disk_clone]).await;
-                            } => {}
-                        }
-                    });
-                }
-                let _ = join_all(futures).await;
-                interval.reset();
-            }
-        }
-    }
-}
-
-async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
-    let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.set_idx as usize);
-    let disk = match get_disk_via_endpoint(endpoint).await {
-        Some(disk) => disk,
-        None => {
-            return Err(Error::other(format!(
-                "Unexpected error disk must be initialized by now after formatting: {endpoint}"
-            )));
-        }
-    };
-
-    if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
-        match err {
-            DiskError::DriveIsRoot => {
-                return Ok(());
-            }
-            DiskError::UnformattedDisk => {}
-            _ => {
-                return Err(err.into());
-            }
-        }
-    }
-
-    let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
-        Ok(tracker) => tracker,
-        Err(err) => {
-            match err {
-                DiskError::FileNotFound => {
-                    return Ok(());
-                }
-                _ => {
-                    info!(
-                        "Unable to load healing tracker on '{}': {}, re-initializing..",
-                        disk.to_string(),
-                        err.to_string()
-                    );
-                }
-            }
-            init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await?
-        }
-    };
-
-    info!(
-        "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.",
-        endpoint.to_string()
-    );
-
-    let Some(store) = new_object_layer_fn() else {
-        return Err(Error::other("errServerNotInitialized"));
-    };
-
-    let mut buckets = store.list_bucket(&BucketOptions::default()).await?;
-    buckets.push(BucketInfo {
-        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)])
-            .to_string_lossy()
-            .to_string(),
-        ..Default::default()
-    });
-    buckets.push(BucketInfo {
-        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)])
-            .to_string_lossy()
-            .to_string(),
-        ..Default::default()
-    });
-
-    buckets.sort_by(|a, b| {
-        let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET);
-        let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET);
-
-        match (a_has_prefix, b_has_prefix) {
-            (true, false) => Ordering::Less,
-            (false, true) => Ordering::Greater,
-            _ => b.created.cmp(&a.created),
-        }
-    });
-
-    if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await {
-        let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new());
-        tracker.objects_total_count = data_usage_info.objects_total_count;
-        tracker.objects_total_size = data_usage_info.objects_total_size;
-    };
-
-    tracker.set_queue_buckets(&buckets).await;
-    tracker.save().await?;
-
-    let tracker = Arc::new(RwLock::new(tracker));
-    let qb = tracker.read().await.queue_buckets.clone();
-    store.pools[pool_idx].disk_set[set_idx]
-        .clone()
-        .heal_erasure_set(&qb, tracker.clone())
-        .await?;
-    let mut tracker_w = tracker.write().await;
-    if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 {
-        tracker_w.retry_attempts += 1;
-        tracker_w.reset_healing().await;
-        if let Err(err) = tracker_w.update().await {
-            info!("update tracker failed: {}", err.to_string());
-        }
-        return Err(Error::other(ERR_RETRY_HEALING));
-    }
-
-    if tracker_w.items_failed > 0 {
-        info!(
-            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).",
-            disk.to_string(),
-            tracker_w.retry_attempts,
-            tracker_w.items_healed,
-            tracker_w.item_skipped,
-            tracker_w.items_failed
-        );
-    } else if tracker_w.retry_attempts > 0 {
-        info!(
-            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).",
-            disk.to_string(),
-            tracker_w.retry_attempts,
-            tracker_w.items_healed,
-            tracker_w.item_skipped
-        );
-    } else {
-        info!(
-            "Healing of drive '{}' is finished (healed: {}, skipped: {}).",
-            disk.to_string(),
-            tracker_w.items_healed,
-            tracker_w.item_skipped
-        );
-    }
-
-    if tracker_w.heal_id.is_empty() {
-        if let Err(err) = tracker_w.delete().await {
-            error!("delete tracker failed: {}", err.to_string());
-        }
-    }
-    let Some(store) = new_object_layer_fn() else {
-        return Err(Error::other("errServerNotInitialized"));
-    };
-    let disks = store.get_disks(pool_idx, set_idx).await?;
-    for disk in disks.into_iter() {
-        if disk.is_none() {
-            continue;
-        }
-        let mut tracker = match load_healing_tracker(&disk).await {
-            Ok(tracker) => tracker,
-            Err(err) => {
-                match err {
-                    DiskError::FileNotFound => {}
-                    _ => {
-                        info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string());
-                    }
-                }
-                continue;
-            }
-        };
-        if tracker.heal_id == tracker_w.heal_id {
-            tracker.finished = true;
-            tracker.update().await?;
-        }
-    }
-    Ok(())
-}
-
-#[derive(Debug)]
-pub struct HealTask {
-    pub bucket: String,
-    pub object: String,
-    pub version_id: String,
-    pub opts: HealOpts,
-    pub resp_tx: Option<Sender<HealResult>>,
-    pub resp_rx: Option<Receiver<HealResult>>,
-}
-
-impl HealTask {
-    pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self {
-        Self {
-            bucket: bucket.to_string(),
-            object: object.to_string(),
-            version_id: version_id.to_string(),
-            opts: *opts,
-            resp_tx: None,
-            resp_rx: None,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct HealResult {
-    pub result: HealResultItem,
-    pub err: Option<Error>,
-}
-
-pub struct HealRoutine {
-    pub tasks_tx: Sender<HealTask>,
-    tasks_rx: RwLock<Receiver<HealTask>>,
-    workers: usize,
-}
-
-impl HealRoutine {
-    pub fn new() -> Arc<Self> {
-        let mut workers = num_cpus::get() / 2;
-        if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") {
-            if let Ok(num_healers) = env_heal_workers.parse::<usize>() {
-                workers = num_healers;
-            }
-        }
-
-        if workers == 0 {
-            workers = 4;
-        }
-
-        let (tx, rx) = mpsc::channel(100);
-        Arc::new(Self {
-            tasks_tx: tx,
-            tasks_rx: RwLock::new(rx),
-            workers,
-        })
-    }
-
-    pub async fn add_worker(&self, bgseq: Arc<HealSequence>) {
-        loop {
-            let mut d_res = HealResultItem::default();
-            let d_err: Option<Error>;
-            match self.tasks_rx.write().await.recv().await {
-                Some(task) => {
-                    info!("got task: {:?}", task);
-                    if task.bucket == NOP_HEAL {
-                        d_err = Some(Error::other("skip file"));
-                    } else if task.bucket == SLASH_SEPARATOR {
-                        match heal_disk_format(task.opts).await {
-                            Ok((res, err)) => {
-                                d_res = res;
-                                d_err = err;
-                            }
-                            Err(err) => d_err = Some(err),
-                        }
-                    } else {
-                        let store = new_object_layer_fn().expect("errServerNotInitialized");
-                        if task.object.is_empty() {
-                            match store.heal_bucket(&task.bucket, &task.opts).await {
-                                Ok(res) => {
-                                    d_res = res;
-                                    d_err = None;
-                                }
-                                Err(err) => d_err = Some(err),
-                            }
-                        } else {
-                            match store
-                                .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
-                                .await
-                            {
-                                Ok((res, err)) => {
-                                    d_res = res;
-                                    d_err = err;
-                                }
-                                Err(err) => d_err = Some(err),
-                            }
-                        }
-                    }
-                    info!("task finished, task: {:?}", task);
-                    if let Some(resp_tx) = task.resp_tx {
-                        let _ = resp_tx
-                            .send(HealResult {
-                                result: d_res,
-                                err: d_err,
-                            })
-                            .await;
-                    } else {
-                        // when respCh is not set caller is not waiting but we
-                        // update the relevant metrics for them
-                        if d_err.is_none() {
-                            bgseq.count_healed(d_res.heal_item_type).await;
-                        } else {
-                            bgseq.count_failed(d_res.heal_item_type).await;
-                        }
-                    }
-                }
-                None => {
-                    info!("add_worker, tasks_rx was closed, return");
-                    return;
-                }
-            }
-        }
-    }
-}
-
-// pub fn active_listeners() -> Result<usize> {
-
-// }
-
-async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Error>)> {
-    let Some(store) = new_object_layer_fn() else {
-        return Err(Error::other("errServerNotInitialized"));
-    };
-
-    let (res, err) = store.heal_format(opts.dry_run).await?;
-    // return any error, ignore error returned when disks have
-    // already healed.
-    if err.is_some() {
-        return Ok((HealResultItem::default(), err));
-    }
-    Ok((res, err))
-}
-
-pub(crate) async fn heal_bucket(bucket: &str) -> Result<()> {
-    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
-    if ok {
-        // bg_seq must be Some when ok is true
-        return bg_seq
-            .unwrap()
-            .queue_heal_task(
-                HealSource {
-                    bucket: bucket.to_string(),
-                    ..Default::default()
-                },
-                HEAL_ITEM_BUCKET.to_string(),
-            )
-            .await;
-    }
-    Ok(())
-}
-
-pub(crate) async fn heal_object(bucket: &str, object: &str, version_id: &str, scan_mode: HealScanMode) -> Result<()> {
-    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
-    if ok {
-        // bg_seq must be Some when ok is true
-        return HealSequence::heal_object(bg_seq.unwrap(), bucket, object, version_id, scan_mode).await;
-    }
-    Ok(())
-}