mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-17 09:40:32 +00:00
Compare commits
6 Commits
1.0.0-alph
...
1.0.0-alph
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
387f4faf78 | ||
|
|
0f7093c5f9 | ||
|
|
6a5c0055e7 | ||
|
|
76288f2501 | ||
|
|
3497ccfada | ||
|
|
24e3d3a2ce |
4
.github/pull_request_template.md
vendored
4
.github/pull_request_template.md
vendored
@@ -19,9 +19,7 @@ Pull Request Template for RustFS
|
||||
|
||||
## Checklist
|
||||
- [ ] I have read and followed the [CONTRIBUTING.md](CONTRIBUTING.md) guidelines
|
||||
- [ ] Code is formatted with `cargo fmt --all`
|
||||
- [ ] Passed `cargo clippy --all-targets --all-features -- -D warnings`
|
||||
- [ ] Passed `cargo check --all-targets`
|
||||
- [ ] Passed `make pre-commit`
|
||||
- [ ] Added/updated necessary tests
|
||||
- [ ] Documentation updated (if needed)
|
||||
- [ ] CI/CD passed (if applicable)
|
||||
|
||||
8
.github/workflows/audit.yml
vendored
8
.github/workflows/audit.yml
vendored
@@ -16,13 +16,13 @@ name: Security Audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
- '.github/workflows/audit.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@v2
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Dependency Review
|
||||
uses: actions/dependency-review-action@v4
|
||||
|
||||
30
.github/workflows/build.yml
vendored
30
.github/workflows/build.yml
vendored
@@ -28,8 +28,8 @@ name: Build and Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["*.*.*"]
|
||||
branches: [main]
|
||||
tags: [ "*.*.*" ]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -45,7 +45,7 @@ on:
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "**.txt"
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
# Build RustFS binaries
|
||||
build-rustfs:
|
||||
name: Build RustFS
|
||||
needs: [build-check]
|
||||
needs: [ build-check ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 60
|
||||
@@ -200,7 +200,7 @@ jobs:
|
||||
# platform: windows
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -527,7 +527,7 @@ jobs:
|
||||
# Build summary
|
||||
build-summary:
|
||||
name: Build Summary
|
||||
needs: [build-check, build-rustfs]
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: always() && needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -579,7 +579,7 @@ jobs:
|
||||
# Create GitHub Release (only for tag pushes)
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
needs: [build-check, build-rustfs]
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -589,7 +589,7 @@ jobs:
|
||||
release_url: ${{ steps.create.outputs.release_url }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -665,7 +665,7 @@ jobs:
|
||||
# Prepare and upload release assets
|
||||
upload-release-assets:
|
||||
name: Upload Release Assets
|
||||
needs: [build-check, build-rustfs, create-release]
|
||||
needs: [ build-check, build-rustfs, create-release ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -673,10 +673,10 @@ jobs:
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
@@ -746,7 +746,7 @@ jobs:
|
||||
# Update latest.json for stable releases only
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [build-check, upload-release-assets]
|
||||
needs: [ build-check, upload-release-assets ]
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -796,14 +796,14 @@ jobs:
|
||||
# Publish release (remove draft status)
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [build-check, create-release, upload-release-assets]
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Update release notes and publish
|
||||
env:
|
||||
|
||||
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@@ -88,7 +88,7 @@ jobs:
|
||||
name: Typos
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@master
|
||||
@@ -101,7 +101,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
10
.github/workflows/docker.yml
vendored
10
.github/workflows/docker.yml
vendored
@@ -36,8 +36,8 @@ permissions:
|
||||
on:
|
||||
# Automatically triggered when build workflow completes
|
||||
workflow_run:
|
||||
workflows: ["Build and Release"]
|
||||
types: [completed]
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
# Manual trigger with same parameters for consistency
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# For workflow_run events, checkout the specific commit that triggered the workflow
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
@@ -382,7 +382,7 @@ jobs:
|
||||
# Docker build summary
|
||||
docker-summary:
|
||||
name: Docker Build Summary
|
||||
needs: [build-check, build-docker]
|
||||
needs: [ build-check, build-docker ]
|
||||
if: always() && needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
6
.github/workflows/performance.yml
vendored
6
.github/workflows/performance.yml
vendored
@@ -16,7 +16,7 @@ name: Performance Testing
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "**/*.rs"
|
||||
- "**/Cargo.toml"
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
147
Cargo.lock
generated
147
Cargo.lock
generated
@@ -169,9 +169,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.98"
|
||||
version = "1.0.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
|
||||
checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
@@ -1511,7 +1511,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde-untagged",
|
||||
"serde-value",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"toml",
|
||||
"unicode-xid",
|
||||
"url",
|
||||
@@ -1529,7 +1529,7 @@ dependencies = [
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1540,9 +1540,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.31"
|
||||
version = "1.2.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
|
||||
checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
@@ -1697,9 +1697,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.43"
|
||||
version = "4.5.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50fd97c9dc2399518aa331917ac6f274280ec5eb34e555dd291899745c48ec6f"
|
||||
checksum = "1c1f056bae57e3e54c3375c41ff79619ddd13460a17d7438712bd0d83fda4ff8"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1707,9 +1707,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.43"
|
||||
version = "4.5.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c35b5830294e1fa0462034af85cc95225a4cb07092c088c55bda3147cfcd8f65"
|
||||
checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
@@ -2062,15 +2062,16 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc-fast"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f"
|
||||
checksum = "ec9f79df9b0383475ae6df8fcf35d4e29528441706385339daf0fe3f4cce040b"
|
||||
dependencies = [
|
||||
"crc",
|
||||
"digest 0.10.7",
|
||||
"libc",
|
||||
"rand 0.9.2",
|
||||
"regex",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3589,7 +3590,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"tokio",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"url",
|
||||
]
|
||||
|
||||
@@ -3861,7 +3862,7 @@ dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"toml",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -4316,9 +4317,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.2"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "global-hotkey"
|
||||
@@ -5400,9 +5401,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.174"
|
||||
version = "0.2.175"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
|
||||
checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
|
||||
|
||||
[[package]]
|
||||
name = "libdbus-sys"
|
||||
@@ -5464,7 +5465,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"serde",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -5525,7 +5526,7 @@ checksum = "656b3b27f8893f7bbf9485148ff9a65f019e3f33bd5cdc87c83cab16b3fd9ec8"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"neli",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
@@ -6468,7 +6469,7 @@ dependencies = [
|
||||
"futures-sink",
|
||||
"js-sys",
|
||||
"pin-project-lite",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -6497,7 +6498,7 @@ dependencies = [
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry_sdk",
|
||||
"prost 0.13.5",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tonic 0.13.1",
|
||||
"tracing",
|
||||
@@ -6545,7 +6546,7 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
"rand 0.9.2",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
@@ -7183,9 +7184,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.95"
|
||||
version = "1.0.97"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
||||
checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -7343,7 +7344,7 @@ dependencies = [
|
||||
"rustc-hash 2.1.1",
|
||||
"rustls 0.23.31",
|
||||
"socket2 0.5.10",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
@@ -7364,7 +7365,7 @@ dependencies = [
|
||||
"rustls 0.23.31",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
@@ -7623,7 +7624,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
|
||||
dependencies = [
|
||||
"getrandom 0.2.16",
|
||||
"libredox",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7839,7 +7840,7 @@ dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -8085,7 +8086,7 @@ dependencies = [
|
||||
"shadow-rs",
|
||||
"socket2 0.6.0",
|
||||
"sysctl",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tikv-jemallocator",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -8093,7 +8094,7 @@ dependencies = [
|
||||
"tokio-stream",
|
||||
"tokio-tar",
|
||||
"tokio-util",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
@@ -8122,7 +8123,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -8170,7 +8171,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -8195,7 +8196,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -8260,12 +8261,12 @@ dependencies = [
|
||||
"smallvec",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
@@ -8288,7 +8289,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -8333,7 +8334,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -8350,9 +8351,9 @@ dependencies = [
|
||||
"rustfs-protos",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
@@ -8407,7 +8408,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snap",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -8440,7 +8441,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"sysinfo",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
@@ -8463,7 +8464,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"strum",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
@@ -8475,7 +8476,7 @@ dependencies = [
|
||||
"flatbuffers 25.2.10",
|
||||
"prost 0.14.1",
|
||||
"rustfs-common",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
"tonic-prost",
|
||||
"tonic-prost-build",
|
||||
]
|
||||
@@ -8812,9 +8813,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.21"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
@@ -8863,7 +8864,7 @@ dependencies = [
|
||||
"smallvec",
|
||||
"std-next",
|
||||
"sync_wrapper",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"tokio",
|
||||
"tower",
|
||||
@@ -9439,7 +9440,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -9457,9 +9458,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.10"
|
||||
version = "0.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
|
||||
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
|
||||
|
||||
[[package]]
|
||||
name = "sledgehammer_bindgen"
|
||||
@@ -9666,7 +9667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc"
|
||||
dependencies = [
|
||||
"simdutf8",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9863,9 +9864,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.36.1"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d"
|
||||
checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -10049,11 +10050,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.12"
|
||||
version = "2.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
|
||||
checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.12",
|
||||
"thiserror-impl 2.0.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10069,9 +10070,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.12"
|
||||
version = "2.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
|
||||
checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -10403,9 +10404,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "308e1db96abdccdf0a9150fb69112bf6ea72640e0bd834ef0c4a618ccc8c8ddc"
|
||||
checksum = "67ac5a8627ada0968acec063a4746bf79588aa03ccb66db2f75d7dce26722a40"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10433,9 +10434,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18262cdd13dec66e8e3f2e3fe535e4b2cc706fab444a7d3678d75d8ac2557329"
|
||||
checksum = "49e323d8bba3be30833707e36d046deabf10a35ae8ad3cae576943ea8933e25d"
|
||||
dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -10445,20 +10446,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d8b5b7a44512c59f5ad45e0c40e53263cbbf4426d74fe6b569e04f1d4206e9c"
|
||||
checksum = "b9c511b9a96d40cb12b7d5d00464446acf3b9105fd3ce25437cfe41c92b1c87d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost 0.14.1",
|
||||
"tonic 0.14.0",
|
||||
"tonic 0.14.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost-build"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "114cca66d757d72422ef8cccf8be3065321860ac9fa4be73aab37a8a20a9a805"
|
||||
checksum = "8ef298fcd01b15e135440c4b8c974460ceca4e6a5af7f1c933b08e4d2875efa1"
|
||||
dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -10678,7 +10679,7 @@ dependencies = [
|
||||
"objc2-foundation 0.3.1",
|
||||
"once_cell",
|
||||
"png",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
@@ -10872,9 +10873,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.17.0"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
|
||||
checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be"
|
||||
dependencies = [
|
||||
"getrandom 0.3.3",
|
||||
"js-sys",
|
||||
@@ -10886,9 +10887,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "uuid-macro-internal"
|
||||
version = "1.17.0"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b682e8c381995ea03130e381928e0e005b7c9eb483c6c8682f50e07b33c2b7"
|
||||
checksum = "22b7ad00068276db5fea436dba78daa7891b8d60db76e4f51cbdefbdecdab97e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -12239,7 +12240,7 @@ dependencies = [
|
||||
"memchr",
|
||||
"pbkdf2",
|
||||
"sha1 0.10.6",
|
||||
"thiserror 2.0.12",
|
||||
"thiserror 2.0.14",
|
||||
"time",
|
||||
"xz2",
|
||||
"zeroize",
|
||||
|
||||
20
Cargo.toml
20
Cargo.toml
@@ -90,7 +90,7 @@ rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
|
||||
rustfs-workers = { path = "crates/workers", version = "0.0.5" }
|
||||
rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
|
||||
aes-gcm = { version = "0.10.3", features = ["std"] }
|
||||
anyhow = "1.0.98"
|
||||
anyhow = "1.0.99"
|
||||
arc-swap = "1.7.1"
|
||||
argon2 = { version = "0.5.3", features = ["std"] }
|
||||
atoi = "2.0.0"
|
||||
@@ -109,10 +109,10 @@ bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.0.1"
|
||||
byteorder = "1.5.0"
|
||||
cfg-if = "1.0.1"
|
||||
crc-fast = "1.3.0"
|
||||
crc-fast = "1.4.0"
|
||||
chacha20poly1305 = { version = "0.10.1" }
|
||||
chrono = { version = "0.4.41", features = ["serde"] }
|
||||
clap = { version = "4.5.43", features = ["derive", "env"] }
|
||||
clap = { version = "4.5.44", features = ["derive", "env"] }
|
||||
const-str = { version = "0.6.4", features = ["std", "proc"] }
|
||||
crc32fast = "1.5.0"
|
||||
criterion = { version = "0.7", features = ["html_reports"] }
|
||||
@@ -129,7 +129,7 @@ form_urlencoded = "1.2.1"
|
||||
futures = "0.3.31"
|
||||
futures-core = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
glob = "0.3.2"
|
||||
glob = "0.3.3"
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
@@ -232,12 +232,12 @@ snafu = "0.8.6"
|
||||
snap = "1.1.1"
|
||||
socket2 = "0.6.0"
|
||||
strum = { version = "0.27.2", features = ["derive"] }
|
||||
sysinfo = "0.36.1"
|
||||
sysinfo = "0.37.0"
|
||||
sysctl = "0.6.0"
|
||||
tempfile = "3.20.0"
|
||||
temp-env = "0.3.6"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.12"
|
||||
thiserror = "2.0.14"
|
||||
time = { version = "0.3.41", features = [
|
||||
"std",
|
||||
"parsing",
|
||||
@@ -251,9 +251,9 @@ tokio-stream = { version = "0.1.17" }
|
||||
tokio-tar = "0.3.1"
|
||||
tokio-test = "0.4.4"
|
||||
tokio-util = { version = "0.7.16", features = ["io", "compat"] }
|
||||
tonic = { version = "0.14.0", features = ["gzip"] }
|
||||
tonic-prost = { version = "0.14.0" }
|
||||
tonic-prost-build = { version = "0.14.0" }
|
||||
tonic = { version = "0.14.1", features = ["gzip"] }
|
||||
tonic-prost = { version = "0.14.1" }
|
||||
tonic-prost-build = { version = "0.14.1" }
|
||||
tower = { version = "0.5.2", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.6", features = ["cors"] }
|
||||
tracing = "0.1.41"
|
||||
@@ -265,7 +265,7 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.4"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.17.0", features = [
|
||||
uuid = { version = "1.18.0", features = [
|
||||
"v4",
|
||||
"fast-rng",
|
||||
"macro-diagnostics",
|
||||
|
||||
@@ -148,27 +148,47 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>> {
|
||||
debug!("Getting object data: {}/{}", bucket, object);
|
||||
|
||||
match (*self.ecstore)
|
||||
let reader = match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => match reader.read_all().await {
|
||||
Ok(data) => Ok(Some(data)),
|
||||
Ok(reader) => reader,
|
||||
Err(e) => {
|
||||
error!("Failed to get object: {}/{} - {}", bucket, object, e);
|
||||
return Err(Error::other(e));
|
||||
}
|
||||
};
|
||||
|
||||
// WARNING: Returning Vec<u8> for large objects is dangerous. To avoid OOM, cap the read size.
|
||||
// If needed, refactor callers to stream instead of buffering entire object.
|
||||
const MAX_READ_BYTES: usize = 16 * 1024 * 1024; // 16 MiB cap
|
||||
let mut buf = Vec::with_capacity(1024 * 1024);
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
let mut n_read: usize = 0;
|
||||
let mut stream = reader.stream;
|
||||
loop {
|
||||
// Read in chunks
|
||||
let mut chunk = vec![0u8; 1024 * 1024];
|
||||
match stream.read(&mut chunk).await {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
n_read += n;
|
||||
if n_read > MAX_READ_BYTES {
|
||||
warn!(
|
||||
"Object data exceeds cap ({} bytes), aborting full read to prevent OOM: {}/{}",
|
||||
MAX_READ_BYTES, bucket, object
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read object data: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
debug!("Object data not found: {}/{}", bucket, object);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!("Failed to get object: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::other(e))
|
||||
return Err(Error::other(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(buf))
|
||||
}
|
||||
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
|
||||
@@ -208,27 +228,34 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool> {
|
||||
debug!("Verifying object integrity: {}/{}", bucket, object);
|
||||
|
||||
// Try to get object info and data to verify integrity
|
||||
// Check object metadata first
|
||||
match self.get_object_meta(bucket, object).await? {
|
||||
Some(obj_info) => {
|
||||
// Check if object has valid metadata
|
||||
if obj_info.size < 0 {
|
||||
warn!("Object has invalid size: {}/{}", bucket, object);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Try to read object data to verify it's accessible
|
||||
match self.get_object_data(bucket, object).await {
|
||||
Ok(Some(_)) => {
|
||||
info!("Object integrity check passed: {}/{}", bucket, object);
|
||||
Ok(true)
|
||||
// Stream-read the object to a sink to avoid loading into memory
|
||||
match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(reader) => {
|
||||
let mut stream = reader.stream;
|
||||
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
|
||||
Ok(_) => {
|
||||
info!("Object integrity check passed: {}/{}", bucket, object);
|
||||
Ok(true)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Object stream read failed: {}/{} - {}", bucket, object, e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!("Object data not found: {}/{}", bucket, object);
|
||||
Ok(false)
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("Object data read failed: {}/{}", bucket, object);
|
||||
Err(e) => {
|
||||
warn!("Failed to get object reader: {}/{} - {}", bucket, object, e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod app;
|
||||
pub(crate) mod env;
|
||||
pub(crate) mod tls;
|
||||
pub mod app;
|
||||
pub mod env;
|
||||
pub mod tls;
|
||||
|
||||
@@ -156,11 +156,12 @@ pub enum StorageError {
|
||||
#[error("Object exists on :{0} as directory {1}")]
|
||||
ObjectExistsAsDirectory(String, String),
|
||||
|
||||
// #[error("Storage resources are insufficient for the read operation")]
|
||||
// InsufficientReadQuorum,
|
||||
#[error("Storage resources are insufficient for the read operation: {0}/{1}")]
|
||||
InsufficientReadQuorum(String, String),
|
||||
|
||||
#[error("Storage resources are insufficient for the write operation: {0}/{1}")]
|
||||
InsufficientWriteQuorum(String, String),
|
||||
|
||||
// #[error("Storage resources are insufficient for the write operation")]
|
||||
// InsufficientWriteQuorum,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -413,6 +414,8 @@ impl Clone for StorageError {
|
||||
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageError::NoHealRequired,
|
||||
StorageError::Lock(e) => StorageError::Lock(e.clone()),
|
||||
StorageError::InsufficientReadQuorum(a, b) => StorageError::InsufficientReadQuorum(a.clone(), b.clone()),
|
||||
StorageError::InsufficientWriteQuorum(a, b) => StorageError::InsufficientWriteQuorum(a.clone(), b.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -476,6 +479,8 @@ impl StorageError {
|
||||
StorageError::TooManyOpenFiles => 0x36,
|
||||
StorageError::NoHealRequired => 0x37,
|
||||
StorageError::Lock(_) => 0x38,
|
||||
StorageError::InsufficientReadQuorum(_, _) => 0x39,
|
||||
StorageError::InsufficientWriteQuorum(_, _) => 0x3A,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -541,6 +546,8 @@ impl StorageError {
|
||||
0x36 => Some(StorageError::TooManyOpenFiles),
|
||||
0x37 => Some(StorageError::NoHealRequired),
|
||||
0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
|
||||
0x39 => Some(StorageError::InsufficientReadQuorum(Default::default(), Default::default())),
|
||||
0x3A => Some(StorageError::InsufficientWriteQuorum(Default::default(), Default::default())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -753,6 +760,17 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
|
||||
StorageError::PrefixAccessDenied(bucket, object)
|
||||
}
|
||||
|
||||
StorageError::ErasureReadQuorum => {
|
||||
let bucket = params.first().cloned().unwrap_or_default().to_owned();
|
||||
let object = params.get(1).cloned().map(decode_dir_object).unwrap_or_default();
|
||||
StorageError::InsufficientReadQuorum(bucket, object)
|
||||
}
|
||||
StorageError::ErasureWriteQuorum => {
|
||||
let bucket = params.first().cloned().unwrap_or_default().to_owned();
|
||||
let object = params.get(1).cloned().map(decode_dir_object).unwrap_or_default();
|
||||
StorageError::InsufficientWriteQuorum(bucket, object)
|
||||
}
|
||||
|
||||
_ => err,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
|
||||
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
||||
@@ -2027,6 +2029,24 @@ impl SetDisks {
|
||||
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
async fn get_object_info_and_quorum(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(ObjectInfo, usize)> {
|
||||
let (fi, _, _) = self.get_object_fileinfo(bucket, object, opts, false).await?;
|
||||
|
||||
let write_quorum = fi.write_quorum(self.default_write_quorum());
|
||||
|
||||
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
// TODO: replicatio
|
||||
|
||||
if fi.deleted {
|
||||
if opts.version_id.is_none() || opts.delete_marker {
|
||||
return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
|
||||
} else {
|
||||
return Err(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]));
|
||||
}
|
||||
}
|
||||
|
||||
Ok((oi, write_quorum))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(
|
||||
@@ -3769,39 +3789,47 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
|
||||
// Per-object guards to keep until function end
|
||||
let mut _guards: Vec<Option<rustfs_lock::LockGuard>> = Vec::with_capacity(objects.len());
|
||||
let mut _guards: HashMap<String, rustfs_lock::LockGuard> = HashMap::new();
|
||||
// Acquire locks for all objects first; mark errors for failures
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
match self
|
||||
.namespace_lock
|
||||
.lock_guard(&dobj.object_name, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?
|
||||
{
|
||||
Some(g) => _guards.push(Some(g)),
|
||||
None => {
|
||||
del_errs[i] = Some(Error::other("can not get lock. please retry"));
|
||||
_guards.push(None);
|
||||
if !_guards.contains_key(&dobj.object_name) {
|
||||
match self
|
||||
.namespace_lock
|
||||
.lock_guard(&dobj.object_name, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
|
||||
.await?
|
||||
{
|
||||
Some(g) => {
|
||||
_guards.insert(dobj.object_name.clone(), g);
|
||||
}
|
||||
None => {
|
||||
del_errs[i] = Some(Error::other("can not get lock. please retry"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// let mut del_fvers = Vec::with_capacity(objects.len());
|
||||
|
||||
let ver_cfg = BucketVersioningSys::get(bucket).await.unwrap_or_default();
|
||||
|
||||
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
|
||||
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
let mut vr = FileInfo {
|
||||
name: dobj.object_name.clone(),
|
||||
version_id: dobj.version_id,
|
||||
idx: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 删除
|
||||
del_objects[i].object_name.clone_from(&vr.name);
|
||||
del_objects[i].version_id = vr.version_id.map(|v| v.to_string());
|
||||
vr.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
|
||||
if del_objects[i].version_id.is_none() {
|
||||
let (suspended, versioned) = (opts.version_suspended, opts.versioned);
|
||||
// 删除
|
||||
// del_objects[i].object_name.clone_from(&vr.name);
|
||||
// del_objects[i].version_id = vr.version_id.map(|v| v.to_string());
|
||||
|
||||
if dobj.version_id.is_none() {
|
||||
let (suspended, versioned) = (ver_cfg.suspended(), ver_cfg.prefix_enabled(dobj.object_name.as_str()));
|
||||
if suspended || versioned {
|
||||
vr.mod_time = Some(OffsetDateTime::now_utc());
|
||||
vr.deleted = true;
|
||||
@@ -3842,15 +3870,22 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
|
||||
// Only add to vers_map if we hold the lock
|
||||
if _guards[i].is_some() {
|
||||
if _guards.contains_key(&dobj.object_name) {
|
||||
vers_map.insert(&dobj.object_name, v);
|
||||
}
|
||||
}
|
||||
|
||||
let mut vers = Vec::with_capacity(vers_map.len());
|
||||
|
||||
for (_, ver) in vers_map {
|
||||
vers.push(ver);
|
||||
for (_, mut fi_vers) in vers_map {
|
||||
fi_vers.versions.sort_by(|a, b| a.deleted.cmp(&b.deleted));
|
||||
fi_vers.versions.reverse();
|
||||
|
||||
if let Some(index) = fi_vers.versions.iter().position(|fi| fi.deleted) {
|
||||
fi_vers.versions.truncate(index + 1);
|
||||
}
|
||||
|
||||
vers.push(fi_vers);
|
||||
}
|
||||
|
||||
let disks = self.disks.read().await;
|
||||
@@ -3906,6 +3941,61 @@ impl StorageAPI for SetDisks {
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
|
||||
let (oi, write_quorum) = match self.get_object_info_and_quorum(bucket, object, &opts).await {
|
||||
Ok((oi, wq)) => (oi, wq),
|
||||
Err(e) => {
|
||||
return Err(to_object_err(e, vec![bucket, object]));
|
||||
}
|
||||
};
|
||||
|
||||
let mark_delete = oi.version_id.is_some();
|
||||
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
let mod_time = if let Some(mt) = opts.mod_time {
|
||||
mt
|
||||
} else {
|
||||
OffsetDateTime::now_utc()
|
||||
};
|
||||
|
||||
let find_vid = Uuid::new_v4();
|
||||
|
||||
if mark_delete && (opts.versioned || opts.version_suspended) {
|
||||
if !delete_marker {
|
||||
delete_marker = opts.version_suspended && opts.version_id.is_none();
|
||||
}
|
||||
|
||||
let mut fi = FileInfo {
|
||||
name: object.to_string(),
|
||||
deleted: delete_marker,
|
||||
mark_deleted: mark_delete,
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default() // TODO: replication
|
||||
};
|
||||
|
||||
fi.set_tier_free_version_id(&find_vid.to_string());
|
||||
|
||||
if opts.skip_free_version {
|
||||
fi.set_skip_tier_free_version();
|
||||
}
|
||||
|
||||
fi.version_id = if let Some(vid) = opts.version_id {
|
||||
Some(Uuid::parse_str(vid.as_str())?)
|
||||
} else if opts.versioned {
|
||||
Some(Uuid::new_v4())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.delete_object_version(bucket, object, &fi, opts.delete_marker)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
|
||||
return Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended));
|
||||
}
|
||||
|
||||
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
||||
|
||||
// Create a single object deletion request
|
||||
let mut vr = FileInfo {
|
||||
name: object.to_string(),
|
||||
@@ -5354,9 +5444,10 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
let mut get_object_reader =
|
||||
<Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let _ = get_object_reader.read_all().await?;
|
||||
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
// Stream to sink to avoid loading entire object into memory during verification
|
||||
let mut reader = get_object_reader.stream;
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -882,11 +882,15 @@ impl StorageAPI for Sets {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
self.get_disks_by_key(object)
|
||||
.verify_object_integrity(bucket, object, opts)
|
||||
.await
|
||||
let gor = self.get_object_reader(bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let mut reader = gor.stream;
|
||||
|
||||
// Stream data to sink instead of reading all into memory to prevent OOM
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2238,9 +2238,10 @@ impl StorageAPI for ECStore {
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
let mut get_object_reader =
|
||||
<Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
let _ = get_object_reader.read_all().await?;
|
||||
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
|
||||
// Stream to sink to avoid loading entire object into memory during verification
|
||||
let mut reader = get_object_reader.stream;
|
||||
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,6 +310,8 @@ pub struct ObjectOptions {
|
||||
pub replication_request: bool,
|
||||
pub delete_marker: bool,
|
||||
|
||||
pub skip_free_version: bool,
|
||||
|
||||
pub transition: TransitionOptions,
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
@@ -496,39 +496,32 @@ impl FileMeta {
|
||||
}
|
||||
|
||||
pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
|
||||
let mod_time = ver.get_mod_time().unwrap().nanosecond();
|
||||
if !ver.valid() {
|
||||
return Err(Error::other("attempted to add invalid version"));
|
||||
}
|
||||
let encoded = ver.marshal_msg()?;
|
||||
|
||||
if self.versions.len() + 1 > 100 {
|
||||
if self.versions.len() + 1 >= 100 {
|
||||
return Err(Error::other(
|
||||
"You've exceeded the limit on the number of versions you can create on this object",
|
||||
));
|
||||
}
|
||||
|
||||
self.versions.push(FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
let mod_time = ver.get_mod_time();
|
||||
let encoded = ver.marshal_msg()?;
|
||||
let new_version = FileMetaShallowVersion {
|
||||
header: ver.header(),
|
||||
meta: encoded,
|
||||
};
|
||||
|
||||
let len = self.versions.len();
|
||||
for (i, existing) in self.versions.iter().enumerate() {
|
||||
if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
|
||||
let vers = self.versions[i..len - 1].to_vec();
|
||||
self.versions[i + 1..].clone_from_slice(vers.as_slice());
|
||||
self.versions[i] = FileMetaShallowVersion {
|
||||
header: ver.header(),
|
||||
meta: encoded,
|
||||
};
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(Error::other("addVersion: Internal error, unable to add version"))
|
||||
// Find the insertion position: insert before the first element with mod_time >= new mod_time
|
||||
// This maintains descending order by mod_time (newest first)
|
||||
let insert_pos = self
|
||||
.versions
|
||||
.iter()
|
||||
.position(|existing| existing.header.mod_time <= mod_time)
|
||||
.unwrap_or(self.versions.len());
|
||||
self.versions.insert(insert_pos, new_version);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// delete_version deletes version, returns data_dir
|
||||
@@ -554,7 +547,15 @@ impl FileMeta {
|
||||
|
||||
match ver.header.version_type {
|
||||
VersionType::Invalid | VersionType::Legacy => return Err(Error::other("invalid file meta version")),
|
||||
VersionType::Delete => return Ok(None),
|
||||
VersionType::Delete => {
|
||||
self.versions.remove(i);
|
||||
if fi.deleted && fi.version_id.is_none() {
|
||||
self.add_version_filemata(ventry)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
return Ok(None);
|
||||
}
|
||||
VersionType::Object => {
|
||||
let v = self.get_idx(i)?;
|
||||
|
||||
@@ -600,6 +601,7 @@ impl FileMeta {
|
||||
|
||||
if fi.deleted {
|
||||
self.add_version_filemata(ventry)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Err(Error::FileVersionNotFound)
|
||||
@@ -961,7 +963,8 @@ impl FileMetaVersion {
|
||||
|
||||
pub fn get_version_id(&self) -> Option<Uuid> {
|
||||
match self.version_type {
|
||||
VersionType::Object | VersionType::Delete => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
VersionType::Object => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
VersionType::Delete => self.delete_marker.as_ref().map(|v| v.version_id).unwrap_or_default(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify"] }
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["path", "sys"] }
|
||||
async-trait = { workspace = true }
|
||||
|
||||
@@ -196,12 +196,13 @@ pub fn create_multi_cert_resolver(
|
||||
|
||||
/// Checks if TLS key logging is enabled.
|
||||
pub fn tls_key_log() -> bool {
|
||||
env::var(rustfs_config::ENV_TLS_KEYLOG)
|
||||
env::var("RUSTFS_TLS_KEYLOG")
|
||||
.map(|v| {
|
||||
v.eq_ignore_ascii_case(rustfs_config::EnableState::One.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::On.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::True.as_str())
|
||||
|| v.eq_ignore_ascii_case(rustfs_config::EnableState::Yes.as_str())
|
||||
let v = v.trim();
|
||||
v.eq_ignore_ascii_case("1")
|
||||
|| v.eq_ignore_ascii_case("on")
|
||||
|| v.eq_ignore_ascii_case("true")
|
||||
|| v.eq_ignore_ascii_case("yes")
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
@@ -183,10 +183,6 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
Error::other(err)
|
||||
})?;
|
||||
|
||||
// init scanner and auto heal with unified cancellation token
|
||||
// let _background_services_cancel_token = create_background_services_cancel_token();
|
||||
// init_data_scanner().await;
|
||||
// init_auto_heal().await;
|
||||
let _ = create_ahm_services_cancel_token();
|
||||
|
||||
// Initialize heal manager with channel processor
|
||||
|
||||
@@ -1318,7 +1318,7 @@ impl S3 for FS {
|
||||
let objects: Vec<ObjectVersion> = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|v| !v.name.is_empty())
|
||||
.filter(|v| !v.name.is_empty() && !v.delete_marker)
|
||||
.map(|v| {
|
||||
ObjectVersion {
|
||||
key: Some(v.name.to_owned()),
|
||||
@@ -1340,6 +1340,19 @@ impl S3 for FS {
|
||||
.map(|v| CommonPrefix { prefix: Some(v) })
|
||||
.collect();
|
||||
|
||||
let delete_markers = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|o| o.delete_marker)
|
||||
.map(|o| DeleteMarkerEntry {
|
||||
key: Some(o.name.clone()),
|
||||
version_id: o.version_id.map(|v| v.to_string()),
|
||||
is_latest: Some(o.is_latest),
|
||||
last_modified: o.mod_time.map(Timestamp::from),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let output = ListObjectVersionsOutput {
|
||||
// is_truncated: Some(object_infos.is_truncated),
|
||||
max_keys: Some(key_count),
|
||||
@@ -1348,6 +1361,7 @@ impl S3 for FS {
|
||||
prefix: Some(prefix),
|
||||
common_prefixes: Some(common_prefixes),
|
||||
versions: Some(objects),
|
||||
delete_markers: Some(delete_markers),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -722,8 +722,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
metadata.get("content-type"),
|
||||
Some(&expected_content_type.to_string()),
|
||||
"Failed for filename: {}",
|
||||
filename
|
||||
"Failed for filename: {filename}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user