Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

# Conflicts:
#	.github/workflows/build.yml
#	.github/workflows/ci.yml
#	Cargo.lock
#	Cargo.toml
#	appauth/src/token.rs
#	crates/config/src/config.rs
#	crates/event-notifier/examples/simple.rs
#	crates/event-notifier/src/global.rs
#	crates/event-notifier/src/lib.rs
#	crates/event-notifier/src/notifier.rs
#	crates/event-notifier/src/store.rs
#	crates/filemeta/src/filemeta.rs
#	crates/notify/examples/webhook.rs
#	crates/utils/Cargo.toml
#	ecstore/Cargo.toml
#	ecstore/src/cmd/bucket_replication.rs
#	ecstore/src/config/com.rs
#	ecstore/src/disk/error.rs
#	ecstore/src/disk/mod.rs
#	ecstore/src/set_disk.rs
#	ecstore/src/store_api.rs
#	ecstore/src/store_list_objects.rs
#	iam/Cargo.toml
#	iam/src/manager.rs
#	policy/Cargo.toml
#	rustfs/src/admin/rpc.rs
#	rustfs/src/main.rs
#	rustfs/src/storage/mod.rs
This commit is contained in:
houseme
2025-06-19 13:16:48 +08:00
249 changed files with 25137 additions and 11731 deletions

View File

@@ -13,9 +13,9 @@ inputs:
description: "Cache key for shared cache"
cache-save-if:
required: true
default: true
default: ${{ github.ref == 'refs/heads/main' }}
description: "Cache save condition"
run-os:
runs-on:
required: true
default: "ubuntu-latest"
description: "Running system"
@@ -24,7 +24,7 @@ runs:
using: "composite"
steps:
- name: Install system dependencies
if: inputs.run-os == 'ubuntu-latest'
if: inputs.runs-on == 'ubuntu-latest'
shell: bash
run: |
sudo apt update
@@ -45,7 +45,6 @@ runs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
cache-all-crates: true
shared-key: ${{ inputs.cache-shared-key }}
save-if: ${{ inputs.cache-save-if }}

View File

@@ -17,42 +17,114 @@ jobs:
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
variant:
- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- {
profile: release,
target: x86_64-unknown-linux-musl,
glibc: "default",
}
- {
profile: release,
target: x86_64-unknown-linux-gnu,
glibc: "default",
}
- { profile: release, target: aarch64-apple-darwin, glibc: "default" }
#- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
- {
profile: release,
target: aarch64-unknown-linux-musl,
glibc: "default",
}
#- { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
exclude:
# Linux targets on non-Linux systems
- os: macos-latest
variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
variant:
{
profile: release,
target: x86_64-unknown-linux-gnu,
glibc: "default",
}
- os: macos-latest
variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
variant:
{
profile: release,
target: x86_64-unknown-linux-musl,
glibc: "default",
}
- os: macos-latest
variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
variant:
{
profile: release,
target: aarch64-unknown-linux-gnu,
glibc: "default",
}
- os: macos-latest
variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
variant:
{
profile: release,
target: aarch64-unknown-linux-musl,
glibc: "default",
}
- os: windows-latest
variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
variant:
{
profile: release,
target: x86_64-unknown-linux-gnu,
glibc: "default",
}
- os: windows-latest
variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
variant:
{
profile: release,
target: x86_64-unknown-linux-musl,
glibc: "default",
}
- os: windows-latest
variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
variant:
{
profile: release,
target: aarch64-unknown-linux-gnu,
glibc: "default",
}
- os: windows-latest
variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
variant:
{
profile: release,
target: aarch64-unknown-linux-musl,
glibc: "default",
}
# Apple targets on non-macOS systems
- os: ubuntu-latest
variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" }
variant:
{
profile: release,
target: aarch64-apple-darwin,
glibc: "default",
}
- os: windows-latest
variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" }
variant:
{
profile: release,
target: aarch64-apple-darwin,
glibc: "default",
}
# Windows targets on non-Windows systems
- os: ubuntu-latest
variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
variant:
{
profile: release,
target: x86_64-pc-windows-msvc,
glibc: "default",
}
- os: macos-latest
variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
variant:
{
profile: release,
target: x86_64-pc-windows-msvc,
glibc: "default",
}
steps:
- name: Checkout repository
@@ -89,7 +161,7 @@ jobs:
if: steps.cache-protoc.outputs.cache-hit != 'true'
uses: arduino/setup-protoc@v3
with:
version: '31.1'
version: "31.1"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Flatc
@@ -107,10 +179,10 @@ jobs:
# Set up Zig for cross-compilation
- uses: mlugg/setup-zig@v2
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux')
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'aarch64-unknown-linux')
- uses: taiki-e/install-action@cargo-zigbuild
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux')
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'aarch64-unknown-linux')
# Download static resources
- name: Download and Extract Static Assets
@@ -150,7 +222,7 @@ jobs:
# Determine whether to use zigbuild
USE_ZIGBUILD=false
if [[ "$GLIBC" != "default" || "$TARGET" == *"linux"* ]]; then
if [[ "$GLIBC" != "default" || "$TARGET" == *"aarch64-unknown-linux"* ]]; then
USE_ZIGBUILD=true
echo "Using zigbuild for cross-compilation"
fi
@@ -180,14 +252,14 @@ jobs:
if [[ "$GLIBC" != "default" ]]; then
BIN_NAME="${BIN_NAME}.glibc${GLIBC}"
fi
# Windows systems use exe suffix, and other systems do not have suffix
if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then
BIN_NAME="${BIN_NAME}.exe"
else
BIN_NAME="${BIN_NAME}.bin"
fi
echo "Binary name will be: $BIN_NAME"
echo "::group::Building rustfs"
@@ -265,17 +337,56 @@ jobs:
path: ${{ steps.package.outputs.artifact_name }}.zip
retention-days: 7
# Install ossutil2 tool for OSS upload
- name: Install ossutil2
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main'
shell: bash
run: |
echo "::group::Installing ossutil2"
# Download and install ossutil based on platform
if [ "${{ runner.os }}" = "Linux" ]; then
curl -o ossutil.zip https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-linux-amd64.zip
unzip -o ossutil.zip
chmod 755 ossutil-2.1.1-linux-amd64/ossutil
sudo mv ossutil-2.1.1-linux-amd64/ossutil /usr/local/bin/
rm -rf ossutil.zip ossutil-2.1.1-linux-amd64
elif [ "${{ runner.os }}" = "macOS" ]; then
if [ "$(uname -m)" = "arm64" ]; then
curl -o ossutil.zip https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-mac-arm64.zip
else
curl -o ossutil.zip https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-mac-amd64.zip
fi
unzip -o ossutil.zip
chmod 755 ossutil-*/ossutil
sudo mv ossutil-*/ossutil /usr/local/bin/
rm -rf ossutil.zip ossutil-*
elif [ "${{ runner.os }}" = "Windows" ]; then
curl -o ossutil.zip https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-windows-amd64.zip
unzip -o ossutil.zip
mv ossutil-*/ossutil.exe /usr/bin/ossutil.exe
rm -rf ossutil.zip ossutil-*
fi
echo "ossutil2 installation completed"
# Set the OSS configuration
ossutil config set Region oss-cn-beijing
ossutil config set endpoint oss-cn-beijing.aliyuncs.com
ossutil config set accessKeyID ${{ secrets.ALICLOUDOSS_KEY_ID }}
ossutil config set accessKeySecret ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
- name: Upload to Aliyun OSS
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main'
uses: JohnGuan/oss-upload-action@main
with:
key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }}
key-secret: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
region: oss-cn-beijing
bucket: rustfs-artifacts
assets: |
${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.zip
${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip
shell: bash
env:
OSSUTIL_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
OSSUTIL_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
OSSUTIL_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
run: |
echo "::group::Uploading files to OSS"
# Upload the artifact file to two different paths
ossutil cp "${{ steps.package.outputs.artifact_name }}.zip" "oss://rustfs-artifacts/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.zip" --force
ossutil cp "${{ steps.package.outputs.artifact_name }}.zip" "oss://rustfs-artifacts/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip" --force
echo "Successfully uploaded artifacts to OSS"
# Determine whether to perform GUI construction based on conditions
- name: Prepare for GUI build
@@ -393,16 +504,17 @@ jobs:
# Upload GUI to Alibaba Cloud OSS
- name: Upload GUI to Aliyun OSS
if: startsWith(github.ref, 'refs/tags/')
uses: JohnGuan/oss-upload-action@main
with:
key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }}
key-secret: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
region: oss-cn-beijing
bucket: rustfs-artifacts
assets: |
${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.zip
${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.latest.zip
shell: bash
env:
OSSUTIL_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
OSSUTIL_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
OSSUTIL_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
run: |
echo "::group::Uploading GUI files to OSS"
# Upload the GUI artifact file to two different paths
ossutil cp "${{ steps.build_gui.outputs.gui_artifact_name }}.zip" "oss://rustfs-artifacts/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.zip" --force
ossutil cp "${{ steps.build_gui.outputs.gui_artifact_name }}.zip" "oss://rustfs-artifacts/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.latest.zip" --force
echo "Successfully uploaded GUI artifacts to OSS"
merge:
runs-on: ubuntu-latest

View File

@@ -11,9 +11,6 @@ on:
- cron: '0 0 * * 0' # at midnight of each sunday
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
jobs:
skip-check:
permissions:
@@ -30,93 +27,52 @@ jobs:
cancel_others: true
paths_ignore: '["*.md"]'
# Quality checks for pull requests
pr-checks:
name: Pull Request Quality Checks
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.2.2
- uses: ./.github/actions/setup
- name: Format Check
run: cargo fmt --all --check
- name: Lint Check
run: cargo check --all-targets
- name: Clippy Check
run: cargo clippy --all-targets --all-features -- -D warnings
- name: Unit Tests
run: cargo test --all --exclude e2e_test
develop:
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4.2.2
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
- name: Test
run: cargo test --all --exclude e2e_test
- name: Format
run: cargo fmt --all --check
- name: Lint
run: cargo check --all-targets
- name: Clippy
run: cargo clippy --all-targets --all-features -- -D warnings
- name: Test
run: cargo test --all --exclude e2e_test
s3s-e2e:
name: E2E (s3s-e2e)
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4.2.2
- uses: ./.github/actions/setup
- name: Install s3s-e2e
uses: taiki-e/cache-cargo-install-action@v2
with:
tool: s3s-e2e
git: https://github.com/Nugine/s3s.git
rev: b7714bfaa17ddfa9b23ea01774a1e7bbdbfc2ca3
- name: Build debug
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins
- name: Pack artifacts
run: |
mkdir -p ./target/artifacts
cp target/debug/rustfs ./target/artifacts/rustfs-debug
- uses: actions/upload-artifact@v4
with:
name: rustfs
path: ./target/artifacts/*
s3s-e2e:
name: E2E (s3s-e2e)
needs:
- skip-check
- develop
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.2.2
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
cache-all-crates: true
- name: Install s3s-e2e
run: |
cargo install s3s-e2e --git https://github.com/Nugine/s3s.git
s3s-e2e --version
- uses: actions/download-artifact@v4
with:
name: rustfs
path: ./target/artifacts
- name: Run s3s-e2e
timeout-minutes: 10
run: |
./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /tmp/rustfs
s3s-e2e --version
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
- uses: actions/upload-artifact@v4
with:
name: s3s-e2e.logs
path: /tmp/rustfs.log
path: /tmp/rustfs.log

227
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,227 @@
name: Build and Push Docker Images
on:
push:
branches:
- main
tags:
- "v*"
pull_request:
branches:
- main
workflow_dispatch:
inputs:
push_to_registry:
description: "Push images to registry"
required: false
default: "true"
type: boolean
env:
REGISTRY_IMAGE_DOCKERHUB: rustfs/rustfs
REGISTRY_IMAGE_GHCR: ghcr.io/${{ github.repository }}
jobs:
# Skip duplicate job runs
skip-check:
permissions:
actions: write
contents: read
runs-on: ubuntu-latest
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
steps:
- id: skip_check
uses: fkirc/skip-duplicate-actions@v5
with:
concurrent_skipping: "same_content_newer"
cancel_others: true
paths_ignore: '["*.md", "docs/**"]'
# Build RustFS binary for different platforms
build-binary:
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
os: ubuntu-latest
arch: amd64
use_cross: false
- target: aarch64-unknown-linux-gnu
os: ubuntu-latest
arch: arm64
use_cross: true
runs-on: ${{ matrix.os }}
timeout-minutes: 120
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
target: ${{ matrix.target }}
components: rustfmt, clippy
- name: Install cross-compilation dependencies (native build)
if: matrix.use_cross == false
run: |
sudo apt-get update
sudo apt-get install -y musl-tools
- name: Install cross tool (cross compilation)
if: matrix.use_cross == true
uses: taiki-e/install-action@v2
with:
tool: cross
- name: Install protoc
uses: arduino/setup-protoc@v3
with:
version: "31.1"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install flatc
uses: Nugine/setup-flatc@v1
with:
version: "25.2.10"
- name: Cache cargo dependencies
uses: actions/cache@v3
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-${{ matrix.target }}-
${{ runner.os }}-cargo-
- name: Generate protobuf code
run: cargo run --bin gproto
- name: Build RustFS binary (native)
if: matrix.use_cross == false
run: |
cargo build --release --target ${{ matrix.target }} --bin rustfs
- name: Build RustFS binary (cross)
if: matrix.use_cross == true
run: |
cross build --release --target ${{ matrix.target }} --bin rustfs
- name: Upload binary artifact
uses: actions/upload-artifact@v4
with:
name: rustfs-${{ matrix.arch }}
path: target/${{ matrix.target }}/release/rustfs
retention-days: 1
# Build and push multi-arch Docker images
build-images:
needs: [skip-check, build-binary]
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
image-type: [production, ubuntu, rockylinux, devenv]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download binary artifacts
uses: actions/download-artifact@v4
with:
path: ./artifacts
- name: Setup binary files
run: |
mkdir -p target/x86_64-unknown-linux-musl/release
mkdir -p target/aarch64-unknown-linux-gnu/release
cp artifacts/rustfs-amd64/rustfs target/x86_64-unknown-linux-musl/release/
cp artifacts/rustfs-arm64/rustfs target/aarch64-unknown-linux-gnu/release/
chmod +x target/*/release/rustfs
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Login to Docker Hub
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set Dockerfile and context
id: dockerfile
run: |
case "${{ matrix.image-type }}" in
production)
echo "dockerfile=Dockerfile" >> $GITHUB_OUTPUT
echo "context=." >> $GITHUB_OUTPUT
echo "suffix=" >> $GITHUB_OUTPUT
;;
ubuntu)
echo "dockerfile=.docker/Dockerfile.ubuntu22.04" >> $GITHUB_OUTPUT
echo "context=." >> $GITHUB_OUTPUT
echo "suffix=-ubuntu22.04" >> $GITHUB_OUTPUT
;;
rockylinux)
echo "dockerfile=.docker/Dockerfile.rockylinux9.3" >> $GITHUB_OUTPUT
echo "context=." >> $GITHUB_OUTPUT
echo "suffix=-rockylinux9.3" >> $GITHUB_OUTPUT
;;
devenv)
echo "dockerfile=.docker/Dockerfile.devenv" >> $GITHUB_OUTPUT
echo "context=." >> $GITHUB_OUTPUT
echo "suffix=-devenv" >> $GITHUB_OUTPUT
;;
esac
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY_IMAGE_DOCKERHUB }}
${{ env.REGISTRY_IMAGE_GHCR }}
tags: |
type=ref,event=branch,suffix=${{ steps.dockerfile.outputs.suffix }}
type=ref,event=pr,suffix=${{ steps.dockerfile.outputs.suffix }}
type=semver,pattern={{version}},suffix=${{ steps.dockerfile.outputs.suffix }}
type=semver,pattern={{major}}.{{minor}},suffix=${{ steps.dockerfile.outputs.suffix }}
type=semver,pattern={{major}},suffix=${{ steps.dockerfile.outputs.suffix }}
type=raw,value=latest,suffix=${{ steps.dockerfile.outputs.suffix }},enable={{is_default_branch}}
flavor: |
latest=false
- name: Build and push multi-arch Docker image
uses: docker/build-push-action@v5
with:
context: ${{ steps.dockerfile.outputs.context }}
file: ${{ steps.dockerfile.outputs.dockerfile }}
platforms: linux/amd64,linux/arm64
push: ${{ (github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))) || github.event.inputs.push_to_registry == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.image-type }}
cache-to: type=gha,mode=max,scope=${{ matrix.image-type }}
build-args: |
BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}