sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-17 01:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/logger

Browse Source
This commit is contained in:
houseme
2025-02-23 09:21:07 +08:00
parent ff41d4f964 e87cc87cbf
commit b11665b1eb
10 changed files with 184 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
.github/actions/setup/action.yml vendored Normal file
View File

@@ -0,0 +1,45 @@
name: "setup"
description: "setup environment for rustfs"
inputs:
rust-version:
required: true
default: "stable"
cache-shared-key:
required: true
default: ""
runs:
using: "composite"
steps:
- name: Install system dependencies
shell: bash
run: |
sudo apt update
sudo apt install -y \
libdbus-1-dev \
libwayland-dev \
libwebkit2gtk-4.1-dev \
libxdo-dev
- uses: arduino/setup-protoc@v3
with:
version: "27.0"
- uses: Nugine/setup-flatc@v1
with:
version: "24.3.25"
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ inputs.rust-version }}
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
shared-key: ${{ inputs.cache-shared-key }}
- uses: mlugg/setup-zig@v1
- uses: taiki-e/install-action@cargo-zigbuild

4
.github/workflows/audit.yml vendored
View File

@@ -21,7 +21,5 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: taiki-e/install-action@v2
with:
tool: cargo-audit
- uses: taiki-e/install-action@cargo-audit
- run: cargo audit -D warnings

45
.github/workflows/build.yml vendored
View File

@@ -12,33 +12,36 @@ jobs:
build-rustfs:
runs-on: ubuntu-latest
strategy:
matrix:
variant:
- { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: arduino/setup-protoc@v3
- uses: ./.github/actions/setup
with:
version: "27.0"
cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
- uses: Nugine/setup-flatc@v1
with:
version: "24.3.25"
- name: Build binaries
- name: Build
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins
touch rustfs/build.rs
cargo build -p rustfs --bins --release
mkdir -p target/artifacts
mv ./target/debug/rustfs ./target/artifacts/rustfs.debug.x86_64-unknown-linux-gnu
mv ./target/release/rustfs ./target/artifacts/rustfs.release.x86_64-unknown-linux-gnu
./scripts/build.py \
--profile ${{ matrix.variant.profile }} \
--target ${{ matrix.variant.target }} \
--glibc ${{ matrix.variant.glibc }}
- uses: actions/upload-artifact@v4
with:
name: rustfs
name: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
path: ./target/artifacts/*
merge:
runs-on: ubuntu-latest
needs: build-rustfs
steps:
- uses: actions/upload-artifact/merge@v4
with:
name: rustfs
delete-merged: true

27
.github/workflows/ci.yml vendored
View File

@@ -37,33 +37,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Install system dependencies
run: |
sudo apt update
sudo apt install -y \
libdbus-1-dev \
libwayland-dev \
libwebkit2gtk-4.1-dev \
libxdo-dev
- uses: arduino/setup-protoc@v3
with:
version: "27.0"
- uses: Nugine/setup-flatc@v1
with:
version: "24.3.25"
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@nightly
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
shared-key: "develop"
- uses: ./.github/actions/setup
- name: Format
run: cargo fmt --all --check

45
.github/workflows/e2e.yml vendored
View File

@@ -37,14 +37,7 @@ jobs:
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: arduino/setup-protoc@v3
with:
version: "27.0"
- uses: Nugine/setup-flatc@v1
with:
version: "24.3.25"
- uses: ./.github/actions/setup
- name: Build binaries
run: |
@@ -53,7 +46,11 @@ jobs:
- name: Install s3s-e2e
run: |
cargo install s3s-e2e --git https://github.com/Nugine/s3s.git
mkdir -p target/s3s-e2e
cargo install s3s-e2e \
--git https://github.com/Nugine/s3s.git \
--target-dir target/s3s-e2e \
--force
s3s-e2e --version
- name: Run tests
@@ -70,7 +67,35 @@ jobs:
export RUST_BACKTRACE=full
s3s-e2e
killall rustfs
mint:
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
- name: Build binaries
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins --release
- name: run this rustfs server
run: |
sudo mkdir -p /data/rustfs
sudo nohup ./target/release/rustfs /data/rustfs &
- name: run mint test task
run: |
ps aux | grep rustfs
SERVER_ADDRESS=$(ifconfig eth0 | awk 'NR==2 { print $2 }')
docker run -e SERVER_ENDPOINT=${SERVER_ADDRESS}:9000 \
-e ACCESS_KEY=rustfsadmin \
-e SECRET_KEY=rustfsadmin \
-e ENABLE_HTTPS=0 \
minio/mint
### Temporarily disabled because it is not working
# e2e:
# # See this url if required matrix jobs are hanging

62
.github/workflows/mint.yml vendored
View File

@@ -1,62 +0,0 @@
name: mint-test
on:
workflow_dispatch:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
env:
CARGO_TERM_COLOR: always
jobs:
mintest:
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: install protoc
run: |
wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip
unzip protoc-27.0-linux-x86_64.zip -d protoc3
mv protoc3/bin/* /usr/local/bin/
chmod +x /usr/local/bin/protoc
rm -rf protoc-27.0-linux-x86_64.zip protoc3
- name: install flatc
run: |
wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip
unzip Linux.flatc.binary.g++-13.zip
mv flatc /usr/local/bin/
chmod +x /usr/local/bin/flatc
rm -rf Linux.flatc.binary.g++-13.zip
- name: checkout source code
uses: actions/checkout@v4
- name: rust-toolchain
uses: actions-rs/toolchain@v1.0.6
with:
toolchain: stable
- name: cache rust dependencies
uses: Swatinem/rust-cache@v2
- name: cargo build & release
run: cargo build --release
- name: run this rustfs server
run: |
sudo mkdir -p /data/rustfs
sudo nohup ./target/release/rustfs /data/rustfs &
- name: run mint test task
run: |
ps aux | grep rustfs
SERVER_ADDRESS=$(ifconfig eth0 | awk 'NR==2 { print $2 }')
docker run -e SERVER_ENDPOINT=${SERVER_ADDRESS}:9000 \
-e ACCESS_KEY=rustfsadmin \
-e SECRET_KEY=rustfsadmin \
-e ENABLE_HTTPS=0 \
minio/mint

80
Cargo.lock generated
View File

@@ -996,12 +996,6 @@ dependencies = [
"syn 2.0.98",
]
[[package]]
name = "const_fn"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e"
[[package]]
name = "const_format"
version = "0.2.34"
@@ -2547,19 +2541,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "git2"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff"
dependencies = [
"bitflags 2.8.0",
"libc",
"libgit2-sys",
"log",
"url",
]
[[package]]
name = "glib"
version = "0.18.5"
@@ -3487,18 +3468,6 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "libgit2-sys"
version = "0.18.0+1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec"
dependencies = [
"cc",
"libc",
"libz-sys",
"pkg-config",
]
[[package]]
name = "libloading"
version = "0.7.4"
@@ -3554,18 +3523,6 @@ dependencies = [
"x11",
]
[[package]]
name = "libz-sys"
version = "1.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
@@ -6020,10 +5977,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ec14cc798c29f4bf74a6c4299c657c04d4e9fba03875c1f0eec569af03aed89"
dependencies = [
"const_format",
"git2",
"is_debug",
"time",
"tzdb",
]
[[package]]
@@ -7008,35 +6963,6 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
[[package]]
name = "tz-rs"
version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4"
dependencies = [
"const_fn",
]
[[package]]
name = "tzdb"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b580f6b365fa89f5767cdb619a55d534d04a4e14c2d7e5b9a31e94598687fb1"
dependencies = [
"iana-time-zone",
"tz-rs",
"tzdb_data",
]
[[package]]
name = "tzdb_data"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4471adcfcbd3052e8c5b5890a04a559837444b3be26b9cbbd622063171cec9d"
dependencies = [
"tz-rs",
]
[[package]]
name = "uds_windows"
version = "1.1.0"
@@ -7159,12 +7085,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version-compare"
version = "0.2.0"

1
Cargo.toml
View File

@@ -83,6 +83,7 @@ s3s = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fb
"tower",
] }
s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fbf7c4c2ab155db37dfe" }
shadow-rs = { version = "0.38.0", default-features = false }
serde = { version = "1.0.217", features = ["derive"] }
serde_json = "1.0.138"
tempfile = "3.16.0"

4
rustfs/Cargo.toml
View File

@@ -61,7 +61,7 @@ url.workspace = true
admin = { path = "../api/admin" }
axum.workspace = true
matchit = "0.8.6"
shadow-rs = "0.38.0"
shadow-rs.workspace = true
const-str = { version = "0.6.1", features = ["std", "proc"] }
atoi = "2.0.0"
serde_urlencoded = "0.7.1"
@@ -90,5 +90,5 @@ hyper-util = { version = "0.1.10", features = [
] }
transform-stream = "0.3.1"
netif = "0.1.6"
shadow-rs = "0.38.0"
shadow-rs.workspace = true
# pin-utils = "0.1.0"

75
scripts/build.py Executable file
View File

@@ -0,0 +1,75 @@
#!/usr/bin/env python
from dataclasses import dataclass
import argparse
import subprocess
from pathlib import Path
@dataclass
class CliArgs:
profile: str
target: str
glibc: str
@staticmethod
def parse():
parser = argparse.ArgumentParser()
parser.add_argument("--profile", type=str, required=True)
parser.add_argument("--target", type=str, required=True)
parser.add_argument("--glibc", type=str, required=True)
args = parser.parse_args()
return CliArgs(args.profile, args.target, args.glibc)
def shell(cmd: str):
print(cmd, flush=True)
subprocess.run(cmd, shell=True, check=True)
def main(args: CliArgs):
use_zigbuild = False
use_old_glibc = False
if args.glibc and args.glibc != "default":
use_zigbuild = True
use_old_glibc = True
cmd = ["cargo", "build"]
if use_zigbuild:
cmd = ["cargo", " zigbuild"]
cmd.extend(["--profile", args.profile])
if use_old_glibc:
cmd.extend(["--target", f"{args.target}.{args.glibc}"])
else:
cmd.extend(["--target", args.target])
cmd.extend(["-p", "rustfs"])
cmd.extend(["--bins"])
shell("touch rustfs/build.rs") # refresh build info for rustfs
shell(" ".join(cmd))
if args.profile == "dev":
profile_dir = "debug"
elif args.profile == "release":
profile_dir = "release"
else:
profile_dir = args.profile
bin_path = Path(f"target/{args.target}/{profile_dir}/rustfs")
bin_name = f"rustfs.{args.profile}.{args.target}"
if use_old_glibc:
bin_name += f".glibc{args.glibc}"
bin_name += ".bin"
out_path = Path(f"target/artifacts/{bin_name}")
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.hardlink_to(bin_path)
if __name__ == "__main__":
main(CliArgs.parse())