From e87cc87cbfef6e73083fc3033922e5f962af2b09 Mon Sep 17 00:00:00 2001 From: Nugine Date: Sat, 22 Feb 2025 20:25:16 +0800 Subject: [PATCH] ci: build for old glibc (#232) * build(deps): shadow_rs * ci(build): build for old glibc * ci: simplify all --- .github/actions/setup/action.yml | 45 ++++++++++++++++++ .github/workflows/audit.yml | 4 +- .github/workflows/build.yml | 45 +++++++++--------- .github/workflows/ci.yml | 27 +---------- .github/workflows/e2e.yml | 45 ++++++++++++++---- .github/workflows/mint.yml | 62 ------------------------- Cargo.lock | 80 -------------------------------- Cargo.toml | 1 + rustfs/Cargo.toml | 4 +- scripts/build.py | 75 ++++++++++++++++++++++++++++++ 10 files changed, 184 insertions(+), 204 deletions(-) create mode 100644 .github/actions/setup/action.yml delete mode 100644 .github/workflows/mint.yml create mode 100755 scripts/build.py diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..9447a8cd --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,45 @@ +name: "setup" + +description: "setup environment for rustfs" + +inputs: + rust-version: + required: true + default: "stable" + cache-shared-key: + required: true + default: "" + +runs: + using: "composite" + steps: + - name: Install system dependencies + shell: bash + run: | + sudo apt update + sudo apt install -y \ + libdbus-1-dev \ + libwayland-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev + + - uses: arduino/setup-protoc@v3 + with: + version: "27.0" + + - uses: Nugine/setup-flatc@v1 + with: + version: "24.3.25" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ inputs.rust-version }} + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + shared-key: ${{ inputs.cache-shared-key }} + + - uses: mlugg/setup-zig@v1 + - uses: taiki-e/install-action@cargo-zigbuild diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 7cd4eb54..da6edbed 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -21,7 +21,5 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: taiki-e/install-action@v2 - with: - tool: cargo-audit + - uses: taiki-e/install-action@cargo-audit - run: cargo audit -D warnings diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6fee49ef..b85c3f80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,33 +12,36 @@ jobs: build-rustfs: runs-on: ubuntu-latest + strategy: + matrix: + variant: + - { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" } + - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" } + steps: - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - uses: arduino/setup-protoc@v3 + - uses: ./.github/actions/setup with: - version: "27.0" + cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} - - uses: Nugine/setup-flatc@v1 - with: - version: "24.3.25" - - - name: Build binaries + - name: Build run: | - touch rustfs/build.rs - cargo build -p rustfs --bins - - touch rustfs/build.rs - cargo build -p rustfs --bins --release - - mkdir -p target/artifacts - mv ./target/debug/rustfs ./target/artifacts/rustfs.debug.x86_64-unknown-linux-gnu - mv ./target/release/rustfs ./target/artifacts/rustfs.release.x86_64-unknown-linux-gnu + ./scripts/build.py \ + --profile ${{ matrix.variant.profile }} \ + --target ${{ matrix.variant.target }} \ + --glibc ${{ matrix.variant.glibc }} - uses: actions/upload-artifact@v4 with: - name: rustfs + name: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} path: ./target/artifacts/* + + merge: + runs-on: ubuntu-latest + needs: build-rustfs + steps: + - uses: actions/upload-artifact/merge@v4 + with: + name: rustfs + delete-merged: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 358e7828..486bfacd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,33 +37,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Install system dependencies - run: | - sudo apt update - sudo apt install -y \ - libdbus-1-dev \ - libwayland-dev \ - libwebkit2gtk-4.1-dev \ - libxdo-dev - - - uses: arduino/setup-protoc@v3 - with: - version: "27.0" - - - uses: Nugine/setup-flatc@v1 - with: - version: "24.3.25" - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@nightly - with: - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - shared-key: "develop" + - uses: ./.github/actions/setup - name: Format run: cargo fmt --all --check diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b50db84c..9ec14b81 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -37,14 +37,7 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: arduino/setup-protoc@v3 - with: - version: "27.0" - - uses: Nugine/setup-flatc@v1 - with: - version: "24.3.25" + - uses: ./.github/actions/setup - name: Build binaries run: | @@ -53,7 +46,11 @@ jobs: - name: Install s3s-e2e run: | - cargo install s3s-e2e --git https://github.com/Nugine/s3s.git + mkdir -p target/s3s-e2e + cargo install s3s-e2e \ + --git https://github.com/Nugine/s3s.git \ + --target-dir target/s3s-e2e \ + --force s3s-e2e --version - name: Run tests @@ -70,7 +67,35 @@ jobs: export RUST_BACKTRACE=full s3s-e2e killall rustfs - + mint: + needs: skip-check + if: needs.skip-check.outputs.should_skip != 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + + - name: Build binaries + run: | + touch rustfs/build.rs + cargo build -p rustfs --bins --release + + - name: run this rustfs server + run: | + sudo mkdir -p /data/rustfs + sudo nohup ./target/release/rustfs /data/rustfs & + + - name: run mint test task + run: | + ps aux | grep rustfs + SERVER_ADDRESS=$(ifconfig eth0 | awk 'NR==2 { print $2 }') + docker run -e SERVER_ENDPOINT=${SERVER_ADDRESS}:9000 \ + -e ACCESS_KEY=rustfsadmin \ + -e SECRET_KEY=rustfsadmin \ + -e ENABLE_HTTPS=0 \ + minio/mint + ### Temporarily disabled because it is not working # e2e: # # See this url if required matrix jobs are hanging diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml deleted file mode 100644 index 75e45666..00000000 --- a/.github/workflows/mint.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: mint-test - -on: - workflow_dispatch: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -env: - CARGO_TERM_COLOR: always - -jobs: - mintest: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: install protoc - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip - unzip protoc-27.0-linux-x86_64.zip -d protoc3 - mv protoc3/bin/* /usr/local/bin/ - chmod +x /usr/local/bin/protoc - rm -rf protoc-27.0-linux-x86_64.zip protoc3 - - - name: install flatc - run: | - wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip - unzip Linux.flatc.binary.g++-13.zip - mv flatc /usr/local/bin/ - chmod +x /usr/local/bin/flatc - rm -rf Linux.flatc.binary.g++-13.zip - - - name: checkout source code - uses: actions/checkout@v4 - - - name: rust-toolchain - uses: actions-rs/toolchain@v1.0.6 - with: - toolchain: stable - - - name: cache rust dependencies - uses: Swatinem/rust-cache@v2 - - - name: cargo build & release - run: cargo build --release - - - name: run this rustfs server - run: | - sudo mkdir -p /data/rustfs - sudo nohup ./target/release/rustfs /data/rustfs & - - - name: run mint test task - run: | - ps aux | grep rustfs - SERVER_ADDRESS=$(ifconfig eth0 | awk 'NR==2 { print $2 }') - docker run -e SERVER_ENDPOINT=${SERVER_ADDRESS}:9000 \ - -e ACCESS_KEY=rustfsadmin \ - -e SECRET_KEY=rustfsadmin \ - -e ENABLE_HTTPS=0 \ - minio/mint - diff --git a/Cargo.lock b/Cargo.lock index b8475b8d..2b12bad9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -996,12 +996,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "const_fn" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" - [[package]] name = "const_format" version = "0.2.34" @@ -2547,19 +2541,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "git2" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" -dependencies = [ - "bitflags 2.8.0", - "libc", - "libgit2-sys", - "log", - "url", -] - [[package]] name = "glib" version = "0.18.5" @@ -3487,18 +3468,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libgit2-sys" -version = "0.18.0+1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - [[package]] name = "libloading" version = "0.7.4" @@ -3554,18 +3523,6 @@ dependencies = [ "x11", ] -[[package]] -name = "libz-sys" -version = "1.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5866,10 +5823,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ec14cc798c29f4bf74a6c4299c657c04d4e9fba03875c1f0eec569af03aed89" dependencies = [ "const_format", - "git2", "is_debug", "time", - "tzdb", ] [[package]] @@ -6835,35 +6790,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" -[[package]] -name = "tz-rs" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" -dependencies = [ - "const_fn", -] - -[[package]] -name = "tzdb" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b580f6b365fa89f5767cdb619a55d534d04a4e14c2d7e5b9a31e94598687fb1" -dependencies = [ - "iana-time-zone", - "tz-rs", - "tzdb_data", -] - -[[package]] -name = "tzdb_data" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4471adcfcbd3052e8c5b5890a04a559837444b3be26b9cbbd622063171cec9d" -dependencies = [ - "tz-rs", -] - [[package]] name = "uds_windows" version = "1.1.0" @@ -6986,12 +6912,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version-compare" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index c2231361..391c9338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ s3s = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fb "tower", ] } s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fbf7c4c2ab155db37dfe" } +shadow-rs = { version = "0.38.0", default-features = false } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.138" tempfile = "3.16.0" diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7c50d0f2..5b3b89f4 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -61,7 +61,7 @@ url.workspace = true admin = { path = "../api/admin" } axum.workspace = true matchit = "0.8.6" -shadow-rs = "0.38.0" +shadow-rs.workspace = true const-str = { version = "0.6.1", features = ["std", "proc"] } atoi = "2.0.0" serde_urlencoded = "0.7.1" @@ -90,5 +90,5 @@ hyper-util = { version = "0.1.10", features = [ ] } transform-stream = "0.3.1" netif = "0.1.6" -shadow-rs = "0.38.0" +shadow-rs.workspace = true # pin-utils = "0.1.0" diff --git a/scripts/build.py b/scripts/build.py new file mode 100755 index 00000000..5d6857a5 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +from dataclasses import dataclass +import argparse +import subprocess +from pathlib import Path + + +@dataclass +class CliArgs: + profile: str + target: str + glibc: str + + @staticmethod + def parse(): + parser = argparse.ArgumentParser() + parser.add_argument("--profile", type=str, required=True) + parser.add_argument("--target", type=str, required=True) + parser.add_argument("--glibc", type=str, required=True) + args = parser.parse_args() + return CliArgs(args.profile, args.target, args.glibc) + + +def shell(cmd: str): + print(cmd, flush=True) + subprocess.run(cmd, shell=True, check=True) + + +def main(args: CliArgs): + use_zigbuild = False + use_old_glibc = False + + if args.glibc and args.glibc != "default": + use_zigbuild = True + use_old_glibc = True + + cmd = ["cargo", "build"] + if use_zigbuild: + cmd = ["cargo", " zigbuild"] + + cmd.extend(["--profile", args.profile]) + + if use_old_glibc: + cmd.extend(["--target", f"{args.target}.{args.glibc}"]) + else: + cmd.extend(["--target", args.target]) + + cmd.extend(["-p", "rustfs"]) + cmd.extend(["--bins"]) + + shell("touch rustfs/build.rs") # refresh build info for rustfs + shell(" ".join(cmd)) + + if args.profile == "dev": + profile_dir = "debug" + elif args.profile == "release": + profile_dir = "release" + else: + profile_dir = args.profile + + bin_path = Path(f"target/{args.target}/{profile_dir}/rustfs") + + bin_name = f"rustfs.{args.profile}.{args.target}" + if use_old_glibc: + bin_name += f".glibc{args.glibc}" + bin_name += ".bin" + + out_path = Path(f"target/artifacts/{bin_name}") + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.hardlink_to(bin_path) + + +if __name__ == "__main__": + main(CliArgs.parse())