diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 964c45e5..7761d2d0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,11 +48,13 @@ jobs: - target: x86_64-unknown-linux-musl os: ubuntu-latest arch: amd64 - - target: aarch64-unknown-linux-musl + use_cross: false + - target: aarch64-unknown-linux-gnu os: ubuntu-latest arch: arm64 + use_cross: true runs-on: ${{ matrix.os }} - timeout-minutes: 60 + timeout-minutes: 120 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -63,13 +65,17 @@ jobs: target: ${{ matrix.target }} components: rustfmt, clippy - - name: Install cross-compilation dependencies + - name: Install cross-compilation dependencies (native build) + if: matrix.use_cross == false run: | sudo apt-get update sudo apt-get install -y musl-tools - if [ "${{ matrix.target }}" = "aarch64-unknown-linux-musl" ]; then - sudo apt-get install -y gcc-aarch64-linux-gnu - fi + + - name: Install cross tool (cross compilation) + if: matrix.use_cross == true + uses: taiki-e/install-action@v2 + with: + tool: cross - name: Install protoc uses: arduino/setup-protoc@v3 @@ -94,11 +100,19 @@ jobs: ${{ runner.os }}-cargo-${{ matrix.target }}- ${{ runner.os }}-cargo- - - name: Build RustFS binary + - name: Generate protobuf code + run: cargo run --bin gproto + + - name: Build RustFS binary (native) + if: matrix.use_cross == false run: | - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc cargo build --release --target ${{ matrix.target }} --bin rustfs + - name: Build RustFS binary (cross) + if: matrix.use_cross == true + run: | + cross build --release --target ${{ matrix.target }} --bin rustfs + - name: Upload binary artifact uses: actions/upload-artifact@v4 with: @@ -128,9 +142,9 @@ jobs: - name: Setup binary files run: | mkdir -p target/x86_64-unknown-linux-musl/release - mkdir -p target/aarch64-unknown-linux-musl/release + mkdir -p target/aarch64-unknown-linux-gnu/release cp artifacts/rustfs-amd64/rustfs target/x86_64-unknown-linux-musl/release/ - cp artifacts/rustfs-arm64/rustfs target/aarch64-unknown-linux-musl/release/ + cp artifacts/rustfs-arm64/rustfs target/aarch64-unknown-linux-gnu/release/ chmod +x target/*/release/rustfs - name: Set up Docker Buildx diff --git a/Dockerfile b/Dockerfile index 035a2c08..555ab559 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,37 @@ FROM alpine:latest -# RUN apk add --no-cache +# Install runtime dependencies +RUN apk add --no-cache \ + ca-certificates \ + tzdata \ + && rm -rf /var/cache/apk/* + +# Create rustfs user and group +RUN addgroup -g 1000 rustfs && \ + adduser -D -s /bin/sh -u 1000 -G rustfs rustfs WORKDIR /app -RUN mkdir -p /data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3 +# Create data directories +RUN mkdir -p /data/rustfs{0,1,2,3} && \ + chown -R rustfs:rustfs /data /app -COPY ./target/x86_64-unknown-linux-musl/release/rustfs /app/rustfs +# Copy binary based on target architecture +COPY --chown=rustfs:rustfs \ + target/*/release/rustfs \ + /app/rustfs RUN chmod +x /app/rustfs -EXPOSE 9000 -EXPOSE 9001 +# Switch to non-root user +USER rustfs +# Expose ports +EXPOSE 9000 9001 -CMD ["/app/rustfs"] \ No newline at end of file +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:9000/health || exit 1 + +# Set default command +CMD ["/app/rustfs"] diff --git a/Dockerfile.multi-stage b/Dockerfile.multi-stage index 63a7f1d3..24b1f616 100644 --- a/Dockerfile.multi-stage +++ b/Dockerfile.multi-stage @@ -16,7 +16,6 @@ RUN apt-get update && apt-get install -y \ pkg-config \ libssl-dev \ lld \ - musl-tools \ && rm -rf /var/lib/apt/lists/* # Install cross-compilation tools for ARM64 @@ -39,13 +38,15 @@ RUN wget https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux. # Set up Rust targets based on platform RUN case "$TARGETPLATFORM" in \ - "linux/amd64") rustup target add x86_64-unknown-linux-musl ;; \ - "linux/arm64") rustup target add aarch64-unknown-linux-musl ;; \ + "linux/amd64") rustup target add x86_64-unknown-linux-gnu ;; \ + "linux/arm64") rustup target add aarch64-unknown-linux-gnu ;; \ *) echo "Unsupported platform: $TARGETPLATFORM" && exit 1 ;; \ esac # Set up environment for cross-compilation -ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc +ENV CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc +ENV CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ WORKDIR /usr/src/rustfs @@ -60,8 +61,8 @@ RUN find . -name "Cargo.toml" -not -path "./Cargo.toml" | \ # Build dependencies only (cache layer) RUN case "$TARGETPLATFORM" in \ - "linux/amd64") cargo build --release --target x86_64-unknown-linux-musl ;; \ - "linux/arm64") cargo build --release --target aarch64-unknown-linux-musl ;; \ + "linux/amd64") cargo build --release --target x86_64-unknown-linux-gnu ;; \ + "linux/arm64") cargo build --release --target aarch64-unknown-linux-gnu ;; \ esac # Copy source code @@ -73,27 +74,28 @@ RUN cargo run --bin gproto # Build the actual application RUN case "$TARGETPLATFORM" in \ "linux/amd64") \ - cargo build --release --target x86_64-unknown-linux-musl --bin rustfs && \ - cp target/x86_64-unknown-linux-musl/release/rustfs /usr/local/bin/rustfs \ + cargo build --release --target x86_64-unknown-linux-gnu --bin rustfs && \ + cp target/x86_64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \ ;; \ "linux/arm64") \ - cargo build --release --target aarch64-unknown-linux-musl --bin rustfs && \ - cp target/aarch64-unknown-linux-musl/release/rustfs /usr/local/bin/rustfs \ + cargo build --release --target aarch64-unknown-linux-gnu --bin rustfs && \ + cp target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \ ;; \ esac -# Runtime stage - minimal Alpine image -FROM alpine:latest +# Runtime stage - Ubuntu minimal for better compatibility +FROM ubuntu:22.04 # Install runtime dependencies -RUN apk add --no-cache \ +RUN apt-get update && apt-get install -y \ ca-certificates \ tzdata \ - && rm -rf /var/cache/apk/* + wget \ + && rm -rf /var/lib/apt/lists/* # Create rustfs user and group -RUN addgroup -g 1000 rustfs && \ - adduser -D -s /bin/sh -u 1000 -G rustfs rustfs +RUN groupadd -g 1000 rustfs && \ + useradd -d /app -g rustfs -u 1000 -s /bin/bash rustfs WORKDIR /app @@ -103,7 +105,7 @@ RUN mkdir -p /data/rustfs{0,1,2,3} && \ # Copy binary from builder stage COPY --from=builder /usr/local/bin/rustfs /app/rustfs -RUN chmod +x /app/rustfs +RUN chmod +x /app/rustfs && chown rustfs:rustfs /app/rustfs # Switch to non-root user USER rustfs diff --git a/docs/docker-build.md b/docs/docker-build.md index 47262176..ce016ff9 100644 --- a/docs/docker-build.md +++ b/docs/docker-build.md @@ -49,7 +49,7 @@ Our GitHub Actions workflow builds multiple image variants: | Variant | Tag Suffix | Description | Use Case | |---------|------------|-------------|----------| -| Production | *(none)* | Minimal Alpine-based runtime | Production deployment | +| Production | *(none)* | Minimal Ubuntu-based runtime | Production deployment | | Ubuntu | `-ubuntu22.04` | Ubuntu 22.04 based build environment | Development/Testing | | Rocky Linux | `-rockylinux9.3` | Rocky Linux 9.3 based build environment | Enterprise environments | | Development | `-devenv` | Full development environment | Development/Debugging | @@ -57,8 +57,8 @@ Our GitHub Actions workflow builds multiple image variants: ### Supported Architectures All images support multi-architecture: -- `linux/amd64` (x86_64) -- `linux/arm64` (aarch64) +- `linux/amd64` (x86_64-unknown-linux-musl) +- `linux/arm64` (aarch64-unknown-linux-gnu) ### Tag Examples @@ -86,6 +86,15 @@ The Docker build workflow (`.github/workflows/docker.yml`) automatically: 4. **Creates multi-arch manifests** for seamless platform selection 5. **Performs security scanning** using Trivy +### Cross-Compilation Strategy + +To handle complex native dependencies, we use different compilation strategies: + +- **x86_64**: Native compilation with `x86_64-unknown-linux-musl` for static linking +- **aarch64**: Cross-compilation with `aarch64-unknown-linux-gnu` using the `cross` tool + +This approach ensures compatibility with various C libraries while maintaining performance. + ### Workflow Triggers - **Push to main branch**: Builds and pushes `main` and `latest` tags @@ -111,11 +120,37 @@ GITHUB_TOKEN=automatically-provided ### Prerequisites - Docker with BuildKit enabled -- Docker Compose (optional) +- Rust toolchain (1.85+) +- Protocol Buffers compiler (protoc 31.1+) +- FlatBuffers compiler (flatc 25.2.10+) +- `cross` tool for ARM64 compilation + +### Installation Commands + +```bash +# Install Rust targets +rustup target add x86_64-unknown-linux-musl +rustup target add aarch64-unknown-linux-gnu + +# Install cross for ARM64 compilation +cargo install cross --git https://github.com/cross-rs/cross + +# Install protoc (macOS) +brew install protobuf + +# Install protoc (Ubuntu) +sudo apt-get install protobuf-compiler + +# Install flatc +# Download from: https://github.com/google/flatbuffers/releases +``` ### Build Commands ```bash +# Test cross-compilation setup +./scripts/test-cross-build.sh + # Build production image for local platform docker build -t rustfs:local . @@ -133,6 +168,19 @@ docker build --platform linux/arm64 -t rustfs:arm64 . docker buildx build --platform linux/amd64,linux/arm64 -t rustfs:multi . ``` +### Cross-Compilation + +```bash +# Generate protobuf code first +cargo run --bin gproto + +# Native x86_64 build +cargo build --release --target x86_64-unknown-linux-musl --bin rustfs + +# Cross-compile for ARM64 +cross build --release --target aarch64-unknown-linux-gnu --bin rustfs +``` + ### Build with Docker Compose ```bash @@ -316,6 +364,18 @@ docker run --rm \ cargo build --release --bin rustfs ``` +### Testing Cross-Compilation + +```bash +# Run the test script to verify cross-compilation setup +./scripts/test-cross-build.sh + +# This will test: +# - x86_64-unknown-linux-musl compilation +# - aarch64-unknown-linux-gnu cross-compilation +# - Docker builds for both architectures +``` + ## 🔐 Security ### Security Scanning @@ -332,7 +392,7 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ ### Security Best Practices 1. **Use non-root user**: Images run as `rustfs` user (UID 1000) -2. **Minimal base images**: Alpine Linux for production +2. **Minimal base images**: Ubuntu minimal for production 3. **Security updates**: Regular base image updates 4. **Secret management**: Use Docker secrets or environment files 5. **Network security**: Use Docker networks and proper firewall rules @@ -341,10 +401,50 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ ### Common Issues -1. **Build failures**: Check build logs and ensure all dependencies are installed -2. **Permission issues**: Ensure proper volume permissions for UID 1000 -3. **Network connectivity**: Verify port mappings and network configuration -4. **Resource limits**: Ensure sufficient memory and CPU for compilation +#### 1. Cross-Compilation Failures + +**Problem**: ARM64 build fails with linking errors +```bash +error: linking with `aarch64-linux-gnu-gcc` failed +``` + +**Solution**: Use the `cross` tool instead of native cross-compilation: +```bash +# Install cross tool +cargo install cross --git https://github.com/cross-rs/cross + +# Use cross for ARM64 builds +cross build --release --target aarch64-unknown-linux-gnu --bin rustfs +``` + +#### 2. Protobuf Generation Issues + +**Problem**: Missing protobuf definitions +```bash +error: failed to run custom build command for `protos` +``` + +**Solution**: Generate protobuf code first: +```bash +cargo run --bin gproto +``` + +#### 3. Docker Build Failures + +**Problem**: Binary not found in Docker build +```bash +COPY failed: file not found in build context +``` + +**Solution**: Ensure binaries are built before Docker build: +```bash +# Build binaries first +cargo build --release --target x86_64-unknown-linux-musl --bin rustfs +cross build --release --target aarch64-unknown-linux-gnu --bin rustfs + +# Then build Docker image +docker build . +``` ### Debug Commands @@ -356,13 +456,16 @@ docker ps -a docker logs rustfs --tail 100 # Access container shell -docker exec -it rustfs sh +docker exec -it rustfs bash # Check resource usage docker stats rustfs # Inspect container configuration docker inspect rustfs + +# Test cross-compilation setup +./scripts/test-cross-build.sh ``` ## 🔄 CI/CD Integration @@ -422,5 +525,6 @@ pipeline { - [Docker Official Documentation](https://docs.docker.com/) - [Docker Compose Reference](https://docs.docker.com/compose/) - [GitHub Actions Documentation](https://docs.github.com/en/actions) +- [Cross-compilation with Rust](https://rust-lang.github.io/rustup/cross-compilation.html) +- [Cross tool documentation](https://github.com/cross-rs/cross) - [RustFS Configuration Guide](../README.md) -- [Kubernetes Deployment Guide](./kubernetes.md)