wip

.github/workflows/docker.yml

@@ -48,11 +48,13 @@ jobs:
           - target: x86_64-unknown-linux-musl
             os: ubuntu-latest
             arch: amd64
-          - target: aarch64-unknown-linux-musl
+            use_cross: false
+          - target: aarch64-unknown-linux-gnu
             os: ubuntu-latest
             arch: arm64
+            use_cross: true
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
+    timeout-minutes: 120
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -63,13 +65,17 @@ jobs:
           target: ${{ matrix.target }}
           components: rustfmt, clippy
 
-      - name: Install cross-compilation dependencies
+      - name: Install cross-compilation dependencies (native build)
+        if: matrix.use_cross == false
         run: |
           sudo apt-get update
           sudo apt-get install -y musl-tools
-          if [ "${{ matrix.target }}" = "aarch64-unknown-linux-musl" ]; then
+
-            sudo apt-get install -y gcc-aarch64-linux-gnu
+      - name: Install cross tool (cross compilation)
-          fi
+        if: matrix.use_cross == true
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cross
 
       - name: Install protoc
         uses: arduino/setup-protoc@v3
@@ -94,11 +100,19 @@ jobs:
             ${{ runner.os }}-cargo-${{ matrix.target }}-
             ${{ runner.os }}-cargo-
 
-      - name: Build RustFS binary
+      - name: Generate protobuf code
+        run: cargo run --bin gproto
+
+      - name: Build RustFS binary (native)
+        if: matrix.use_cross == false
         run: |
-          export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc
           cargo build --release --target ${{ matrix.target }} --bin rustfs
 
+      - name: Build RustFS binary (cross)
+        if: matrix.use_cross == true
+        run: |
+          cross build --release --target ${{ matrix.target }} --bin rustfs
+
       - name: Upload binary artifact
         uses: actions/upload-artifact@v4
         with:
@@ -128,9 +142,9 @@ jobs:
       - name: Setup binary files
         run: |
           mkdir -p target/x86_64-unknown-linux-musl/release
-          mkdir -p target/aarch64-unknown-linux-musl/release
+          mkdir -p target/aarch64-unknown-linux-gnu/release
           cp artifacts/rustfs-amd64/rustfs target/x86_64-unknown-linux-musl/release/
-          cp artifacts/rustfs-arm64/rustfs target/aarch64-unknown-linux-musl/release/
+          cp artifacts/rustfs-arm64/rustfs target/aarch64-unknown-linux-gnu/release/
           chmod +x target/*/release/rustfs
 
       - name: Set up Docker Buildx

Dockerfile

@@ -1,17 +1,37 @@
 FROM alpine:latest
 
-# RUN apk add --no-cache <package-name>
+# Install runtime dependencies
+RUN apk add --no-cache \
+    ca-certificates \
+    tzdata \
+    && rm -rf /var/cache/apk/*
+
+# Create rustfs user and group
+RUN addgroup -g 1000 rustfs && \
+    adduser -D -s /bin/sh -u 1000 -G rustfs rustfs
 
 WORKDIR /app
 
-RUN mkdir -p /data/rustfs0  /data/rustfs1  /data/rustfs2  /data/rustfs3 
+# Create data directories
+RUN mkdir -p /data/rustfs{0,1,2,3} && \
+    chown -R rustfs:rustfs /data /app
 
-COPY ./target/x86_64-unknown-linux-musl/release/rustfs /app/rustfs
+# Copy binary based on target architecture
+COPY --chown=rustfs:rustfs \
+  target/*/release/rustfs \
+  /app/rustfs
 
 RUN chmod +x /app/rustfs
 
-EXPOSE 9000
+# Switch to non-root user
-EXPOSE 9001
+USER rustfs
 
+# Expose ports
+EXPOSE 9000 9001
 
-CMD ["/app/rustfs"]
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:9000/health || exit 1
+
+# Set default command
+CMD ["/app/rustfs"]

Dockerfile.multi-stage

@@ -16,7 +16,6 @@ RUN apt-get update && apt-get install -y \
     pkg-config \
     libssl-dev \
     lld \
-    musl-tools \
     && rm -rf /var/lib/apt/lists/*
 
 # Install cross-compilation tools for ARM64
@@ -39,13 +38,15 @@ RUN wget https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.
 
 # Set up Rust targets based on platform
 RUN case "$TARGETPLATFORM" in \
-        "linux/amd64") rustup target add x86_64-unknown-linux-musl ;; \
+        "linux/amd64") rustup target add x86_64-unknown-linux-gnu ;; \
-        "linux/arm64") rustup target add aarch64-unknown-linux-musl ;; \
+        "linux/arm64") rustup target add aarch64-unknown-linux-gnu ;; \
         *) echo "Unsupported platform: $TARGETPLATFORM" && exit 1 ;; \
     esac
 
 # Set up environment for cross-compilation
-ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
+ENV CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
+ENV CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
 
 WORKDIR /usr/src/rustfs
 
@@ -60,8 +61,8 @@ RUN find . -name "Cargo.toml" -not -path "./Cargo.toml" | \
 
 # Build dependencies only (cache layer)
 RUN case "$TARGETPLATFORM" in \
-        "linux/amd64") cargo build --release --target x86_64-unknown-linux-musl ;; \
+        "linux/amd64") cargo build --release --target x86_64-unknown-linux-gnu ;; \
-        "linux/arm64") cargo build --release --target aarch64-unknown-linux-musl ;; \
+        "linux/arm64") cargo build --release --target aarch64-unknown-linux-gnu ;; \
     esac
 
 # Copy source code
@@ -73,27 +74,28 @@ RUN cargo run --bin gproto
 # Build the actual application
 RUN case "$TARGETPLATFORM" in \
         "linux/amd64") \
-            cargo build --release --target x86_64-unknown-linux-musl --bin rustfs && \
+            cargo build --release --target x86_64-unknown-linux-gnu --bin rustfs && \
-            cp target/x86_64-unknown-linux-musl/release/rustfs /usr/local/bin/rustfs \
+            cp target/x86_64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
             ;; \
         "linux/arm64") \
-            cargo build --release --target aarch64-unknown-linux-musl --bin rustfs && \
+            cargo build --release --target aarch64-unknown-linux-gnu --bin rustfs && \
-            cp target/aarch64-unknown-linux-musl/release/rustfs /usr/local/bin/rustfs \
+            cp target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
             ;; \
     esac
 
-# Runtime stage - minimal Alpine image
+# Runtime stage - Ubuntu minimal for better compatibility
-FROM alpine:latest
+FROM ubuntu:22.04
 
 # Install runtime dependencies
-RUN apk add --no-cache \
+RUN apt-get update && apt-get install -y \
     ca-certificates \
     tzdata \
-    && rm -rf /var/cache/apk/*
+    wget \
+    && rm -rf /var/lib/apt/lists/*
 
 # Create rustfs user and group
-RUN addgroup -g 1000 rustfs && \
+RUN groupadd -g 1000 rustfs && \
-    adduser -D -s /bin/sh -u 1000 -G rustfs rustfs
+    useradd -d /app -g rustfs -u 1000 -s /bin/bash rustfs
 
 WORKDIR /app
 
@@ -103,7 +105,7 @@ RUN mkdir -p /data/rustfs{0,1,2,3} && \
 
 # Copy binary from builder stage
 COPY --from=builder /usr/local/bin/rustfs /app/rustfs
-RUN chmod +x /app/rustfs
+RUN chmod +x /app/rustfs && chown rustfs:rustfs /app/rustfs
 
 # Switch to non-root user
 USER rustfs

docs/docker-build.md

@@ -49,7 +49,7 @@ Our GitHub Actions workflow builds multiple image variants:
 
 | Variant | Tag Suffix | Description | Use Case |
 |---------|------------|-------------|----------|
-| Production | *(none)* | Minimal Alpine-based runtime | Production deployment |
+| Production | *(none)* | Minimal Ubuntu-based runtime | Production deployment |
 | Ubuntu | `-ubuntu22.04` | Ubuntu 22.04 based build environment | Development/Testing |
 | Rocky Linux | `-rockylinux9.3` | Rocky Linux 9.3 based build environment | Enterprise environments |
 | Development | `-devenv` | Full development environment | Development/Debugging |
@@ -57,8 +57,8 @@ Our GitHub Actions workflow builds multiple image variants:
 ### Supported Architectures
 
 All images support multi-architecture:
-- `linux/amd64` (x86_64)
+- `linux/amd64` (x86_64-unknown-linux-musl)
-- `linux/arm64` (aarch64)
+- `linux/arm64` (aarch64-unknown-linux-gnu)
 
 ### Tag Examples
 
@@ -86,6 +86,15 @@ The Docker build workflow (`.github/workflows/docker.yml`) automatically:
 4. **Creates multi-arch manifests** for seamless platform selection
 5. **Performs security scanning** using Trivy
 
+### Cross-Compilation Strategy
+
+To handle complex native dependencies, we use different compilation strategies:
+
+- **x86_64**: Native compilation with `x86_64-unknown-linux-musl` for static linking
+- **aarch64**: Cross-compilation with `aarch64-unknown-linux-gnu` using the `cross` tool
+
+This approach ensures compatibility with various C libraries while maintaining performance.
+
 ### Workflow Triggers
 
 - **Push to main branch**: Builds and pushes `main` and `latest` tags
@@ -111,11 +120,37 @@ GITHUB_TOKEN=automatically-provided
 ### Prerequisites
 
 - Docker with BuildKit enabled
-- Docker Compose (optional)
+- Rust toolchain (1.85+)
+- Protocol Buffers compiler (protoc 31.1+)
+- FlatBuffers compiler (flatc 25.2.10+)
+- `cross` tool for ARM64 compilation
+
+### Installation Commands
+
+```bash
+# Install Rust targets
+rustup target add x86_64-unknown-linux-musl
+rustup target add aarch64-unknown-linux-gnu
+
+# Install cross for ARM64 compilation
+cargo install cross --git https://github.com/cross-rs/cross
+
+# Install protoc (macOS)
+brew install protobuf
+
+# Install protoc (Ubuntu)
+sudo apt-get install protobuf-compiler
+
+# Install flatc
+# Download from: https://github.com/google/flatbuffers/releases
+```
 
 ### Build Commands
 
 ```bash
+# Test cross-compilation setup
+./scripts/test-cross-build.sh
+
 # Build production image for local platform
 docker build -t rustfs:local .
 
@@ -133,6 +168,19 @@ docker build --platform linux/arm64 -t rustfs:arm64 .
 docker buildx build --platform linux/amd64,linux/arm64 -t rustfs:multi .
 ```
 
+### Cross-Compilation
+
+```bash
+# Generate protobuf code first
+cargo run --bin gproto
+
+# Native x86_64 build
+cargo build --release --target x86_64-unknown-linux-musl --bin rustfs
+
+# Cross-compile for ARM64
+cross build --release --target aarch64-unknown-linux-gnu --bin rustfs
+```
+
 ### Build with Docker Compose
 
 ```bash
@@ -316,6 +364,18 @@ docker run --rm \
   cargo build --release --bin rustfs
 ```
 
+### Testing Cross-Compilation
+
+```bash
+# Run the test script to verify cross-compilation setup
+./scripts/test-cross-build.sh
+
+# This will test:
+# - x86_64-unknown-linux-musl compilation
+# - aarch64-unknown-linux-gnu cross-compilation
+# - Docker builds for both architectures
+```
+
 ## 🔐 Security
 
 ### Security Scanning
@@ -332,7 +392,7 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
 ### Security Best Practices
 
 1. **Use non-root user**: Images run as `rustfs` user (UID 1000)
-2. **Minimal base images**: Alpine Linux for production
+2. **Minimal base images**: Ubuntu minimal for production
 3. **Security updates**: Regular base image updates
 4. **Secret management**: Use Docker secrets or environment files
 5. **Network security**: Use Docker networks and proper firewall rules
@@ -341,10 +401,50 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
 
 ### Common Issues
 
-1. **Build failures**: Check build logs and ensure all dependencies are installed
+#### 1. Cross-Compilation Failures
-2. **Permission issues**: Ensure proper volume permissions for UID 1000
+
-3. **Network connectivity**: Verify port mappings and network configuration
+**Problem**: ARM64 build fails with linking errors
-4. **Resource limits**: Ensure sufficient memory and CPU for compilation
+```bash
+error: linking with `aarch64-linux-gnu-gcc` failed
+```
+
+**Solution**: Use the `cross` tool instead of native cross-compilation:
+```bash
+# Install cross tool
+cargo install cross --git https://github.com/cross-rs/cross
+
+# Use cross for ARM64 builds
+cross build --release --target aarch64-unknown-linux-gnu --bin rustfs
+```
+
+#### 2. Protobuf Generation Issues
+
+**Problem**: Missing protobuf definitions
+```bash
+error: failed to run custom build command for `protos`
+```
+
+**Solution**: Generate protobuf code first:
+```bash
+cargo run --bin gproto
+```
+
+#### 3. Docker Build Failures
+
+**Problem**: Binary not found in Docker build
+```bash
+COPY failed: file not found in build context
+```
+
+**Solution**: Ensure binaries are built before Docker build:
+```bash
+# Build binaries first
+cargo build --release --target x86_64-unknown-linux-musl --bin rustfs
+cross build --release --target aarch64-unknown-linux-gnu --bin rustfs
+
+# Then build Docker image
+docker build .
+```
 
 ### Debug Commands
 
@@ -356,13 +456,16 @@ docker ps -a
 docker logs rustfs --tail 100
 
 # Access container shell
-docker exec -it rustfs sh
+docker exec -it rustfs bash
 
 # Check resource usage
 docker stats rustfs
 
 # Inspect container configuration
 docker inspect rustfs
+
+# Test cross-compilation setup
+./scripts/test-cross-build.sh
 ```
 
 ## 🔄 CI/CD Integration
@@ -422,5 +525,6 @@ pipeline {
 - [Docker Official Documentation](https://docs.docker.com/)
 - [Docker Compose Reference](https://docs.docker.com/compose/)
 - [GitHub Actions Documentation](https://docs.github.com/en/actions)
+- [Cross-compilation with Rust](https://rust-lang.github.io/rustup/cross-compilation.html)
+- [Cross tool documentation](https://github.com/cross-rs/cross)
 - [RustFS Configuration Guide](../README.md)
-- [Kubernetes Deployment Guide](./kubernetes.md)