From af693f7b3f8f6bdc64e94aed80bced8375b2d60a Mon Sep 17 00:00:00 2001 From: overtrue Date: Thu, 17 Jul 2025 04:19:20 +0800 Subject: [PATCH] refactor: restructure Docker build pipeline to depend on binary builds - Change docker.yml to use workflow_call triggered by build.yml - Remove redundant force_build parameter from build.yml - Simplify build_docker parameter (build implies push in CI/CD) - Add proper dependency chain: build.yml -> docker.yml -> registry - Update documentation to reflect new architecture - Mark Dockerfile.source as local development only --- .github/workflows/build.yml | 42 +++++- .github/workflows/docker.yml | 261 +++++++++++++---------------------- Dockerfile.source | 11 +- docker-buildx.sh | 2 + 4 files changed, 145 insertions(+), 171 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 231fbe63..f3e5d45e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,6 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Build and Release Workflow +# +# This workflow builds RustFS binaries and automatically triggers Docker image builds. +# +# Flow: +# 1. Build binaries for multiple platforms +# 2. Upload binaries to OSS storage +# 3. Trigger docker.yml to build and push images using the uploaded binaries +# +# Manual Parameters: +# - build_docker: Build and push Docker images (default: true) + name: Build and Release on: @@ -52,10 +64,10 @@ on: - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC workflow_dispatch: inputs: - force_build: - description: "Force build even without changes" + build_docker: + description: "Build and push Docker images after binary build" required: false - default: false + default: true type: boolean env: @@ -117,7 +129,6 @@ jobs: echo "๐Ÿ› ๏ธ Development build detected" elif [[ "${{ github.event_name }}" == "schedule" ]] || \ [[ "${{ github.event_name }}" == "workflow_dispatch" ]] || \ - [[ "${{ github.event.inputs.force_build }}" == "true" ]] || \ [[ "${{ contains(github.event.head_commit.message, '--build') }}" == "true" ]]; then # Scheduled or manual build should_build=true @@ -461,3 +472,26 @@ jobs: echo "๐Ÿท๏ธ GitHub Release will be created automatically by the release workflow" ;; esac + + echo "" + echo "๐Ÿณ Docker Images:" + if [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then + echo "โญ๏ธ Docker image build was skipped (binary only build)" + else + echo "๐Ÿ”„ Docker images will be built and pushed automatically using the binary artifacts" + fi + + # Call Docker workflow after successful build + call-docker-build: + name: Build And Push Docker Images + needs: [build-check, build-rustfs] + if: needs.build-check.outputs.should_build == 'true' && github.event.inputs.build_docker != 'false' + uses: ./.github/workflows/docker.yml + with: + build_type: ${{ needs.build-check.outputs.build_type }} + version: ${{ needs.build-check.outputs.version }} + short_sha: ${{ needs.build-check.outputs.short_sha }} + is_prerelease: ${{ needs.build-check.outputs.is_prerelease }} + push_images: true + force_rebuild: false + secrets: inherit diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cfec5faf..24612922 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,42 +12,50 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Docker Images Workflow +# +# This workflow builds Docker images using pre-built binaries from the build workflow. +# +# Trigger Types: +# 1. workflow_call: Called by build.yml after binary build completes (primary use) +# 2. workflow_dispatch: Manual trigger for standalone Docker builds +# +# Architecture: +# build.yml (binaries) -> docker.yml (images) -> registry + name: Docker Images on: - push: - tags: ["*.*.*"] - branches: [main] - paths-ignore: - - "**.md" - - "**.txt" - - ".github/**" - - "docs/**" - - "deploy/**" - - "scripts/dev_*.sh" - - "LICENSE*" - - "README*" - - "**/*.png" - - "**/*.jpg" - - "**/*.svg" - - ".gitignore" - - ".dockerignore" - pull_request: - branches: [main] - paths-ignore: - - "**.md" - - "**.txt" - - ".github/**" - - "docs/**" - - "deploy/**" - - "scripts/dev_*.sh" - - "LICENSE*" - - "README*" - - "**/*.png" - - "**/*.jpg" - - "**/*.svg" - - ".gitignore" - - ".dockerignore" + # Called by build.yml after binary build completes + workflow_call: + inputs: + build_type: + description: "Build type: development, release, or prerelease" + required: true + type: string + version: + description: "Version to build" + required: true + type: string + short_sha: + description: "Short commit SHA" + required: true + type: string + is_prerelease: + description: "Whether this is a prerelease" + required: true + type: boolean + push_images: + description: "Push images to registries" + required: false + default: true + type: boolean + force_rebuild: + description: "Force rebuild even if binary exists" + required: false + default: false + type: boolean + # Manual trigger with same parameters as build.yml for consistency workflow_dispatch: inputs: push_images: @@ -71,6 +79,7 @@ env: CARGO_TERM_COLOR: always REGISTRY_DOCKERHUB: rustfs/rustfs REGISTRY_GHCR: ghcr.io/${{ github.repository }} + DOCKER_PLATFORMS: linux/amd64,linux/arm64 jobs: # Docker build strategy check @@ -94,7 +103,7 @@ jobs: - name: Check build conditions id: check run: | - should_build=false + should_build=true # Always build when called should_push=false build_type="none" version="" @@ -102,26 +111,39 @@ jobs: is_prerelease=false create_latest=false - # Get short SHA for all builds - short_sha=$(git rev-parse --short HEAD) + if [[ "${{ github.event_name }}" == "workflow_call" ]]; then + # Called by build.yml - use provided parameters + build_type="${{ inputs.build_type }}" + version="${{ inputs.version }}" + short_sha="${{ inputs.short_sha }}" + is_prerelease="${{ inputs.is_prerelease }}" + should_push="${{ inputs.push_images }}" - # Always build on workflow_dispatch or when changes detected - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] || \ - [[ "${{ github.event_name }}" == "push" ]] || \ - [[ "${{ github.event_name }}" == "pull_request" ]]; then - should_build=true - fi + # Determine create_latest based on build_type + if [[ "$build_type" == "release" ]] && [[ "$is_prerelease" == "false" ]]; then + create_latest=true + fi - # Determine build type and version - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ -n "${{ github.event.inputs.version }}" ]]; then + echo "๐Ÿ”— Called by build workflow:" + echo " ๐Ÿ“‹ Build type: $build_type" + echo " ๐Ÿ”ข Version: $version" + echo " ๐Ÿ“Ž Short SHA: $short_sha" + echo " ๐Ÿงช Is prerelease: $is_prerelease" + echo " ๐Ÿš€ Push images: $should_push" + + elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then # Manual trigger with version input input_version="${{ github.event.inputs.version }}" version="${input_version}" - force_rebuild="${{ github.event.inputs.force_rebuild }}" + should_push="${{ github.event.inputs.push_images }}" + + # Get short SHA + short_sha=$(git rev-parse --short HEAD) echo "๐ŸŽฏ Manual Docker build triggered:" echo " ๐Ÿ“‹ Requested version: $input_version" - echo " ๐Ÿ”ง Force rebuild: $force_rebuild" + echo " ๐Ÿ”ง Force rebuild: ${{ github.event.inputs.force_rebuild }}" + echo " ๐Ÿš€ Push images: $should_push" case "$input_version" in "latest") @@ -159,38 +181,6 @@ jobs: echo "โš ๏ธ Warning: Custom version format may not follow standard patterns" ;; esac - elif [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then - # Tag push - release or prerelease - tag_name="${GITHUB_REF#refs/tags/}" - version="${tag_name}" - - # Check if this is a prerelease - if [[ "$tag_name" == *"alpha"* ]] || [[ "$tag_name" == *"beta"* ]] || [[ "$tag_name" == *"rc"* ]]; then - build_type="prerelease" - is_prerelease=true - echo "๐Ÿš€ Docker prerelease build detected: $tag_name" - else - build_type="release" - create_latest=true - echo "๐Ÿ“ฆ Docker release build detected: $tag_name" - fi - elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - # Main branch push - development build - build_type="development" - version="dev-${short_sha}" - echo "๐Ÿ› ๏ธ Docker development build detected" - else - # Other branches - development build - build_type="development" - version="dev-${short_sha}" - echo "๐Ÿ”ง Docker development build detected" - fi - - # Push only on main branch, tags, or manual trigger - if [[ "${{ github.ref }}" == "refs/heads/main" ]] || \ - [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]] || \ - [[ "${{ github.event.inputs.push_images }}" == "true" ]]; then - should_push=true fi echo "should_build=$should_build" >> $GITHUB_OUTPUT @@ -211,25 +201,15 @@ jobs: echo " - Create latest: $create_latest" # Build multi-arch Docker images + # Strategy: Build images using pre-built binaries from dl.rustfs.com + # Supports both release and dev channel binaries based on build context + # Note: Dockerfile.source is available for local development but not used in CI/CD build-docker: name: Build Docker Images needs: build-check if: needs.build-check.outputs.should_build == 'true' runs-on: ubuntu-latest timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - variant: - - name: production - dockerfile: Dockerfile - platforms: linux/amd64,linux/arm64 - - name: source - dockerfile: Dockerfile.source - platforms: linux/amd64,linux/arm64 - - name: dev - dockerfile: Dockerfile.source - platforms: linux/amd64,linux/arm64 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -254,47 +234,29 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Extract metadata and generate tags + - name: Extract metadata and generate tags id: meta run: | BUILD_TYPE="${{ needs.build-check.outputs.build_type }}" VERSION="${{ needs.build-check.outputs.version }}" SHORT_SHA="${{ needs.build-check.outputs.short_sha }}" CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}" - VARIANT="${{ matrix.variant.name }}" # Generate tags based on build type TAGS="" if [[ "$BUILD_TYPE" == "development" ]]; then - # Development build: dev-${short_sha}-${variant} and dev-${variant} - TAGS="${{ env.REGISTRY_DOCKERHUB }}:dev-${SHORT_SHA}-${VARIANT}" - - # Add rolling dev tag for each variant - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:dev-${VARIANT}" - - # Special handling for production variant - if [[ "$VARIANT" == "production" ]]; then - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:dev-${SHORT_SHA}" - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:dev" - fi + # Development build: dev-${short_sha} and dev + TAGS="${{ env.REGISTRY_DOCKERHUB }}:dev-${SHORT_SHA}" + TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:dev" else - # Release/Prerelease build: ${version}-${variant} - TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}-${VARIANT}" - - # Special handling for production variant - create main version tag - if [[ "$VARIANT" == "production" ]]; then - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${VERSION}" - fi + # Release/Prerelease build: ${version} + TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}" # Add channel tags for prereleases and latest for stable if [[ "$CREATE_LATEST" == "true" ]]; then # Stable release - if [[ "$VARIANT" == "production" ]]; then - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest" - else - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest-${VARIANT}" - fi + TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest" elif [[ "$BUILD_TYPE" == "prerelease" ]]; then # Prerelease channel tags (alpha, beta, rc) if [[ "$VERSION" == *"alpha"* ]]; then @@ -306,11 +268,7 @@ jobs: fi if [[ -n "$CHANNEL" ]]; then - if [[ "$VARIANT" == "production" ]]; then - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}" - else - TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}-${VARIANT}" - fi + TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}" fi fi fi @@ -325,7 +283,6 @@ jobs: LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}" LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" - LABELS="$LABELS,org.opencontainers.image.variant=$VARIANT" LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE" echo "labels=$LABELS" >> $GITHUB_OUTPUT @@ -339,20 +296,21 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ${{ matrix.variant.dockerfile }} - platforms: ${{ matrix.variant.platforms }} + file: Dockerfile + platforms: ${{ env.DOCKER_PLATFORMS }} push: ${{ needs.build-check.outputs.should_push == 'true' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: | - type=gha,scope=docker-${{ matrix.variant.name }} + type=gha,scope=docker-binary cache-to: | - type=gha,mode=max,scope=docker-${{ matrix.variant.name }} + type=gha,mode=max,scope=docker-binary build-args: | BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') VERSION=${{ needs.build-check.outputs.version }} BUILD_TYPE=${{ needs.build-check.outputs.build_type }} REVISION=${{ github.sha }} + RELEASE=${{ needs.build-check.outputs.version }} BUILDKIT_INLINE_CACHE=1 # Enable advanced BuildKit features for better performance provenance: false @@ -361,38 +319,8 @@ jobs: no-cache: false pull: true - # Create manifest for main production image (only for stable releases) - create-manifest: - name: Create Manifest - needs: [build-check, build-docker] - if: needs.build-check.outputs.should_push == 'true' && needs.build-check.outputs.create_latest == 'true' && needs.build-check.outputs.build_type == 'release' - runs-on: ubuntu-latest - steps: - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create and push manifest - run: | - VERSION="${{ needs.build-check.outputs.version }}" - - echo "๐Ÿณ Creating manifest for stable release: $VERSION" - - # Create main image tag (without variant suffix) for stable releases only - # Note: The "production" variant already creates the main tags without suffix - echo "Manifest creation is handled by the production variant build step" - echo "Main tags ${VERSION} and latest are created directly by the production variant" - - echo "โœ… Manifest created successfully for stable release" + # Note: Manifest creation is no longer needed as we only build one variant + # Multi-arch manifests are automatically created by docker/build-push-action # Docker build summary docker-summary: @@ -410,23 +338,24 @@ jobs: echo "๐Ÿณ Docker build completed successfully!" echo "๐Ÿ“ฆ Build type: $BUILD_TYPE" echo "๐Ÿ”ข Version: $VERSION" + echo "๐Ÿš€ Strategy: Images using pre-built binaries (supports both release and dev channels)" echo "" case "$BUILD_TYPE" in "development") - echo "๐Ÿ› ๏ธ Development Docker images have been built with dev-${VERSION} tags" - echo "โš ๏ธ These are development images - not suitable for production use" + echo "๐Ÿ› ๏ธ Development Docker image has been built with dev-${VERSION} tags" + echo "โš ๏ธ This is a development image - not suitable for production use" ;; "release") - echo "๐Ÿš€ Release Docker images have been built with v${VERSION} tags" - echo "โœ… These images are ready for production use" + echo "๐Ÿš€ Release Docker image has been built with ${VERSION} tags" + echo "โœ… This image is ready for production use" if [[ "$CREATE_LATEST" == "true" ]]; then - echo "๐Ÿท๏ธ Latest tags have been created for stable release" + echo "๐Ÿท๏ธ Latest tag has been created for stable release" fi ;; "prerelease") - echo "๐Ÿงช Prerelease Docker images have been built with v${VERSION} tags" - echo "โš ๏ธ These are prerelease images - use with caution" - echo "๐Ÿšซ Latest tags NOT created for prerelease" + echo "๐Ÿงช Prerelease Docker image has been built with ${VERSION} tags" + echo "โš ๏ธ This is a prerelease image - use with caution" + echo "๐Ÿšซ Latest tag NOT created for prerelease" ;; esac diff --git a/Dockerfile.source b/Dockerfile.source index c4bb4dff..24180e6e 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -1,4 +1,13 @@ -# Multi-stage Dockerfile for RustFS +# Multi-stage Dockerfile for RustFS - LOCAL DEVELOPMENT ONLY +# +# โš ๏ธ IMPORTANT: This Dockerfile is for local development and testing only. +# โš ๏ธ It builds RustFS from source code and is NOT used in CI/CD pipelines. +# โš ๏ธ CI/CD pipeline uses pre-built binaries from Dockerfile instead. +# +# Usage for local development: +# docker build -f Dockerfile.source -t rustfs:dev-local . +# docker run --rm -p 9000:9000 rustfs:dev-local +# # Supports cross-compilation for amd64 and arm64 architectures ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/docker-buildx.sh b/docker-buildx.sh index 05404503..bffa62f3 100755 --- a/docker-buildx.sh +++ b/docker-buildx.sh @@ -248,6 +248,8 @@ done # Main execution main() { print_message $BLUE "๐Ÿณ RustFS Docker Buildx Build Script" + print_message $YELLOW "๐Ÿ“‹ Build Strategy: Uses pre-built binaries from dl.rustfs.com" + print_message $YELLOW "๐Ÿš€ Production images only - optimized for distribution" echo "" # Check prerequisites