From 4260bdd7020821c25f0d3ac6c025e60fb95aa06f Mon Sep 17 00:00:00 2001 From: Robert Dailey Date: Sat, 30 Jul 2022 18:51:14 -0500 Subject: [PATCH] chore(docker): New approach for edge builds Edge builds previously would either download from the latest release on github or directly compile the code. However, dotnet apparently has some compatibility issues when run inside of a container built with qemu + buildx. The approach chosen going forward is to simply copy the builds from the github workflow artifacts directly into the container during the build process. This ended up causing a lot of change, mainly cleanup and simplifying things. --- .github/workflows/build.yml | 62 ++++++++++++---- .github/workflows/docker.yml | 64 ++++------------- .github/workflows/draft-new-release.yml | 78 --------------------- CONTRIBUTING.md | 48 ++++++------- ci/Test-Version.ps1 | 12 ++++ docker/.gitignore | 1 + docker/Build-Artifacts.ps1 | 9 +++ docker/Dockerfile | 4 +- docker/docker-compose.yml | 3 - docker/scripts/build/build-using-clone.sh | 7 -- docker/scripts/build/build-using-release.sh | 14 ---- docker/scripts/build/build.sh | 8 +-- 12 files changed, 106 insertions(+), 204 deletions(-) delete mode 100644 .github/workflows/draft-new-release.yml create mode 100644 ci/Test-Version.ps1 create mode 100644 docker/Build-Artifacts.ps1 delete mode 100644 docker/scripts/build/build-using-clone.sh delete mode 100644 docker/scripts/build/build-using-release.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83c981c9..ef6b76cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ jobs: run: dotnet test src --configuration Release --logger GitHubActions build: - name: Build + name: Build Non-MUSL needs: test strategy: fail-fast: true @@ -53,9 +53,6 @@ jobs: - linux-arm64 - osx-x64 - osx-arm64 - - linux-musl-x64 - - linux-musl-arm - - linux-musl-arm64 # Must run on Windows so that version info gets properly set in host EXE. See: # https://github.com/dotnet/runtime/issues/3828 runs-on: windows-latest @@ -70,14 +67,9 @@ jobs: with: dotnet-version: ${{ env.dotnetVersion }} - - name: Determine if single file build or not - if: contains(matrix.runtime, 'musl') - id: single_file - run: echo '::set-output name=arg::-noSingleFile' - - name: Publish shell: pwsh - run: ci/Publish.ps1 ${{ matrix.runtime }} ${{ steps.single_file.outputs.arg }} + run: ci/Publish.ps1 ${{ matrix.runtime }} - name: Upload Artifacts uses: actions/upload-artifact@v3 @@ -92,9 +84,9 @@ jobs: fail-fast: false matrix: include: - - {os: windows-latest, runtime: win-x64} - - {os: ubuntu-latest, runtime: linux-x64} - - {os: macos-latest, runtime: osx-x64} + - { os: windows-latest, runtime: win-x64 } + - { os: ubuntu-latest, runtime: linux-x64 } + - { os: macos-latest, runtime: osx-x64 } runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -109,9 +101,44 @@ jobs: shell: pwsh run: ci/SmokeTest.ps1 ./recyclarr + # NOTE: This is duplicated from the 'build' job. Sadly, reusable workflows cannot be invoked from + # matrix jobs. See here: + # https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations + musl: + name: Build MUSL + needs: test + strategy: + fail-fast: true + matrix: + runtime: + - linux-musl-x64 + - linux-musl-arm + - linux-musl-arm64 + runs-on: windows-latest + steps: + - name: Checkout Source Code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # avoid shallow clone for GitVersion + + - name: Setup .NET Core SDK ${{ env.dotnetVersion }} + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ env.dotnetVersion }} + + - name: Publish + shell: pwsh + run: ci/Publish.ps1 ${{ matrix.runtime }} -noSingleFile + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: recyclarr-${{ matrix.runtime }} + path: publish/${{ matrix.runtime }}/* + release: name: Release - needs: smoke + needs: [smoke, musl] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') steps: @@ -154,12 +181,17 @@ jobs: draft: false prerelease: ${{ steps.gitversion.outputs.preReleaseTag != '' }} + docker: + name: Docker + needs: musl + uses: ./.github/workflows/docker.yml + # The main purpose of this job is to group all the other jobs together into one single job status # that can be set as a requirement to merge pull requests. This is easier than enumerating all # jobs in a workflow to ensure they all pass. check: if: always() - name: Check Build Succeeds + name: Report Build Status needs: [build, smoke] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index af5d4fd3..db51284b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,39 +2,10 @@ name: Docker Image on: - push: - # Tags are explicitly ignored on push. We still want branches to be processed, but they won't if - # the `branches` property is missing. See more detail here: - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpushbranchestagsbranches-ignoretags-ignore - tags-ignore: ["*"] - branches: ["*"] - paths: - - docker/** - - .github/workflows/docker.yml - - pull_request: - paths: - - docker/** - - .github/workflows/docker.yml - - release: - types: [published] - - workflow_dispatch: - inputs: - release_tag: - description: Release Tag - required: true - type: string - should_publish: - description: Publish Image to GHCR? - required: false - default: false - type: boolean + workflow_call: env: - SHOULD_PUBLISH: ${{ github.event_name == 'release' || inputs.should_publish == 'true' || github.event.ref == 'refs/heads/master' }} - VERSION: ${{ github.event.release.tag_name || inputs.release_tag }} + SHOULD_PUBLISH: ${{ startsWith(github.ref, 'refs/tags/') || github.ref_name == 'master' }} jobs: docker: @@ -50,24 +21,19 @@ jobs: - name: Set up Buildx uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: --debug + # with: + # buildkitd-flags: --debug - - name: Check if tag is a version number + - name: Check Version id: check_version - run: | - regex="[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" - if [[ '${{ env.VERSION }}' =~ $regex ]]; then - echo '::set-output name=match::true' - else - echo '::set-output name=match::false' - fi + shell: pwsh + run: ci/Test-Version.ps1 ${{ github.ref_name }} - - name: Set Version Tags + - name: Set Docker Tags id: meta uses: docker/metadata-action@v4 env: - SEMVER: type=semver,enable=${{ steps.check_version.outputs.match }},value=${{ env.VERSION }} + SEMVER: type=semver,enable=${{ steps.check_version.outputs.match }},value=${{ github.ref_name }} with: images: ghcr.io/${{ github.repository }} tags: | @@ -76,10 +42,10 @@ jobs: ${{ env.SEMVER }},pattern={{major}}.{{minor}} ${{ env.SEMVER }},pattern={{major}} - - name: Enable building from source - id: info - if: ${{ github.event.ref == 'refs/heads/master' }} - run: echo '::set-output name=build_from_branch::master' + - name: Grab Artifacts + uses: actions/download-artifact@v3 + with: + path: docker/artifacts - name: Login to GHCR if: env.SHOULD_PUBLISH @@ -95,10 +61,6 @@ jobs: context: ./docker push: ${{ env.SHOULD_PUBLISH }} no-cache: true - build-args: | - REPOSITORY=${{ github.repository }} - ${{ env.VERSION && format('RELEASE_TAG={0}', env.VERSION) }} - BUILD_FROM_BRANCH=${{ steps.info.outputs.build_from_branch }} platforms: linux/arm/v7,linux/arm64,linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/draft-new-release.yml b/.github/workflows/draft-new-release.yml deleted file mode 100644 index 1d2e35ff..00000000 --- a/.github/workflows/draft-new-release.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Draft New Release - -on: - workflow_dispatch: - -jobs: - draft_new_release: - name: Draft a new release - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 # avoid shallow clone for NBGV - token: ${{ secrets.GITHUB_TOKEN }} # Allows git push - - - name: Set up NBGV - uses: dotnet/nbgv@master - id: nbgv - - - run: echo "VERSION=${{ steps.nbgv.outputs.SimpleVersion }}${{ steps.nbgv.outputs.PrereleaseVersion }}" >> $GITHUB_ENV - - - name: Initialize mandatory git config - run: | - git config user.name "GitHub Actions" - git config user.email noreply@github.com - - # TODO: Support specifying a SHA1 to branch from in the workflow run? - - name: Create Release Branch - run: | - nbgv prepare-release - git checkout release/${{ steps.nbgv.outputs.SimpleVersion }} - - - name: Update changelog - uses: thomaseizinger/keep-a-changelog-new-release@1.1.0 - with: - version: ${{ env.VERSION }} - - - name: Commit Changelog - run: git commit -m 'Finalize changelog for version ${{ env.VERSION }}' -- CHANGELOG.md - - - name: Push master and release branch - run: git push origin master +release/${{ steps.nbgv.outputs.SimpleVersion }} - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - id: cpr - with: - token: ${{ secrets.GITHUB_TOKEN }} - delete-branch: true - base: master - - - name: Enable Pull Request Automerge - uses: peter-evans/enable-pull-request-automerge@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - pull-request-number: ${{ steps.cpr.outputs.pull-request-number }} - merge-method: merge - title: "Preparation for Release: ${{ env.VERSION }}" - body: | - This pull request represents changes to be made in preparation of the next release, - ${{ env.VERSION }}. - - Once the build and release tasks in this PR are completed, the release will be created - and this PR will be automatically merged. - - - name: Auto Approve Pull Request - uses: actions/github-script@v3 - if: steps.cpr.outputs.pull-request-operation == 'created' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - await github.pulls.createReview({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: ${{ steps.cpr.outputs.pull-request-number }}, - event: 'APPROVE' - }) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27395f3f..a067e041 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,31 +21,36 @@ that everyone should follow. ## Docker Development -The project's `Dockerfile` builds in two different mods: Development and production mode. +The project's `Dockerfile` build requires the Recyclarr build output to be placed in a specific +location in order to succeed. The location is below, relative to the clone root: -### Production Build +```txt +docker/artifacts/recyclarr-${{runtime}} +``` -This is the default build type for the image. Given a specific version number, it will grab the -appropriate binary from the corresponding Github Release and install that into the image. +Where `${{runtime}}` is one of the runtimes compatible with `dotnet publish`, such as +`linux-musl-x64`. -### Development Build +There is a convenience script named `docker/Build-Artifacts.ps1` that will perform a build and place +the output in the appropriate location for you. This simplifies the process of testing docker +locally to these steps: -This build allows you to make changes to Recyclarr and pull those into a local docker image build. -This is especially useful if you want to test changes in Recyclarr before it is released, since the -production mode of Recyclarr requires a Github release to pull from. +1. Run the convenience script to build and publish Recyclarr to the Docker artifacts directory: -To enable development builds, specify the build argument `BUILD_FROM_BRANCH`. The workflow I use -goes something like this: + ```sh + pwsh ci/Build-Artifacts.ps1 + ``` + + > *Note:* The runtime defaults to `linux-musl-x64` but you can pass in an override as the first + > placeholder argument to the above command. -1. Create a branch to work out of: `git checkout -b docker origin/master`. -1. Make some C# code changes, commit, and **push to the remote repo**. -1. Build the docker image locally: +1. Execute a Docker build locally via compose: ```sh - docker compose build --no-cache --progress plain --build-arg BUILD_FROM_BRANCH=docker + docker compose build --no-cache --progress plain ``` -1. Execute it locally: +1. Run the container to test it: ```sh docker compose run --rm recyclarr sonarr @@ -53,24 +58,11 @@ goes something like this: ### Build Arguments -- `RELEASE_TAG` (Default: `latest`)
- The git tag (e.g. `v2.1.2`) that represents the Github Release in the upstream repository to grab - binaries from. May also use `latest` to represent the latest Github Release. Only used in - Production builds. - - `TARGETPLATFORM` (Default: empty)
Required. Specifies the runtime architecture of the image and is used to pull the correct prebuilt binary from the specified Github Release. See the table in the Platform Support section for a list of valid values. -- `REPOSITORY` (Default: `recyclarr/recyclarr`)
- The Github repository name (either `user/repo` or `organization/repo` format) used to grab the - prebuilt release from (in Production builds) or to clone (in Development builds). - -- `BUILD_FROM_BRANCH` (Default: empty)
- If specified, Development build mode is enabled and the branch name specified here is used to - compile Recyclarr and use its final binary in the resulting docker image. - ### Platform Support | Docker Platform | Recyclarr Runtime | diff --git a/ci/Test-Version.ps1 b/ci/Test-Version.ps1 new file mode 100644 index 00000000..637cda23 --- /dev/null +++ b/ci/Test-Version.ps1 @@ -0,0 +1,12 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)] + [string]$Version +) + +if ($Version -match 'v\d+\.\d+\.\d+') { + '::set-output name=match::true' +} +else { + '::set-output name=match::false' +} diff --git a/docker/.gitignore b/docker/.gitignore index 5b8a7b86..96e9fc04 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1 +1,2 @@ /config/ +/artifacts/ diff --git a/docker/Build-Artifacts.ps1 b/docker/Build-Artifacts.ps1 new file mode 100644 index 00000000..93807567 --- /dev/null +++ b/docker/Build-Artifacts.ps1 @@ -0,0 +1,9 @@ +[CmdletBinding()] +param ( + $runtime = "linux-musl-x64" +) + +$artifactDir="$PSScriptRoot\artifacts" + +Remove-Item $artifactDir -Recurse -Force -ErrorAction SilentlyContinue +dotnet publish "$PSScriptRoot\..\src\Recyclarr" -o "$artifactDir\recyclarr-$runtime" -r $runtime diff --git a/docker/Dockerfile b/docker/Dockerfile index 8b7a7edf..2afccd08 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,14 +2,12 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS build WORKDIR /build -ARG RELEASE_TAG=latest ARG TARGETPLATFORM -ARG REPOSITORY=recyclarr/recyclarr -ARG BUILD_FROM_BRANCH RUN apk add unzip bash COPY --chmod=544 ./scripts/build/*.sh . +COPY ./artifacts ./artifacts RUN ./build.sh ############################################################################# diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d9c657d6..a062e2a7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - networks: recyclarr: name: recyclarr @@ -13,7 +11,6 @@ services: context: . args: - TARGETPLATFORM=linux/amd64 - - BUILD_FROM_BRANCH=master init: true networks: [recyclarr] volumes: diff --git a/docker/scripts/build/build-using-clone.sh b/docker/scripts/build/build-using-clone.sh deleted file mode 100644 index 56d3f829..00000000 --- a/docker/scripts/build/build-using-clone.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -ex - -# Do not shallow clone because gitversion needs history! -git clone -b $BUILD_FROM_BRANCH "https://github.com/$REPOSITORY.git" source - -dotnet publish source/src/Recyclarr -o publish -c Release -r $runtime diff --git a/docker/scripts/build/build-using-release.sh b/docker/scripts/build/build-using-release.sh deleted file mode 100644 index 93b61f63..00000000 --- a/docker/scripts/build/build-using-release.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -ex - -# The download path is a bit different when using the latest release instead of a specific -# release -if [ "$RELEASE_TAG" = "latest" ]; then - download_path="latest/download"; -else - download_path="download/$RELEASE_TAG"; -fi - -# Download and extract the recyclarr binary from the release -wget --quiet -O recyclarr.zip "https://github.com/$REPOSITORY/releases/$download_path/recyclarr-$runtime.zip" -unzip recyclarr.zip -d publish diff --git a/docker/scripts/build/build.sh b/docker/scripts/build/build.sh index 72205a0b..b0c45cae 100644 --- a/docker/scripts/build/build.sh +++ b/docker/scripts/build/build.sh @@ -9,10 +9,8 @@ case "$TARGETPLATFORM" in *) echo >&2 "ERROR: Unsupported target platform: $TARGETPLATFORM"; exit 1 ;; esac -if [ -z "$BUILD_FROM_BRANCH" ]; then - . ./build-using-release.sh -else - . ./build-using-clone.sh -fi +path="artifacts/recyclarr-$runtime/" + +mv "$path" publish chmod a+rx publish/recyclarr