A .github/workflows/ci_build_major_branch.yml => .github/workflows/ci_build_major_branch.yml +122 -0
@@ 0,0 1,122 @@
+name: CI Build Major Branch
+
+permissions:
+ contents: read
+ actions: write
+
+on:
+ push:
+ branches: [master, develop]
+ workflow_dispatch:
+ inputs:
+ branch:
+ type: choice
+ description: "Branch to build"
+ options: [master, develop]
+
+env:
+ # https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits
+ # We've decreased it from 20 to 15 to allow for other GHA to run unimpeded
+ CONCURRENT_JOBS: 15
+
+# Ensure we only have one build running at a time, cancelling any active builds if a new commit is pushed to the respective branch
+concurrency:
+ group: ci_build-${{ github.event.inputs.branch || github.ref_name }}
+ cancel-in-progress: true
+
+jobs:
+ determine_concurrency:
+ name: "Determine concurrency"
+ if: github.repository == 'qmk/qmk_firmware'
+ runs-on: ubuntu-latest
+ container: ghcr.io/qmk/qmk_cli
+
+ outputs:
+ slice_length: ${{ steps.generate_slice_length.outputs.slice_length }}
+
+ steps:
+ - name: Install prerequisites
+ run: |
+ apt-get update
+ apt-get install -y jq
+
+ - name: Disable safe.directory check
+ run: |
+ git config --global --add safe.directory '*'
+
+ - name: Checkout QMK Firmware
+ uses: actions/checkout@v4
+
+ - name: Determine concurrency
+ id: generate_slice_length
+ run: |
+ target_count=$( {
+ qmk find -km default 2>/dev/null
+ qmk find -km via 2>/dev/null
+ } | sort | uniq | wc -l)
+ slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution as we're splitting default and via
+ echo "slice_length=$slice_length" >> $GITHUB_OUTPUT
+
+ build_targets:
+ name: "Compile keymap ${{ matrix.keymap }}"
+ needs: determine_concurrency
+ strategy:
+ fail-fast: false
+ matrix:
+ keymap: [default, via]
+ uses: ./.github/workflows/ci_build_major_branch_keymap.yml
+ with:
+ branch: ${{ inputs.branch || github.ref_name }}
+ keymap: ${{ matrix.keymap }}
+ slice_length: ${{ needs.determine_concurrency.outputs.slice_length }}
+
+ rollup_tasks:
+ name: "Housekeeping"
+ needs: build_targets
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Download firmwares
+ uses: actions/download-artifact@v4
+ with:
+ pattern: firmware-*
+ path: firmwares
+ merge-multiple: true
+
+ - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }}
+ uses: jakejarvis/s3-sync-action@master
+ with:
+ args: --acl public-read --follow-symlinks --delete
+ env:
+ AWS_S3_BUCKET: qmk-ci
+ AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
+ AWS_REGION: nyc3
+ AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
+ SOURCE_DIR: firmwares
+ DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }}
+
+ - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest
+ uses: jakejarvis/s3-sync-action@master
+ with:
+ args: --acl public-read --follow-symlinks --delete
+ env:
+ AWS_S3_BUCKET: qmk-ci
+ AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
+ AWS_REGION: nyc3
+ AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
+ SOURCE_DIR: firmwares
+ DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest
+
+ - name: Check if failure marker file exists
+ id: check_failure_marker
+ uses: andstor/file-existence-action@v3
+ with:
+ files: firmwares/.failed
+
+ - name: Fail build if needed
+ if: steps.check_failure_marker.outputs.exists == 'true'
+ run: |
+ # Exit with failure if the compilation stage failed
+ exit 1
A .github/workflows/ci_build_major_branch_keymap.yml => .github/workflows/ci_build_major_branch_keymap.yml +181 -0
@@ 0,0 1,181 @@
+name: CI Build Major Branch Keymap
+
+permissions:
+ contents: read
+ actions: write
+
+on:
+ workflow_call:
+ inputs:
+ branch:
+ type: string
+ required: true
+ keymap:
+ type: string
+ required: true
+ slice_length:
+ type: string
+ required: true
+
+jobs:
+ generate_targets:
+ name: "Generate targets (${{ inputs.keymap }})"
+ runs-on: ubuntu-latest
+ container: ghcr.io/qmk/qmk_cli
+
+ outputs:
+ targets: ${{ steps.generate_targets.outputs.targets }}
+
+ steps:
+ - name: Install prerequisites
+ run: |
+ apt-get update
+ apt-get install -y jq
+
+ - name: Disable safe.directory check
+ run: |
+ git config --global --add safe.directory '*'
+
+ - name: Checkout QMK Firmware
+ uses: actions/checkout@v4
+
+ - name: Generate build targets
+ id: generate_targets
+ run: |
+ { # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer
+ counter=0
+ echo -n '{'
+ qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
+ if [ $counter -gt 0 ]; then
+ echo -n ','
+ fi
+ counter=$((counter+1))
+ printf "\"group %02d\":{" $counter
+ echo -n '"targets":"'
+ echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n
+ echo -n '"}'
+ done
+ echo -n '}'
+ } | sed -e 's@\n@@g' > targets.json
+
+ # Output the target keys as a variable
+ echo "targets=$(jq -c 'keys' targets.json)" >> $GITHUB_OUTPUT
+
+ - name: Upload targets json
+ uses: actions/upload-artifact@v4
+ with:
+ name: targets-${{ inputs.keymap }}
+ path: targets.json
+
+ build_targets:
+ name: "Compile ${{ matrix.target }} (${{ inputs.keymap }})"
+ needs: generate_targets
+ runs-on: ubuntu-latest
+ container: ghcr.io/qmk/qmk_cli
+ continue-on-error: true
+
+ strategy:
+ matrix:
+ target: ${{ fromJson(needs.generate_targets.outputs.targets) }}
+
+ steps:
+ - name: Install prerequisites
+ run: |
+ apt-get update
+ apt-get install -y jq
+
+ - name: Disable safe.directory check
+ run: |
+ git config --global --add safe.directory '*'
+
+ - name: Checkout QMK Firmware
+ uses: actions/checkout@v4
+
+ - name: Get target definitions
+ uses: actions/download-artifact@v4
+ with:
+ name: targets-${{ inputs.keymap }}
+ path: .
+
+ - name: Deploy submodules
+ run: |
+ qmk git-submodule -f
+
+ - name: Dump targets
+ run: |
+ jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort
+
+ - name: Build targets
+ continue-on-error: true
+ run: |
+ export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
+ qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $(jq -r '.["${{ matrix.target }}"].targets' targets.json) || touch .failed
+
+ - name: Upload binaries
+ uses: actions/upload-artifact@v4
+ with:
+ name: firmware-${{ inputs.keymap }}-${{ matrix.target }}
+ if-no-files-found: ignore
+ path: |
+ *.bin
+ *.hex
+ *.uf2
+ .build/failed.*
+ .failed
+
+ - name: Fail build if any group failed
+ run: |
+ # Exit with failure if the compilation stage failed
+ [ ! -f .failed ] || exit 1
+
+ repack_firmware:
+ if: always()
+ name: "Repack artifacts"
+ needs: build_targets
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout QMK Firmware
+ uses: actions/checkout@v4
+
+ - name: Download firmwares
+ uses: actions/download-artifact@v4
+ with:
+ pattern: firmware-${{ inputs.keymap }}-*
+ path: .
+ merge-multiple: true
+
+ - name: Upload all firmwares
+ uses: actions/upload-artifact@v4
+ with:
+ name: firmware-${{ inputs.keymap }}
+ if-no-files-found: ignore
+ path: |
+ *.bin
+ *.hex
+ *.uf2
+ .build/failed.*
+ .failed
+
+ - name: Generate output logs
+ run: |
+ # Generate the step summary markdown
+ ./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
+ # Truncate to a maximum of 1MB to deal with GitHub workflow limit
+ truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
+
+ - name: Delete temporary build artifacts
+ uses: geekyeggo/delete-artifact@v4
+ with:
+ name: |
+ firmware-${{ inputs.keymap }}-*
+ targets-${{ inputs.keymap }}
+
+ - name: 'CI Discord Notification'
+ if: always()
+ working-directory: util/ci/
+ env:
+ DISCORD_WEBHOOK: ${{ secrets.CI_DISCORD_WEBHOOK }}
+ run: |
+ python3 -m pip install -r requirements.txt
+ python3 ./discord-results.py --branch ${{ inputs.branch || github.ref_name }} --keymap ${{ inputs.keymap }} --url ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}