Enhance checks for invalid keyboard build targets (#26122)

Co-authored-by: フィルターペーパー <76888457+filterpaper@users.noreply.github.com>

.github/workflows/ci_build_major_branch.yml

@@ -32,6 +32,7 @@ jobs:
     container: ghcr.io/qmk/qmk_cli
 
     outputs:
+      keymaps: ${{ steps.generate_slice_length.outputs.keymaps }}
       slice_length: ${{ steps.generate_slice_length.outputs.slice_length }}
 
     steps:
@@ -47,12 +48,20 @@ jobs:
 
       - name: Determine concurrency
         id: generate_slice_length
+        shell: 'bash {0}'
         run: |
-          target_count=$( {
-            qmk find -km default 2>/dev/null
-            qmk find -km xap 2>/dev/null
-          } | sort | uniq | wc -l)
+          targets=()
+          target_count=0
+          for target in "default" "xap"; do
+              count=$(qmk find -km $target 2>/dev/null | wc -l)
+              if [ $count -gt 0 ]; then
+                  target_count=$(($target_count + $count))
+                  targets+=($target)
+              fi
+          done
+          keymaps=$(jq -c -n '$ARGS.positional' --args "${targets[@]}")
           slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution
+          echo "keymaps=$keymaps" >> $GITHUB_OUTPUT
           echo "slice_length=$slice_length" >> $GITHUB_OUTPUT
 
   build_targets:
@@ -61,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        keymap: [default, xap]
+        keymap: ${{ fromJson(needs.determine_concurrency.outputs.keymaps) }}
     uses: ./.github/workflows/ci_build_major_branch_keymap.yml
     with:
       branch: ${{ inputs.branch || github.ref_name }}

.github/workflows/ci_build_major_branch_keymap.yml

@@ -38,17 +38,18 @@ jobs:
         run: pip3 install -r requirements-dev.txt
 
       - name: Generate build targets
+        shell: 'bash {0}'
         id: generate_targets
         run: |
           { # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer
             counter=0
             echo -n '{'
-            qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
+            qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf --random-source=<(openssl enc -aes-256-ctr -pass pass:qmk -nosalt </dev/zero 2>/dev/null) | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
                 if [ $counter -gt 0 ]; then
                     echo -n ','
                 fi
                 counter=$((counter+1))
-                printf "\"group %02d\":{" $counter
+                printf "\"group-%02d\":{" $counter
                 echo -n '"targets":"'
                 echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n
                 echo -n '"}'
@@ -72,6 +73,9 @@ jobs:
     container: ghcr.io/qmk/qmk_cli
     continue-on-error: true
 
+    env:
+      CCACHE_CONFIGPATH: ~/.cache
+
     strategy:
       matrix:
         target: ${{ fromJson(needs.generate_targets.outputs.targets) }}
@@ -83,6 +87,8 @@ jobs:
 
       - name: Checkout QMK Firmware
         uses: actions/checkout@v6
+        with:
+          submodules: recursive
 
       - name: Install dependencies
         run: pip3 install -r requirements-dev.txt
@@ -93,24 +99,47 @@ jobs:
           name: targets-${{ inputs.keymap }}
           path: .
 
-      - name: Deploy submodules
-        run: |
-          qmk git-submodule -f
-
       - name: Dump targets
         run: |
           jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort
 
+      - name: Restore Cache
+        id: cache
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_CONFIGPATH }}
+          key: compile-${{ inputs.keymap }}-${{ matrix.target }}
+
       - name: Build targets
         continue-on-error: true
         run: |
-          export NCPUS=$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null) -1 ))
           targets=$(jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort)
           if [ -z "${targets}" ]; then
             echo "Zero build targets detected"
             exit 0
           fi
-          qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $targets || touch .failed
+          qmk mass-compile -t -j $(nproc) -e DUMP_CI_METADATA=yes -e USE_CCACHE=yes $targets || touch .failed
+
+      - name: Dump ccache stats
+        run: |
+          ccache -s
+
+      # Delete the old cache on hit to emulate a cache update. See https://github.com/actions/cache/issues/342.
+      - name: Delete old cache
+        env:
+          GH_TOKEN: ${{ github.token }}
+        if: steps.cache.outputs.cache-hit
+        run: |
+          count=$(gh cache list --ref ${{ github.ref }} --key ${{ steps.cache.outputs.cache-primary-key }} --json id | jq length)
+          if [ $count -gt 0 ]; then
+            gh cache delete --ref ${{ github.ref }} ${{ steps.cache.outputs.cache-primary-key }}
+          fi
+
+      - name: Save Cache
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ env.CCACHE_CONFIGPATH }}
+          key: compile-${{ inputs.keymap }}-${{ matrix.target }}
 
       - name: Upload binaries
         uses: actions/upload-artifact@v7

.github/workflows/lint.yml

@@ -89,23 +89,7 @@ jobs:
       if: always()
       shell: 'bash {0}'
       run: |
-        exit_code=0
-
-        for file in $(find keyboards/ -name rules.mk | grep -v /keymaps/ | grep -v /common/ | grep -v /lib/); do
-            dir=$(dirname $file)
-
-            $(find $dir -name keyboard.json -exec false {} +)
-            if [[ $? == 0 ]]; then
-                echo "$dir::Legacy target detected"
-
-                ((++exit_code))
-            fi
-        done
-
-        if [[ $exit_code -gt 255 ]]; then
-            exit 255
-        fi
-        exit $exit_code
+        qmk ci-validate-keyboard-targets
 
     - name: Verify keyboard aliases
       if: always()

lib/python/qmk/cli/__init__.py

@@ -56,6 +56,7 @@ safe_commands = [
 
 subcommands = [
     'qmk.cli.ci.validate_aliases',
+    'qmk.cli.ci.validate_keyboard_targets',
     'qmk.cli.bux',
     'qmk.cli.c2json',
     'qmk.cli.cd',

lib/python/qmk/cli/ci/validate_keyboard_targets.py

@@ -0,0 +1,28 @@
+"""Validates the list of keyboard targets.
+"""
+from milc import cli
+
+from pathlib import Path
+
+
+@cli.subcommand('Validates the list of keyboard targets.', hidden=True)
+def ci_validate_keyboard_targets(cli):
+    errors = set()
+
+    for rules_mk in Path('keyboards').glob('**/rules.mk'):
+        if any({'keymaps', 'common', 'lib'} & set(rules_mk.parts)):
+            continue
+
+        folder = rules_mk.parent
+        if not any(folder.glob('**/keyboard.json')):
+            errors.add(folder)
+
+    for keymap in Path('keyboards').glob('**/keymaps/'):
+        folder = keymap.parent
+        if not any(folder.glob('**/keyboard.json')):
+            errors.add(folder)
+
+    for error in errors:
+        print(f"{error}::Legacy target detected")
+
+    exit(min(len(errors), 255))