raylib: ui diff test (#36213)

* add raylib ui test

* match qt

* exe

* vibing is epic

* this is epic

* format

* add more settings

* fix to actually use raylib

* add kb

* global

* pair

* rm cmts

* show event

* this is so stupid

clean up

* clean up

* rename dir

* clean up

* no more vibe

* rm

* ugh it's always slightly different for no reason

* nvm region is actually broken

* 1l

.github/workflows/raylib_ui_preview.yaml

@@ -0,0 +1,174 @@
+name: "raylib ui preview"
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/ui/**'
+      - 'system/ui/**'
+  workflow_dispatch:
+
+env:
+  UI_JOB_NAME: "Create raylib UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui"
+
+jobs:
+  preview:
+    if: github.repository == 'commaai/openpilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Waiting for ui generation to start
+        run: sleep 30
+
+      - name: Waiting for ui generation to end
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ env.SHA }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Getting workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Getting proposed ui
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: raylib-report-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui_raylib
+          ref: openpilot_master_ui_raylib
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git checkout --orphan=new_master_ui_raylib
+          git rm -rf *
+          git branch -D openpilot_master_ui_raylib
+          git branch -m openpilot_master_ui_raylib
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "raylib screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui_raylib --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get update && sudo apt-get install -y imagemagick
+
+          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+            # Check if the master file exists
+            if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then
+              # This is a new file in PR UI that doesn't exist in master
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "raylib screenshots for PR #${{ github.event.number }}"
+          git push origin ${{ env.BRANCH_NAME }} --force
+
+      - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots_raylib **${{ github.run_id }}**)_ -->
+            ## raylib UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots_raylib
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/selfdrive_tests.yaml

@@ -277,3 +277,29 @@ jobs:
         with:
           name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
           path: selfdrive/ui/tests/test_ui/report_1/screenshots
+
+  create_raylib_ui_report:
+    name: Create raylib UI Report
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create raylib UI Report
+        run: >
+          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                          source selfdrive/test/setup_xvfb.sh &&
+                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
+      - name: Upload Raylib UI Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots