CI: test the tests (#32869)

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,123 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    if: github.repository == 'commaai/openpilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.value }}
+    steps:
+      - id: ci_runs_setup
+        run: |
+          CI_RUNS=${{ inputs.ci_runs || '50' }}
+          mylist="value=["
+
+          for i in $(seq 1 $CI_RUNS);
+          do
+              if [ $i != $CI_RUNS ]; then
+                  mylist+="\"$i\", "
+              else
+                  mylist+="\"$i\"]"
+              fi
+          done
+
+          echo "$mylist" >> $GITHUB_OUTPUT
+          echo "Number of CI runs for report: $CI_RUNS"
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix:
+        value: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.value }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              const jobName = job.name.split('/')[2].trim();
+              report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push(job.html_url); break;
+                case "failure":
+                  report[jobName].failures.push(job.html_url); break;
+                case "cancelled":
+                  report[jobName].cancelled.push(job.html_url); break;
+              }
+            });
+            return JSON.stringify(report);
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          echo $JOB_RESULTS > job_results.json
+          generate_html_table() {
+            echo "<table>"
+            echo "<thead>"
+            echo "  <tr>"
+            echo "    <th>Job</th>"
+            echo "    <th>Succeeded ✅</th>"
+            echo "    <th>Failed ❌</th>"
+            echo "    <th>Cancelled (timed out) ⏰</th>"
+            echo "  </tr>"
+            echo "</thead>"
+            jq -r '
+              "<tbody>",
+              keys[] as $job |
+              "<tr>",
+              "  <td>\($job)</td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].successes | length))</summary>",
+              "      \(.[$job].successes[])<br>",
+              "    </details>",
+              "  </td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].failures | length))</summary>",
+              "      \(.[$job].failures[])<br>",
+              "    </details>",
+              "  </td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].cancelled | length))</summary>",
+              "      \(.[$job].cancelled[])<br>",
+              "    </details>",
+              "  </td>",
+              "</tr>"
+            ' job_results.json
+            echo  "</tbody>"
+            echo  "</table>"
+          }
+          echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY
+          generate_html_table >> $GITHUB_STEP_SUMMARY
+

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,21 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  selfdrive_tests:
+    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
+  tools_tests:
+    uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/docs.yaml

@@ -5,9 +5,14 @@ on:
     branches:
       - master
   pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:

.github/workflows/selfdrive_tests.yaml

@@ -6,9 +6,15 @@ on:
       - master
   pull_request:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:
@@ -320,5 +326,5 @@ jobs:
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report
-          path: selfdrive/ui/tests/test_ui/report
+          name: report-${{ inputs.run_number }}
+          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}

.github/workflows/tools_tests.yaml

@@ -5,9 +5,14 @@ on:
     branches:
       - master
   pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:

selfdrive/ui/tests/test_ui/run.py

@@ -116,7 +116,7 @@ CASES = {
 
 TEST_DIR = pathlib.Path(__file__).parent
 
-TEST_OUTPUT_DIR = TEST_DIR / "report"
+TEST_OUTPUT_DIR = TEST_DIR / "report_1"
 SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"