Update ci weekly report summary (#32911)

* update ci weekly report summary

* don't add skipped jobs to report

* uncomment job condition

* use jinja template

* add run number to failure links

* add run number to failure links

* Log for run #<num>

* use list

---------

Co-authored-by: Maxime Desroches <desroches.maxime@gmail.com>
old-commit-hash: 6745c66352ffacabbf1201f766ab7ff308724d18

.github/workflows/ci_weekly_report.yaml

@@ -10,38 +10,30 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
 jobs:
   setup:
     if: github.repository == 'commaai/openpilot'
     runs-on: ubuntu-latest
     outputs:
-      ci_runs: ${{ steps.ci_runs_setup.outputs.value }}
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
     steps:
       - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
         run: |
-          CI_RUNS=${{ inputs.ci_runs || '50' }}
-          mylist="value=["
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
-          for i in $(seq 1 $CI_RUNS);
-          do
-              if [ $i != $CI_RUNS ]; then
-                  mylist+="\"$i\", "
-              else
-                  mylist+="\"$i\"]"
-              fi
-          done
-
-          echo "$mylist" >> $GITHUB_OUTPUT
-          echo "Number of CI runs for report: $CI_RUNS"
   ci_matrix_run:
     needs: [ setup ]
     strategy:
       fail-fast: false
-      matrix:
-        value: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
     uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
     with:
-      run_number: ${{ matrix.value }}
+      run_number: ${{ matrix.run_number }}
 
   report:
     needs: [ci_matrix_run]
@@ -62,62 +54,48 @@ jobs:
               })
             var report = {}
             jobs.slice(1, jobs.length-1).forEach(job => {
-              const jobName = job.name.split('/')[2].trim();
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
               report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
               switch (job.conclusion) {
                 case "success":
-                  report[jobName].successes.push(job.html_url); break;
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
                 case "failure":
-                  report[jobName].failures.push(job.html_url); break;
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
                 case "cancelled":
-                  report[jobName].cancelled.push(job.html_url); break;
+                  report[jobName].cancelled.push({ "run_number": run, "link": job.html_url }); break;
               }
             });
-            return JSON.stringify(report);
+            return JSON.stringify({"jobs": report});
 
       - name: Add job results to summary
         env:
           JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
         run: |
-          echo $JOB_RESULTS > job_results.json
-          generate_html_table() {
-            echo "<table>"
-            echo "<thead>"
-            echo "  <tr>"
-            echo "    <th>Job</th>"
-            echo "    <th>Succeeded ✅</th>"
-            echo "    <th>Failed ❌</th>"
-            echo "    <th>Cancelled (timed out) ⏰</th>"
-            echo "  </tr>"
-            echo "</thead>"
-            jq -r '
-              "<tbody>",
-              keys[] as $job |
-              "<tr>",
-              "  <td>\($job)</td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].successes | length))</summary>",
-              "      \(.[$job].successes[])<br>",
-              "    </details>",
-              "  </td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].failures | length))</summary>",
-              "      \(.[$job].failures[])<br>",
-              "    </details>",
-              "  </td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].cancelled | length))</summary>",
-              "      \(.[$job].cancelled[])<br>",
-              "    </details>",
-              "  </td>",
-              "</tr>"
-            ' job_results.json
-            echo  "</tbody>"
-            echo  "</table>"
-          }
-          echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY
-          generate_html_table >> $GITHUB_STEP_SUMMARY
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
 
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY