sunnypilot/.github/workflows/build-all-tinygrad-models.yaml

name: Build All Tinygrad Models and Push to GitLab

on:
  workflow_dispatch:
    inputs:
      runner_type:
        description: 'Runner type'
        required: false
        default: '[self-hosted, james-mac]'
        type: choice
        options:
          - ubuntu-latest
          - '[self-hosted, james-mac]'
      branch:
        description: 'Branch to run workflow from'
        required: false
        default: 'master-new'
        type: string

jobs:
  setup:
    runs-on: ${{ github.event.inputs.runner_type }}
    outputs:
      models: ${{ steps.get-models.outputs.models }}
      RECOMPILED_DIR: ${{ steps.set-recompiled.outputs.RECOMPILED_DIR }}
      JSON_FILE: ${{ steps.get-json.outputs.JSON_FILE }}
      SRC_JSON_FILE: ${{ steps.get-json.outputs.SRC_JSON_FILE }}
    steps:
      - name: Set up SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}

      - name: Add GitLab.com SSH key to known_hosts
        run: |
          mkdir -p ~/.ssh
          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts

      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
        uses: actions/checkout@v4
        with:
          repository: sunnypilot/sunnypilot-docs
          ref: gh-pages
          path: docs
          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

      - name: Clone GitLab docs repo
        env:
          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
        run: |
          echo "Cloning GitLab"
          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
          cd gitlab_docs
          git checkout main
          cd ..

      - name: Set next recompiled dir
        id: set-recompiled
        run: |
          cd gitlab_docs
          echo "checkout models/"
          git sparse-checkout set --no-cone models/
          git checkout main
          cd models
          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
          if [[ -z "$latest_dir" ]]; then
            next_dir=1
          else
            next_dir=$((latest_dir+1))
          fi
          recompiled_dir="recompiled${next_dir}"
          if [ -d "$recompiled_dir" ]; then
            echo "ERROR: $recompiled_dir already exists in GitLab repo"
            exit 1
          fi
          mkdir -p "$recompiled_dir"
          touch "$recompiled_dir/.gitkeep"
          echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
          echo "Created new recompiled dir: $recompiled_dir"
          cd ../..

      - name: Get next JSON version to use (from GitHub docs repo)
        id: get-json
        run: |
          cd docs/docs
          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
          next=$((latest+1))
          json_file="driving_models_v${next}.json"
          cp "driving_models_v${latest}.json" "$json_file"
          echo "json_file=$json_file" >> $GITHUB_OUTPUT
          echo "JSON_FILE=docs/docs/$json_file" >> $GITHUB_ENV
          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
          echo "SRC_JSON_FILE: docs/docs/driving_models_v${latest}.json"

      - name: Get tinygrad models from JSON
        id: get-models
        working-directory: docs/docs
        run: |
          MODELS=$(jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name, is_20hz}]' "$(basename "${SRC_JSON_FILE}")")
          echo "models=${MODELS}" >> $GITHUB_OUTPUT
          echo "Parsed models: ${MODELS}"
        shell: bash

  get-models:
    needs: setup
    runs-on: ${{ github.event.inputs.runner_type }}
    strategy:
      matrix:
        model: ${{ fromJson(needs.setup.outputs.models) }}
      fail-fast: false
    env:
      OUTPUT_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
    steps:
      - name: Checkout commaai/openpilot
        id: checkout_upstream
        continue-on-error: true
        uses: actions/checkout@v4
        with:
          repository: commaai/openpilot
          ref: ${{ matrix.model.ref }}
          submodules: recursive
          path: openpilot

      - name: Fallback to sunnypilot/sunnypilot
        if: steps.checkout_upstream.outcome == 'failure'
        uses: actions/checkout@v4
        with:
          repository: sunnypilot/sunnypilot
          ref: ${{ matrix.model.ref }}
          submodules: recursive
          path: openpilot

      - name: Get commit date
        id: commit-date
        run: |
          cd openpilot
          commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
          echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT

      - name: Pull lfs
        run: |
          cd openpilot
          git lfs pull

      - name: Copy models
        run: |
          mkdir -p "${{ env.OUTPUT_DIR }}"
          cp openpilot/selfdrive/modeld/models/*.onnx "${{ env.OUTPUT_DIR }}/" || echo "No models found."

      - name: Upload model artifacts
        uses: actions/upload-artifact@v4
        with:
          name: onnx-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
          path: ${{ env.OUTPUT_DIR }}

  build-models:
    needs: [setup, get-models]
    runs-on: [self-hosted, tici]
    strategy:
      matrix:
        model: ${{ fromJson(needs.setup.outputs.models) }}
      fail-fast: false
    env:
      BUILD_DIR: /data/openpilot
      OUTPUT_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
      SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
      TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
      MODELS_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
      POWERSAVE_SCRIPT: ${{ github.workspace }}/scripts/manage-powersave.py
      MODEL_GENERATOR: ${{ github.workspace }}/release/ci/model_generator.py
      GET_MODEL_METADATA: ${{ github.workspace }}/selfdrive/modeld/get_model_metadata.py
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.branch }}
          submodules: recursive
      - run: git lfs pull
      - uses: actions/cache@v4
        with:
          path: ${{ env.SCONS_CACHE_DIR }}
          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
          restore-keys: |
            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
            scons-${{ runner.os }}-${{ runner.arch }}-master-new-model
            scons-${{ runner.os }}-${{ runner.arch }}-master-model
            scons-${{ runner.os }}-${{ runner.arch }}-master-new
            scons-${{ runner.os }}-${{ runner.arch }}-master
            scons-${{ runner.os }}-${{ runner.arch }}
      - run: |
          source /etc/profile
          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
          printenv >> $GITHUB_ENV
      - uses: actions/download-artifact@v4
        with:
          name: onnx-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
          path: ${{ env.MODELS_DIR }}
      - run: |
          python3 release/ci/build_model.py \
            --build-dir "${BUILD_DIR}" \
            --output-dir "${OUTPUT_DIR}" \
            --scons-cache-dir "${SCONS_CACHE_DIR}" \
            --tinygrad-path "${TINYGRAD_PATH}" \
            --models-dir "${MODELS_DIR}" \
            --custom-name "${{ matrix.model.display_name }}" \
            --upstream-branch "${{ matrix.model.ref }}" \
            --is-20hz "${{ matrix.model.is_20hz }}" \
            --powersave-script "${POWERSAVE_SCRIPT}" \
            --model-generator "${MODEL_GENERATOR}" \
            --get-model-metadata "${GET_MODEL_METADATA}"
      - run: |
          sudo rsync -avm \
            --include='*.dlc' \
            --include='*.thneed' \
            --include='*.pkl' \
            --include='*.onnx' \
            --exclude='*' \
            --delete-excluded \
            --chown=comma:comma \
            "${MODELS_DIR}/" "${OUTPUT_DIR}/"
      - uses: actions/upload-artifact@v4
        with:
          name: model-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
          path: ${{ env.OUTPUT_DIR }}

  postprocess:
    needs: [setup, build-models]
    runs-on: ${{ github.event.inputs.runner_type }}
    if: needs.build-models.result == 'success'
    steps:
      - name: Download all model artifacts
        uses: actions/download-artifact@v4
        with:
          path: gitlab_docs/models/${{ needs.setup.outputs.RECOMPILED_DIR }}

      - name: Push recompiled dir to GitLab
        env:
          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
        run: |
          cd gitlab_docs
          git checkout main
          mkdir -p models/"$(basename $RECOMPILED_DIR)"
          git add models/"$(basename $RECOMPILED_DIR)"
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"
          git commit -m "Add $(basename $RECOMPILED_DIR) from build-all-tinygrad-models"
          git push origin main

      - name: Update JSON with new models
        run: |
          python3 docs/json_parser.py \
            --json-path "$JSON_FILE" \
            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR"

      - name: Push updated JSON to GitHub docs repo
        run: |
          cd docs
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"
          git checkout gh-pages
          git add docs/"$(basename $JSON_FILE)"
          git commit -m "Update $(basename $JSON_FILE) after recompiling models" || true
          git push origin gh-pages