CI: Actions cleanup + speedup (#25514)

* actions cache cleanup * release build cleanup * fetch dpeth old-commit-hash: 71e76c3d0f
2026-02-18 23:33:58 +08:00 · 2022-08-21 19:14:36 -07:00
parent f68770eb3a
commit fca5728055
4 changed files with 88 additions and 252 deletions
--- a/.github/workflows/badges.yaml
+++ b/.github/workflows/badges.yaml
@@ -7,12 +7,7 @@ on:
 env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai
-
-  BUILD: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c

 jobs:
  badges:
@@ -23,23 +18,7 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-
-    - name: Build Docker image
-      run: eval "$BUILD"
-
+    - uses: ./.github/workflows/setup
    - name: Push badges
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -17,7 +17,7 @@ env:
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c

  BUILD_CL: |
      docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
@@ -27,11 +27,10 @@ env:
  UNIT_TEST: coverage run --append -m unittest discover

 jobs:
-  # TODO: once actions/cache supports read only mode, use the cache for all jobs
  build_release:
    name: build release
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 30
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
@@ -39,44 +38,29 @@ jobs:
      with:
        fetch-depth: 0
        submodules: true
-    - name: Pull LFS
-      run: git lfs pull
+    - name: Build devel
+      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
+    - uses: ./.github/workflows/setup
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build devel
+    - name: Build openpilot and run checks
      run: |
-        TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-        cp Dockerfile.openpilot_base $STRIPPED_DIR
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py"
+    - name: Run tests
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh && \
+                        python -m unittest discover selfdrive/car"
+    - name: pre-commit
+      run: |
+        cd $GITHUB_WORKSPACE
        cp .pre-commit-config.yaml $STRIPPED_DIR
        cp .pylintrc $STRIPPED_DIR
        cp mypy.ini $STRIPPED_DIR
-    - name: Build Docker image
-      run: |
-          eval "$BUILD"
-          rm $STRIPPED_DIR/Dockerfile.openpilot_base
-    - name: Build openpilot and run checks
-      run: |
-          cd $STRIPPED_DIR
-          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
-                          pre-commit run --all && \
-                          rm .pre-commit-config.yaml && \
-                          rm .pylintrc && \
-                          rm mypy.ini && \
-                          release/check-dirty.sh && \
-                          python -m unittest discover selfdrive/car"
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "pre-commit run --all"

  build_all:
    name: build all
@@ -86,26 +70,14 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
+    - uses: ./.github/workflows/setup
      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build Docker image
-      run: eval "$BUILD"
+        save-cache: true
    - name: Build openpilot with all flags
      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test && release/check-dirty.sh"
    - name: Cleanup scons cache
      run: |
-          ${{ env.RUN }} "scons -j$(nproc) --extras --test && \
-                          rm -rf /tmp/scons_cache/* && \
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --extras --test --cache-populate"

  #build_mac:
@@ -175,21 +147,9 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
+    - uses: ./.github/workflows/setup
    - name: Build Docker image
      run: |
-        eval "$BUILD"
        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
        docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
    - name: Build openpilot
@@ -244,30 +204,11 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache dependencies
-      id: dependency-cache
-      uses: actions/cache@v2
-      with:
-        path: /tmp/comma_download_cache
-        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build Docker image
-      run: eval "$BUILD"
+    - uses: ./.github/workflows/setup
    - name: Run valgrind
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
-                        FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
+                        python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt
@@ -277,28 +218,10 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - name: Get current date
-      id: date
-      run: echo "::set-output name=time::$(date +'%s')"
-    - name: Output timestamp
-      run: echo $TIMESTAMP
-      env:
-        TIMESTAMP: ${{ steps.date.outputs.time }}
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
-        restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-    - name: Build Docker image
-      run: eval "$BUILD"
+    - uses: ./.github/workflows/setup
    - name: Run unit tests
      run: |
        ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
@@ -339,30 +262,17 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache dependencies
+    - uses: ./.github/workflows/setup
+    - name: Cache test routes
      id: dependency-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: /tmp/comma_download_cache
-        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build Docker image
-      run: eval "$BUILD"
+        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
    - name: Run replay
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
-                        FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        coverage xml"
    - name: Print diff
      if: always()
@@ -389,35 +299,16 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Pull LFS
-      run: git lfs pull
-    - name: Cache dependencies
-      id: dependency-cache
-      uses: actions/cache@v2
-      with:
-        path: /tmp/comma_download_cache
-        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/model_replay.py') }}
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
+    - uses: ./.github/workflows/setup
    - name: Build Docker image
-      # Sim docker is needed to get the intel OPENCL drivers
+      # Sim docker is needed to get the OpenCL drivers
      run: eval "$BUILD_CL"
    - name: Run replay
      run: |
        ${{ env.RUN_CL }} "scons -j$(nproc) && \
-                            ONNXCPU=1 FILEREADER_CACHE=1 CI=1 coverage run \
-                            selfdrive/test/process_replay/model_replay.py -j$(nproc) && \
-                            coverage xml"
+                           ONNXCPU=1 CI=1 coverage run \
+                           selfdrive/test/process_replay/model_replay.py -j$(nproc) && \
+                           coverage xml"

  test_longitudinal:
    name: longitudinal
@@ -427,20 +318,7 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build Docker image
-      run: eval "$BUILD"
+    - uses: ./.github/workflows/setup
    - name: Test longitudinal
      run: |
        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
@@ -469,30 +347,17 @@ jobs:
    - uses: actions/checkout@v3
      with:
        submodules: true
-    - name: Cache dependencies
+    - uses: ./.github/workflows/setup
+    - name: Cache test routes
      id: dependency-cache
-      uses: actions/cache@v2
+      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      with:
        path: /tmp/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
-    - name: Build Docker image
-      run: eval "$BUILD"
    - name: Test car models
      run: |
        ${{ env.RUN }} "scons -j$(nproc) --test && \
-                        FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \
+                        coverage run -m pytest selfdrive/car/tests/test_models.py && \
                        coverage xml && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
@@ -530,20 +395,7 @@ jobs:
        with:
          submodules: true
          ref: ${{ github.event.pull_request.base.ref }}
-      - name: Cache scons
-        id: scons-cache
-        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-        env:
-          CACHE_SKIP_SAVE: true
-        with:
-          path: /tmp/scons_cache
-          key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          restore-keys: |
-            scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-            scons-
-      - name: Build Docker image
-        run: eval "$BUILD"
+      - uses: ./.github/workflows/setup
      - name: Get base car info
        run: |
          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -0,0 +1,45 @@
+name: 'openpilot env setup'
+
+env:
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
+  BUILD: |
+      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
+      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
+      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+
+inputs:
+  save-cache:
+    default: false
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
+    - shell: bash
+      run: git lfs pull
+
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "::set-output name=date::$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d')"
+    - shell: bash
+      run: echo "${{ steps.date.outputs.date }}"
+    - shell: bash
+      run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
+      if: github.ref != 'refs/heads/master' || inputs.save-cache == 'false'
+    - id: scons-cache
+      # TODO: change the version to the released version
+      # when https://github.com/actions/cache/pull/489 (or 571) is merged.
+      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ steps.date.outputs.date }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ steps.date.outputs.date }}-
+          scons-
+
+    # build our docker image
+    - shell: bash
+      run: eval "$BUILD"
--- a/.github/workflows/tools_tests.yaml
+++ b/.github/workflows/tools_tests.yaml
@@ -21,46 +21,6 @@ env:
       GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c

 jobs:
-  build_latest_ubuntu:
-    name: build latest ubuntu
-    runs-on: ubuntu-20.04
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - name: Cache pyenv
-      id: ubuntu-latest-pyenv
-      uses: actions/cache@v3
-      with:
-        path: |
-          ~/.pyenv
-          ~/.local/share/virtualenvs/
-        key: ubuntu-latest-python-${{ hashFiles('tools/ubuntu_setup.sh') }}-
-    - name: Cache scons
-      id: ubuntu-latest-scons
-      uses: actions/cache@v3
-      with:
-        path: /tmp/scons_cache
-        key: ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}-
-        restore-keys: |
-          ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}-
-          ubuntu-latest-scons-
-
-    - name: tools/ubuntu_setup.sh
-      run: |
-        source tools/openpilot_env.sh
-        tools/ubuntu_setup.sh
-    - name: Build openpilot
-      run: |
-        source tools/openpilot_env.sh
-        pipenv run scons -j$(nproc) --extras --test
-    - name: Cleanup scons cache
-      run: |
-        source tools/openpilot_env.sh
-        rm -rf /tmp/scons_cache/*
-        pipenv run scons -j$(nproc) --extras --test --cache-populate
-
  plotjuggler:
    name: plotjuggler
    runs-on: ubuntu-20.04