CI tuneup (#24541)

* little housekeeping * all the commits * one more cars
2026-02-18 21:14:01 +08:00 · 2022-05-15 17:57:34 -07:00
parent 6a48a7acef
commit b2569d871f
5 changed files with 64 additions and 88 deletions
--- a/.github/workflows/prebuilt.yaml
+++ b/.github/workflows/prebuilt.yaml
@@ -30,7 +30,7 @@ jobs:
        ref: master
        wait-interval: 30
        running-workflow-name: 'build prebuilt'
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -26,10 +26,11 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
-      STRIPPED_DIR: tmppilot
+      STRIPPED_DIR: /tmp/releasepilot
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
+        fetch-depth: 0
        submodules: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
@@ -46,16 +47,10 @@ jobs:
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
-    - name: Strip non-release files
+    - name: Build devel
      run: |
-        mkdir $STRIPPED_DIR
-        cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
-        cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
-        cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
+        TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
        cp Dockerfile.openpilot_base $STRIPPED_DIR
-
-        # need this to build on x86
-        cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
@@ -63,32 +58,43 @@ jobs:
          cd $STRIPPED_DIR
          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
                          python -m unittest discover selfdrive/car"
-    - name: Cleanup scons cache
-      run: |
-          cd $STRIPPED_DIR
-          ${{ env.RUN }} "scons -j$(nproc) && \
-                          rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"

  build_all:
    name: build all
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
+    - name: Cache scons
+      id: scons-cache
+      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
+      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      env:
+        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
+        restore-keys: |
+          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot with all flags
      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"
+    - name: Cleanup scons cache
+      run: |
+          ${{ env.RUN }} "scons -j$(nproc) --extras --test && \
+                          rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"

  #build_mac:
  #  name: build macos
  #  runs-on: macos-latest
  #  timeout-minutes: 60
  #  steps:
-  #  - uses: actions/checkout@v2
+  #  - uses: actions/checkout@v3
  #    with:
  #      submodules: true
  #  - name: Determine pre-existing Homebrew packages
@@ -147,9 +153,21 @@ jobs:
    env:
      IMAGE_NAME: openpilotwebcamci
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
+    - name: Cache scons
+      id: scons-cache
+      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
+      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      env:
+        CACHE_SKIP_SAVE: true
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+        restore-keys: |
+          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+          scons-
    - name: Build Docker image
      run: |
        eval "$BUILD"
@@ -170,7 +188,7 @@ jobs:
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
@@ -185,20 +203,20 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
-      run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"
+      run: ${{ env.RUN }} "pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
@@ -241,7 +259,7 @@ jobs:
      run: echo $TIMESTAMP
      env:
        TIMESTAMP: ${{ steps.date.outputs.time }}
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
@@ -289,7 +307,7 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
@@ -334,27 +352,12 @@ jobs:
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

-  #model_replay:
-  #  name: model replay
-  #  runs-on: ubuntu-20.04
-  #  timeout-minutes: 50
-  #  steps:
-  #  - uses: actions/checkout@v2
-  #    with:
-  #      submodules: true
-  #  - name: Build Docker image
-  #    run: eval "$BUILD"
-  #  - name: Run replay
-  #    run: |
-  #      ${{ env.RUN }} "scons -j$(nproc) && \
-  #                      selfdrive/test/process_replay/model_replay.py"
-
  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache scons
@@ -394,9 +397,9 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        job: [0, 1, 2, 3]
+        job: [0, 1, 2, 3, 4]
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Cache dependencies
@@ -426,7 +429,7 @@ jobs:
                        coverage xml && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
-        NUM_JOBS: 4
+        NUM_JOBS: 5
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
@@ -436,7 +439,7 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build docker container
--- a/.github/workflows/tools_tests.yaml
+++ b/.github/workflows/tools_tests.yaml
@@ -21,7 +21,7 @@ jobs:
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
@@ -42,28 +42,16 @@ jobs:
      IMAGE_NAME: openpilot-sim
    if: github.repository == 'commaai/openpilot'
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
+        lfs: true
        submodules: true
-
-    # HACK: cache LFS objects since they count against our quota
-    # https://github.com/actions/checkout/issues/165#issuecomment-657673315
-    - name: Create LFS file list
-      run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
-    - name: Restore LFS cache
-      uses: actions/cache@v2
-      id: lfs-cache
-      with:
-        path: .git/lfs
-        key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}
-    - name: Git LFS Pull
-      run: git lfs pull
-
-    - name: Build Docker image
-      run: |
-        eval "$BUILD"
-        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
-        docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
+    - name: Build base image
+      run: eval "$BUILD"
+    - name: Pull latest simulator image
+      run: eval "$BUILD"docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
+    - name: Build simulator image
+      run: docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
    - name: Push to container registry
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
--- a/21
+++ b/21
@@ -80,27 +80,6 @@ pipeline {
          stages {
            stage('parallel tests') {
              parallel {
-                /*
-                stage('Power Consumption Tests') {
-                  steps {
-                    lock(resource: "", label: "c2-zookeeper", inversePrecedence: true, variable: 'device_ip', quantity: 1) {
-                      timeout(time: 90, unit: 'MINUTES') {
-                        sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
-                        phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
-                        phone(device_ip, "build", "scons -j4 && sync")
-                        sh script: "/home/batman/tools/zookeeper/disable.py $device_ip", label: "turn off device"
-                        sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
-                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 3", label: "idle power consumption after boot"
-                        sh script: "/home/batman/tools/zookeeper/ignition.py 1", label: "go onroad"
-                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 10", label: "onroad power consumption"
-                        sh script: "/home/batman/tools/zookeeper/ignition.py 0", label: "go offroad"
-                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 2", label: "idle power consumption offroad"
-                      }
-                    }
-                  }
-                }
-                */
-
                stage('build') {
                  environment {
                    R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
--- a/release/files_pc
+++ b/release/files_pc
@@ -1,3 +1,9 @@
+selfdrive/modeld/runners/onnx*
+
 third_party/mapbox-gl-native-qt/x86_64/*.so

 third_party/qt-plugins/x86_64/geoservices/*.so
+
+third_party/libyuv/x64/**
+third_party/snpe/x86_64/**
+third_party/snpe/x86_64-linux-clang/**