From 3ac9208364c85b6ad651630cc85c52bf817e9d6e Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Mon, 18 Nov 2024 21:16:15 -0800 Subject: [PATCH] ci: increase timeout on cache misses + switch some caches to github (#34056) * cache * test ns * try this * try * try now? * bp * bp agian * fix * remove * test * try * fix * fix * regen cache * fix --- .github/workflows/auto-cache/action.yaml | 9 +++++++ .github/workflows/selfdrive_tests.yaml | 24 +++++-------------- .../workflows/setup-with-retry/action.yaml | 3 --- .github/workflows/setup/action.yaml | 17 ------------- 4 files changed, 15 insertions(+), 38 deletions(-) diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml index e99c46e3c3..fadd422b6e 100644 --- a/.github/workflows/auto-cache/action.yaml +++ b/.github/workflows/auto-cache/action.yaml @@ -14,17 +14,25 @@ inputs: description: 'whether to save the cache' default: 'false' required: false +outputs: + cache-hit: + description: 'cache hit occurred' + value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }} runs: using: "composite" steps: - name: setup namespace cache + id: ns-cache if: ${{ contains(runner.name, 'nsc') }} uses: namespacelabs/nscloud-cache-action@v1 with: path: ${{ inputs.path }} - name: setup github cache + id: gha-cache if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }} uses: 'actions/cache@v4' with: @@ -33,6 +41,7 @@ runs: restore-keys: ${{ inputs.restore-keys }} - name: setup github cache + id: gha-cache-ro if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }} uses: 'actions/cache/restore@v4' with: diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 32f2ab306e..ec40660f15 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -95,10 +95,8 @@ jobs: echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV" $DOCKER_LOGIN - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - uses: ./.github/workflows/compile-openpilot - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache + timeout-minutes: 30 build_mac: name: build macOS @@ -168,16 +166,8 @@ jobs: with: submodules: true - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - name: Build openpilot - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache run: ${{ env.RUN }} "scons -j$(nproc)" - - name: Setup cache - uses: ./.github/workflows/auto-cache - with: - path: .ci_cache/comma_download_cache - key: unit_tests_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }} - name: Run unit tests timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }} run: | @@ -203,19 +193,17 @@ jobs: with: submodules: true - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - name: Cache test routes id: dependency-cache uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache - key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }} + key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }} - name: Build openpilot run: | ${{ env.RUN }} "scons -j$(nproc)" - name: Run replay - timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }} + timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }} run: | ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ chmod -R 777 /tmp/comma_download_cache && \ @@ -263,15 +251,15 @@ jobs: submodules: true - uses: ./.github/workflows/setup-with-retry - name: Cache test routes - id: dependency-cache - uses: ./.github/workflows/auto-cache + id: routes-cache + uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }} - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" - name: Test car models - timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }} + timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 20 }} run: | ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \ chmod -R 777 /tmp/comma_download_cache" diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml index 369ac45c9a..ad297403cf 100644 --- a/.github/workflows/setup-with-retry/action.yaml +++ b/.github/workflows/setup-with-retry/action.yaml @@ -17,7 +17,6 @@ runs: uses: ./.github/workflows/setup continue-on-error: true with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true - if: steps.setup1.outcome == 'failure' shell: bash @@ -27,7 +26,6 @@ runs: uses: ./.github/workflows/setup continue-on-error: true with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true - if: steps.setup2.outcome == 'failure' shell: bash @@ -36,5 +34,4 @@ runs: if: steps.setup2.outcome == 'failure' uses: ./.github/workflows/setup with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml index 1315ba989e..935db34154 100644 --- a/.github/workflows/setup/action.yaml +++ b/.github/workflows/setup/action.yaml @@ -1,10 +1,6 @@ name: 'openpilot env setup' inputs: - docker_hub_pat: - description: 'Auth token for Docker Hub, required for BuildJet jobs' - required: true - default: '' is_retried: description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' required: false @@ -36,19 +32,6 @@ runs: - shell: bash run: git lfs pull - # on BuildJet runners, must be logged into DockerHub to avoid rate limiting - # https://buildjet.com/for-github-actions/docs/guides/docker - - shell: bash - if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }} - run: | - echo "Need to set the Docker Hub PAT secret as an input to this action" - exit 1 - - name: Login to Docker Hub - if: contains(runner.name, 'buildjet') - shell: bash - run: | - docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }} - # build cache - id: date shell: bash