mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-02-18 17:43:54 +08:00
373 lines
16 KiB
YAML
373 lines
16 KiB
YAML
name: sunnypilot prebuilt action
|
||
|
||
env:
|
||
BUILD_DIR: "/data/openpilot"
|
||
OUTPUT_DIR: ${{ github.workspace }}/output
|
||
CI_DIR: ${{ github.workspace }}/release/ci
|
||
SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
|
||
PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
|
||
|
||
# Branch configurations
|
||
STAGING_SOURCE_BRANCH: 'master'
|
||
|
||
# Runtime configuration
|
||
SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
|
||
|
||
on:
|
||
push:
|
||
branches: [ master, master-dev ]
|
||
tags: [ 'release/*' ]
|
||
pull_request_target:
|
||
types: [ labeled ]
|
||
workflow_dispatch:
|
||
inputs:
|
||
wait_for_tests:
|
||
description: 'Wait for tests to finish'
|
||
required: false
|
||
type: boolean
|
||
default: false
|
||
|
||
jobs:
|
||
prepare_strategy:
|
||
runs-on: ubuntu-24.04
|
||
if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||
outputs:
|
||
environment: ${{ steps.strategy.outputs.environment }}
|
||
new_branch: ${{ steps.strategy.outputs.new_branch }}
|
||
extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
|
||
version: ${{ steps.strategy.outputs.version }}
|
||
cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
|
||
publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
|
||
is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
|
||
build: ${{ steps.strategy.outputs.build }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
- name: Extract deploy strategy
|
||
id: strategy
|
||
run: |
|
||
echo '::group::Strategy Extraction'
|
||
BRANCH="${{ github.head_ref || github.ref_name }}"
|
||
echo "Current branch: $BRANCH"
|
||
|
||
STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
|
||
CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
|
||
.configs[] | select(.branch == $branch)
|
||
')
|
||
|
||
BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
|
||
if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
|
||
echo "No exact strategy match found. Falling back to feature/fork logic."
|
||
IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
|
||
FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
|
||
NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
|
||
|
||
echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
|
||
echo "version=$BUILD" >> $GITHUB_OUTPUT
|
||
echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
|
||
echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
|
||
echo "environment=feature-branch" >> $GITHUB_OUTPUT
|
||
echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "Matched config: $CONFIG"
|
||
environment=$(echo "$CONFIG" | jq -r '.environment')
|
||
echo "environment=$environment" >> $GITHUB_OUTPUT
|
||
echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
|
||
cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
|
||
echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
|
||
echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
|
||
|
||
is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
|
||
echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
|
||
|
||
stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
|
||
echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
|
||
echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
|
||
fi
|
||
echo "build=$BUILD" >> $GITHUB_OUTPUT
|
||
cat $GITHUB_OUTPUT
|
||
|
||
validate_tests:
|
||
runs-on: ubuntu-24.04
|
||
needs: [ prepare_strategy ]
|
||
if: ${{
|
||
((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
|
||
(github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
|
||
contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||
}}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
- name: Wait for Tests
|
||
uses: ./.github/workflows/wait-for-action # Path to where you place the action
|
||
with:
|
||
workflow: tests.yaml # The workflow file to monitor
|
||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||
should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
|
||
|
||
build:
|
||
needs: [ validate_tests, prepare_strategy ]
|
||
concurrency:
|
||
group: build-${{ github.head_ref || github.ref_name }}
|
||
cancel-in-progress: false
|
||
runs-on: [self-hosted, tici]
|
||
outputs:
|
||
new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
|
||
version: ${{ needs.prepare_strategy.outputs.version }}
|
||
extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
|
||
commit_sha: ${{ github.sha }}
|
||
if: ${{
|
||
(always() && !cancelled() && !failure()) &&
|
||
needs.prepare_strategy.result == 'success' &&
|
||
(needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
|
||
(!contains(github.event_name, 'pull_request') ||
|
||
(github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||
}}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
submodules: recursive
|
||
ref: ${{ env.SOURCE_BRANCH }}
|
||
repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
|
||
- run: git lfs pull
|
||
|
||
- name: Cache SCons
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: ${{env.SCONS_CACHE_DIR}}
|
||
key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
|
||
# Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
|
||
# for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
|
||
restore-keys: |
|
||
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
|
||
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
|
||
scons-${{ runner.os }}-${{ runner.arch }}
|
||
|
||
- name: Set environment variables
|
||
id: set-env
|
||
run: |
|
||
echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
|
||
echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
|
||
echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
|
||
echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
|
||
|
||
# Set up common environment
|
||
source /etc/profile;
|
||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||
printenv >> $GITHUB_ENV
|
||
if [[ "${{ runner.debug }}" == "1" ]]; then
|
||
cat $GITHUB_OUTPUT
|
||
fi
|
||
|
||
- name: Setup build environment
|
||
run: |
|
||
mkdir -p "${BUILD_DIR}/"
|
||
sudo find $BUILD_DIR/ -mindepth 1 -delete
|
||
echo "Starting build stage..."
|
||
echo "BUILD_DIR: ${BUILD_DIR}"
|
||
echo "CI_DIR: ${CI_DIR}"
|
||
echo "VERSION: ${{ steps.set-env.outputs.version }}"
|
||
echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
|
||
echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
|
||
echo "-------"
|
||
if [[ "${{ runner.debug }}" == "1" ]]; then
|
||
printenv
|
||
fi
|
||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||
|
||
- name: Build Main Project
|
||
run: |
|
||
export PYTHONPATH="$BUILD_DIR"
|
||
./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
|
||
cd $BUILD_DIR
|
||
sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
|
||
echo "Building sunnypilot's modeld..."
|
||
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld
|
||
echo "Building sunnypilot's modeld_v2..."
|
||
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
|
||
echo "Building sunnypilot's locationd..."
|
||
scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
|
||
echo "Building openpilot's locationd..."
|
||
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
|
||
echo "Building rest of sunnypilot"
|
||
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
|
||
touch ${BUILD_DIR}/prebuilt
|
||
if [[ "${{ runner.debug }}" == "1" ]]; then
|
||
ls -la ${BUILD_DIR}
|
||
fi
|
||
|
||
- name: Prepare Output
|
||
run: |
|
||
sudo rm -rf ${OUTPUT_DIR}
|
||
mkdir -p ${OUTPUT_DIR}
|
||
rsync -am${RUNNER_DEBUG:+v} \
|
||
--exclude='.sconsign.dblite' \
|
||
--exclude='*.a' \
|
||
--exclude='*.o' \
|
||
--exclude='*.os' \
|
||
--exclude='*.pyc' \
|
||
--exclude='moc_*' \
|
||
--exclude='__pycache__' \
|
||
--exclude='Jenkinsfile' \
|
||
--exclude='**/release/' \
|
||
--exclude='**/.github/' \
|
||
--exclude='**/selfdrive/ui/replay/' \
|
||
--exclude='**/__pycache__/' \
|
||
--exclude='${{env.SCONS_CACHE_DIR}}' \
|
||
--exclude='**/.git/' \
|
||
--exclude='**/SConstruct' \
|
||
--exclude='**/SConscript' \
|
||
--exclude='**/.venv/' \
|
||
--exclude='selfdrive/modeld/models/driving_vision.onnx' \
|
||
--exclude='selfdrive/modeld/models/driving_policy.onnx' \
|
||
--exclude='sunnypilot/modeld*/models/supercombo.onnx' \
|
||
--exclude='third_party/*x86*' \
|
||
--exclude='third_party/*Darwin*' \
|
||
--delete-excluded \
|
||
--chown=comma:comma \
|
||
${BUILD_DIR}/ ${OUTPUT_DIR}/
|
||
|
||
- name: 'Tar.gz files'
|
||
run: |
|
||
tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
|
||
ls -la prebuilt.tar.gz
|
||
|
||
- name: 'Upload Artifact'
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: prebuilt
|
||
path: prebuilt.tar.gz
|
||
|
||
- name: Re-enable powersave
|
||
if: always()
|
||
run: |
|
||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||
|
||
|
||
publish:
|
||
concurrency:
|
||
# We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
|
||
# This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
|
||
# Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
|
||
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
|
||
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
|
||
if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
|
||
needs: [ build, prepare_strategy ]
|
||
runs-on: ubuntu-24.04
|
||
environment: ${{ needs.prepare_strategy.outputs.environment }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Download build artifacts
|
||
uses: actions/download-artifact@v4
|
||
with:
|
||
name: prebuilt
|
||
|
||
- name: Untar prebuilt
|
||
run: |
|
||
mkdir -p ${{ env.OUTPUT_DIR }}
|
||
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
|
||
|
||
- name: Configure Git
|
||
run: |
|
||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||
git config --global user.name "github-actions[bot]"
|
||
|
||
- name: Publish to Public Repository
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
run: |
|
||
echo '${{ toJSON(needs.build.outputs) }}'
|
||
ls -la ${{ env.OUTPUT_DIR }}
|
||
|
||
${{ env.CI_DIR }}/publish.sh \
|
||
"${{ github.workspace }}" \
|
||
"${{ env.OUTPUT_DIR }}" \
|
||
"${{ needs.build.outputs.new_branch }}" \
|
||
"${{ needs.build.outputs.version }}" \
|
||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||
"${{ needs.build.outputs.extra_version_identifier }}"
|
||
|
||
echo ""
|
||
echo "---- ℹ️ To update the list of branches that auto deploy prebuilts -----"
|
||
echo ""
|
||
echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
|
||
echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
|
||
echo "3. Update as needed (JSON array with no spaces)"
|
||
|
||
- name: Tag ${{ needs.prepare_strategy.outputs.environment }}
|
||
if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
|
||
run: |
|
||
TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
|
||
git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
|
||
git push -f origin ${TAG}
|
||
|
||
notify:
|
||
needs:
|
||
- prepare_strategy
|
||
- build
|
||
- publish
|
||
runs-on: ubuntu-24.04
|
||
if: ${{ (always() && !cancelled() && !failure())
|
||
&& needs.publish.result == 'success'
|
||
&& (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||
&& (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Prepare notification message
|
||
id: message
|
||
run: |
|
||
TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
|
||
export VERSION="${{ needs.prepare_strategy.outputs.version }}"
|
||
export branch_name="${{ env.SOURCE_BRANCH }}"
|
||
export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
|
||
export commit_sha="${{ github.sha }}"
|
||
export commit_short_sha="${{ github.sha }}"
|
||
export commit_short_sha="${commit_short_sha:0:7}"
|
||
export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
|
||
export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
|
||
|
||
MESSAGE=$(cat << 'EOF' | envsubst
|
||
${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
|
||
EOF
|
||
)
|
||
|
||
{
|
||
echo 'content<<EOFMARKER'
|
||
echo "$MESSAGE"
|
||
echo 'EOFMARKER'
|
||
} >> $GITHUB_OUTPUT
|
||
shell: bash
|
||
|
||
- name: Post to Discourse
|
||
uses: ./.github/workflows/post-to-discourse
|
||
with:
|
||
discourse-url: ${{ vars.DISCOURSE_URL }}
|
||
api-key: ${{ secrets.DISCOURSE_API_KEY }}
|
||
api-username: "system"
|
||
topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
|
||
message: ${{ steps.message.outputs.content }}
|
||
|
||
manage-pr-labels:
|
||
name: Remove prebuilt label
|
||
runs-on: ubuntu-latest
|
||
if: (always() && contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||
env:
|
||
LABEL: prebuilt
|
||
steps:
|
||
- name: Remove trust-fork-pr label if present
|
||
uses: actions/github-script@v7
|
||
with:
|
||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||
script: |
|
||
const prNumber = context.payload.pull_request.number;
|
||
|
||
await github.rest.issues.removeLabel({
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
issue_number: prNumber,
|
||
name: process.env.LABEL
|
||
});
|
||
|
||
console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);
|