diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index e845d80c6..dc02d1d8d 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -5,6 +5,8 @@ env: OUTPUT_DIR: ${{ github.workspace }}/output SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache UPSTREAM_REPO: "commaai/openpilot" + TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo + MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models on: workflow_dispatch: @@ -15,35 +17,55 @@ on: default: 'master' type: string custom_name: - description: 'Custom name for the model' - required: false - type: string - file_name: - description: 'File name prefix for the model files' + description: 'Custom name for the model (no date, only name)' required: false type: string is_20hz: description: 'Is this a 20Hz model' required: false type: boolean - default: false + default: true run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}] jobs: - build_model: - runs-on: self-hosted - + get_model: + runs-on: ubuntu-latest + outputs: + model_date: ${{ steps.commit-date.outputs.model_date }} steps: - uses: actions/checkout@v4 with: repository: ${{ env.UPSTREAM_REPO }} ref: ${{ github.event.inputs.upstream_branch }} submodules: recursive + - name: Get commit date + id: commit-date + run: | + # Get the commit date in YYYY-MM-DD format + commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y') + echo "model_date=${commit_date}" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + - run: git lfs pull + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: models + path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx + + build_model: + runs-on: self-hosted + needs: get_model + env: + MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }}) + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive - run: git lfs pull - - name: Cache SCons uses: actions/cache@v4 with: @@ -60,31 +82,63 @@ jobs: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }} scons-${{ runner.os }}-${{ runner.arch }} + - name: Set environment variables + id: set-env + run: | + # Set up common environment + source /etc/profile; + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + printenv >> $GITHUB_ENV + if [[ "${{ runner.debug }}" == "1" ]]; then + cat $GITHUB_OUTPUT + fi + - name: Setup build environment run: | mkdir -p "${BUILD_DIR}/" sudo find $BUILD_DIR/ -mindepth 1 -delete echo "Starting build stage..." - echo "Building from: ${{ env.UPSTREAM_REPO }} branch: ${{ github.event.inputs.upstream_branch }}" - - - name: Patch SConstruct to pass arbitrary cache - run: | - sed -i.bak 's#cache_dir =#default_cache_dir =#' ${{ github.workspace }}/SConstruct - printf '/default_cache_dir/a\\\ncache_dir = ARGUMENTS.get("cache_dir", default_cache_dir)\n' | sed -i.bak -f - ${{ github.workspace }}/SConstruct - cat ${{ github.workspace }}/SConstruct + echo "BUILD_DIR: ${BUILD_DIR}" + echo "CI_DIR: ${CI_DIR}" + echo "VERSION: ${{ steps.set-env.outputs.version }}" + echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}" + echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" + echo "-------" + if [[ "${{ runner.debug }}" == "1" ]]; then + printenv + fi + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + rm -rf ${{ env.MODELS_DIR }}/*.onnx + + - name: Download model artifacts + uses: actions/download-artifact@v4 + with: + name: models + path: ${{ github.workspace }}/selfdrive/modeld/models - name: Build Model run: | source /etc/profile export UV_PROJECT_ENVIRONMENT=${HOME}/venv export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT - scons -j$(nproc) cache_dir=${{ env.SCONS_CACHE_DIR }} ${{ github.workspace }}/selfdrive/modeld + export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}" + + # Loop through all .onnx files + find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do + base_name=$(basename "$onnx_file" .onnx) + output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl" + + echo "Compiling: $onnx_file -> $output_file" + QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file" + QCOM=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true + done - name: Prepare Output run: | - sudo rm -rf ${OUTPUT_DIR} - mkdir -p ${OUTPUT_DIR} - + sudo rm -rf ${{ env.OUTPUT_DIR }} + mkdir -p ${{ env.OUTPUT_DIR }} + # Copy the model files rsync -avm \ --include='*.dlc' \ @@ -94,40 +148,22 @@ jobs: --exclude='*' \ --delete-excluded \ --chown=comma:comma \ - ./selfdrive/modeld/models/ ${OUTPUT_DIR}/ - - # Rename files if file_name is provided - if [ ! -z "${{ inputs.file_name }}" ]; then - mv ${OUTPUT_DIR}/supercombo.thneed ${OUTPUT_DIR}/supercombo-${{ inputs.file_name }}.thneed - mv ${OUTPUT_DIR}/supercombo_metadata.pkl ${OUTPUT_DIR}/supercombo-${{ inputs.file_name }}_metadata.pkl - fi - - # Calculate SHA256 hashes - HASH=$(sha256sum ${OUTPUT_DIR}/supercombo*.thneed | cut -d' ' -f1) - METADATA_HASH=$(sha256sum ${OUTPUT_DIR}/supercombo*_metadata.pkl | cut -d' ' -f1) - - # Create metadata.json - cat > ${OUTPUT_DIR}/metadata.json << EOF - { - "display_name": "${{ inputs.custom_name || inputs.upstream_branch }}", - "full_name": "${{ inputs.file_name || 'default' }}", - "is_20hz": ${{ inputs.is_20hz || false }}, - "files": { - "drive_model": { - "file_name": "$(basename ${OUTPUT_DIR}/supercombo*.thneed)", - "sha256": "${HASH}" - }, - "metadata": { - "file_name": "$(basename ${OUTPUT_DIR}/supercombo*_metadata.pkl)", - "sha256": "${METADATA_HASH}" - } - }, - "ref": "${{ inputs.upstream_branch }}", - "build_time": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" - } + ${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/ + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "${{ env.MODELS_DIR }}" \ + --output-dir "${{ env.OUTPUT_DIR }}" \ + --custom-name "${{ env.MODEL_NAME }}" \ + --upstream-branch "${{ inputs.upstream_branch }}" \ + ${{ inputs.is_20hz && '--is-20hz' || '' }} - name: Upload Build Artifacts uses: actions/upload-artifact@v4 with: - name: model-${{ github.event.inputs.custom_name || github.event.inputs.upstream_branch }}-${{ github.run_number }} + name: model-${{ env.MODEL_NAME }}-${{ github.run_number }} path: ${{ env.OUTPUT_DIR }} + + - name: Re-enable powersave + if: always() + run: | + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py new file mode 100755 index 000000000..069b8e41a --- /dev/null +++ b/release/ci/model_generator.py @@ -0,0 +1,131 @@ +import os +import sys +import hashlib +import json +import re +from pathlib import Path +from datetime import datetime, UTC + + +def create_short_name(full_name): + # Remove parentheses and extract alphanumeric words + clean_name = re.sub(r'\([^)]*\)', '', full_name) + words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)] + + if len(words) == 1: + # If there's only one word, return it as is, lowercased, truncated to 8 characters + truncated = words[0][:8] + return truncated.lower() if truncated.isupper() else truncated + + # Handle special case: Name + Version (e.g., "Word A1" -> "WordA1") + if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]): + first_word = words[0].lower() if words[0].isupper() else words[0] + return (first_word + words[1])[:8] + + # Normal case: first letter and trailing numbers from each word + result = ''.join(word if word.isdigit() else word[0] + (re.search(r'\d+$', word) or [''])[0] for word in words) + return result[:8] + + +def generate_metadata(model_path: Path, output_dir: Path, short_name: str): + model_path = model_path + output_path = output_dir + base = model_path.stem + + # Define output files for tinygrad and metadata + tinygrad_file = output_path / f"{base}_tinygrad.pkl" + metadata_file = output_path / f"{base}_metadata.pkl" + + if not tinygrad_file.exists() or not metadata_file.exists(): + print(f"Error: Missing files for model {base} ({tinygrad_file} or {metadata_file})", file=sys.stderr) + return + + # Calculate the sha256 hashes + with open(tinygrad_file, 'rb') as f: + tinygrad_hash = hashlib.sha256(f.read()).hexdigest() + + with open(metadata_file, 'rb') as f: + metadata_hash = hashlib.sha256(f.read()).hexdigest() + + # Rename the files if a custom file name is provided + if short_name: + tinygrad_file = tinygrad_file.rename(output_path / f"{base}_{short_name.lower()}_tinygrad.pkl") + metadata_file = metadata_file.rename(output_path / f"{base}_{short_name.lower()}_metadata.pkl") + + # Build the metadata structure + model_metadata = { + "type": base.split("_")[-1] if "dmonitoring" not in base else "dmonitoring", + "artifact": { + "file_name": tinygrad_file.name, + "download_uri": { + "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/", + "sha256": tinygrad_hash + } + }, + "metadata": { + "file_name": metadata_file.name, + "download_uri": { + "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/", + "sha256": metadata_hash + } + } + } + + # Return model metadata + return model_metadata + + +def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown"): + metadata_json = { + "short_name": short_name, + "display_name": custom_name or upstream_branch, + "is_20hz": is_20hz, + "ref": upstream_branch, + "environment": "development", + "runner": "tinygrad", + "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "models": models, + "overrides": {}, + "index": -1, + "minimum_selector_version": "-1", + "generation": "-1", + } + + # Write metadata to output_dir + with open(output_dir / "metadata.json", "w") as f: + json.dump(metadata_json, f, indent=2) + + print(f"Generated metadata.json with {len(models)} models.") + + +if __name__ == "__main__": + import argparse + import glob + + parser = argparse.ArgumentParser(description="Generate metadata for model files") + parser.add_argument("--model-dir", default="./models", help="Directory containing ONNX model files") + parser.add_argument("--output-dir", default="./output", help="Output directory for metadata") + parser.add_argument("--custom-name", help="Custom display name for the model") + parser.add_argument("--is-20hz", action="store_true", help="Whether this is a 20Hz model") + parser.add_argument("--upstream-branch", default="unknown", help="Upstream branch name") + args = parser.parse_args() + + # Find all ONNX files in the given directory + model_paths = glob.glob(os.path.join(args.model_dir, "*.onnx")) + if not model_paths: + print(f"No ONNX files found in {args.model_dir}", file=sys.stderr) + sys.exit(1) + + _output_dir = Path(args.output_dir) + _output_dir.mkdir(exist_ok=True, parents=True) + _models = [] + + for _model_path in model_paths: + _model_metadata = generate_metadata(Path(_model_path), _output_dir, create_short_name(args.custom_name)) + if _model_metadata: + _models.append(_model_metadata) + + if _models: + create_metadata_json(_models, _output_dir, args.custom_name, create_short_name(args.custom_name), args.is_20hz, args.upstream_branch) + else: + print("No models processed.", file=sys.stderr)