Quick GC pass heading into 2025 (#34330)

* first pass

* bye bye snpe

scripts/build_small.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-# git clone --mirror
-SRC=/tmp/openpilot.git/
-OUT=/tmp/smallpilot/
-
-echo "starting size $(du -hs .git/)"
-
-rm -rf $OUT
-
-cd $SRC
-git remote update
-
-# copy contents
-#rsync -a --exclude='.git/' $DIR $OUT
-
-cp -r $SRC $OUT
-
-cd $OUT
-
-# remove all tags
-git tag -l | xargs git tag -d
-
-# remove non-master branches
-BRANCHES="release2 release3 devel master-ci nightly"
-for branch in $BRANCHES; do
-  git branch -D $branch
-  git branch -D ${branch}-staging || true
-done
-
-#git gc
-git reflog expire --expire=now --all
-git gc --prune=now
-git gc --aggressive --prune=now
-echo "new one is $(du -hs .)"

scripts/code_stats.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import os
-import ast
-import stat
-import subprocess
-
-fouts = {x.decode('utf-8') for x in subprocess.check_output(['git', 'ls-files']).strip().split()}
-
-pyf = []
-for d in ["cereal", "common", "scripts", "selfdrive", "tools"]:
-  for root, _, files in os.walk(d):
-    for f in files:
-      if f.endswith(".py"):
-        pyf.append(os.path.join(root, f))
-
-imps: set[str] = set()
-
-class Analyzer(ast.NodeVisitor):
-  def visit_Import(self, node):
-    for alias in node.names:
-      imps.add(alias.name)
-    self.generic_visit(node)
-
-  def visit_ImportFrom(self, node):
-    imps.add(node.module)
-    self.generic_visit(node)
-
-tlns = 0
-carlns = 0
-scriptlns = 0
-testlns = 0
-for f in sorted(pyf):
-  if f not in fouts:
-    continue
-  xbit = bool(os.stat(f)[stat.ST_MODE] & stat.S_IXUSR)
-  src = open(f).read()
-  lns = len(src.split("\n"))
-  tree = ast.parse(src)
-  Analyzer().visit(tree)
-  print(f"{lns:5d} {f} {xbit}")
-  if 'test' in f:
-    testlns += lns
-  elif f.startswith(('tools/', 'scripts/', 'selfdrive/debug')):
-    scriptlns += lns
-  elif f.startswith('selfdrive/car'):
-    carlns += lns
-  else:
-    tlns += lns
-
-print(f"{tlns} lines of openpilot python")
-print(f"{carlns} lines of car ports")
-print(f"{scriptlns} lines of tools/scripts/debug")
-print(f"{testlns} lines of tests")
-#print(sorted(list(imps)))

scripts/count_cars.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-from collections import Counter
-from pprint import pprint
-
-from opendbc.car.docs import get_all_car_docs
-
-if __name__ == "__main__":
-  cars = get_all_car_docs()
-  make_count = Counter(l.make for l in cars)
-  print("\n", "*" * 20, len(cars), "total", "*" * 20, "\n")
-  pprint(make_count)

scripts/git_rewrite/rewrite-git-history.sh

@@ -1,391 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SRC=/tmp/openpilot/
-SRC_CLONE=/tmp/openpilot-clone/
-OUT=/tmp/openpilot-tiny/
-
-REWRITE_IGNORE_BRANCHES=(
-  dashcam3
-  devel
-  master-ci
-  nightly
-  release2
-  release3
-  release3-staging
-)
-
-VALIDATE_IGNORE_FILES=(
-  ".github/ISSUE_TEMPLATE/bug_report.md"
-  ".github/pull_request_template.md"
-)
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-LOGS_DIR=$DIR/git-rewrite-$(date +"%Y-%m-%dT%H:%M:%S%z")
-mkdir -p $LOGS_DIR
-
-GIT_REWRITE_LOG=$LOGS_DIR/git-rewrite-log.txt
-BRANCH_DIFF_LOG=$LOGS_DIR/branch-diff-log.txt
-COMMIT_DIFF_LOG=$LOGS_DIR/commit-diff-log.txt
-
-START_TIME=$(date +%s)
-exec > >(while IFS= read -r line; do
-  CURRENT_TIME=$(date +%s)
-  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
-  echo "[${ELAPSED_TIME}s] $line"
-done | tee -a "$GIT_REWRITE_LOG") 2>&1
-
-# INSTALL git-filter-repo
-if [ ! -f /tmp/git-filter-repo ]; then
-  echo "Installing git-filter-repo..."
-  curl -sSo /tmp/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo
-  chmod +x /tmp/git-filter-repo
-fi
-
-# MIRROR openpilot
-if [ ! -d $SRC ]; then
-  echo "Mirroring openpilot..."
-  git clone --mirror https://github.com/commaai/openpilot.git $SRC # 4.18 GiB (488034 objects)
-
-  cd $SRC
-
-  echo "Starting size $(du -sh .)"
-
-  git remote update
-
-  # the git-filter-repo analysis is bliss - can be found in the repo root/filter-repo/analysis
-  echo "Analyzing with git-filter-repo..."
-  /tmp/git-filter-repo --force --analyze
-
-  echo "Pushing to openpilot-archive..."
-  # push to archive repo - in smaller parts because the 2 GB push limit - https://docs.github.com/en/get-started/using-git/troubleshooting-the-2-gb-push-limit
-  ARCHIVE_REPO=git@github.com:commaai/openpilot-archive.git
-  git push --prune $ARCHIVE_REPO +refs/heads/master:refs/heads/master # push master first so it's the default branch (when openpilot-archive is an empty repo)
-  git push --prune $ARCHIVE_REPO +refs/heads/*:refs/heads/* # 956.39 MiB (110725 objects)
-  git push --prune $ARCHIVE_REPO +refs/tags/*:refs/tags/* # 1.75 GiB (21694 objects)
-  # git push --mirror $ARCHIVE_REPO || true # fails to push refs/pull/* (deny updating a hidden ref) for pull requests
-  # we fail and continue - more reading: https://stackoverflow.com/a/34266401/639708 and https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
-fi
-
-# REWRITE master and tags
-if [ ! -d $SRC_CLONE ]; then
-  echo "Cloning $SRC..."
-  GIT_LFS_SKIP_SMUDGE=1 git clone $SRC $SRC_CLONE
-
-  cd $SRC_CLONE
-
-  echo "Checking out old history..."
-
-  git checkout tags/v0.7.1 > /dev/null 2>&1
-  # checkout as main, since we need master ref later
-  git checkout -b main
-
-  echo "Creating setup commits..."
-
-  # rm these so we don't get conflicts later
-  git rm -r cereal opendbc panda selfdrive/ui/ui > /dev/null
-  git commit -m "removed conflicting files" > /dev/null
-
-  # skip-smudge to get rid of some lfs errors that it can't find the reference of some lfs files
-  # we don't care about fetching/pushing lfs right now
-  git lfs install --skip-smudge --local
-
-  # squash initial setup commits
-  git cherry-pick -n -X theirs 6c33a5c..59b3d06 > /dev/null
-  git commit -m "switching to master" > /dev/null
-
-  # squash the two commits
-  git reset --soft HEAD~2
-  git commit -m "switching to master" -m "$(git log --reverse --format=%B 6c33a5c..59b3d06)" -m "removed conflicting files" > /dev/null
-
-  # get commits we want to cherry-pick
-  # will start with the next commit after #59b3d06 tools is local now
-  COMMITS=$(git rev-list --reverse 59b3d06..master)
-
-  # we need this for logging
-  TOTAL_COMMITS=$(echo $COMMITS | wc -w | xargs)
-  CURRENT_COMMIT_NUMBER=0
-
-  # empty this file
-  > commit-map.txt
-
-  echo "Rewriting master commits..."
-
-  for COMMIT in $COMMITS; do
-      CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
-      # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"\\r
-      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"
-
-      # set environment variables to preserve author/committer and dates
-      export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
-      export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
-      export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
-      export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
-      export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
-      export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
-
-      # cherry-pick the commit
-      if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
-        # check if the failure is because of an empty commit
-        if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
-          echo "Empty commit detected. Skipping commit $COMMIT"
-          git cherry-pick --skip
-          # log it was empty to the mapping file
-          echo "$COMMIT EMPTY" >> commit-map.txt
-        else
-          # handle other errors or conflicts
-          echo "Cherry-pick failed. Handling error..."
-          echo "$GIT_OUTPUT"
-          exit 1
-        fi
-      else
-        # capture the new commit hash
-        NEW_COMMIT=$(git rev-parse HEAD)
-
-        # save the old and new commit hashes to the mapping file
-        echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
-
-        # append the old commit ID to the commit message
-        git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
-      fi
-
-    # prune every 3000 commits to avoid gc errors
-    if [ $((CURRENT_COMMIT_NUMBER % 3000)) -eq 0 ]; then
-      echo "Pruning repo..."
-      git gc
-    fi
-  done
-
-  echo "Rewriting tags..."
-
-  # remove all old tags
-  git tag -l | xargs git tag -d
-
-  # read each line from the tag-commit-map.txt
-  while IFS=' ' read -r TAG OLD_COMMIT; do
-    # search for the new commit in commit-map.txt corresponding to the old commit
-    NEW_COMMIT=$(grep "^$OLD_COMMIT " "commit-map.txt" | awk '{print $2}')
-
-    # check if this is a rebased commit
-    if [ -z "$NEW_COMMIT" ]; then
-      # if not, then just use old commit hash
-      NEW_COMMIT=$OLD_COMMIT
-    fi
-
-    echo "Rewriting tag $TAG from commit $NEW_COMMIT"
-    git tag -f "$TAG" "$NEW_COMMIT"
-  done < "$DIR/tag-commit-map.txt"
-
-  # uninstall lfs since we don't want to touch (push to) lfs right now
-  # git push will also push lfs, if we don't uninstall (--local so just for this repo)
-  git lfs uninstall --local
-
-  # force push new master
-  git push --force origin main:master
-
-  # force push new tags
-  git push --force --tags
-fi
-
-# REWRITE branches based on master
-if [ ! -f "$SRC_CLONE/rewrite-branches-done" ]; then
-  cd $SRC_CLONE
-  > rewrite-branches-done
-
-  # empty file
-  > $BRANCH_DIFF_LOG
-
-  echo "Rewriting branches based on master..."
-
-  # will store raw diffs here, if exist
-  mkdir -p differences
-
-  # get a list of all branches except master and REWRITE_IGNORE_BRANCHES
-  BRANCHES=$(git branch -r | grep -v ' -> ' | sed 's/.*origin\///' | grep -v '^master$' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n'))
-
-  for BRANCH in $BRANCHES; do
-    # check if the branch is based on master history
-    MERGE_BASE=$(git merge-base master origin/$BRANCH) || true
-    if [ -n "$MERGE_BASE" ]; then
-      echo "Rewriting branch: $BRANCH"
-
-      # create a new branch based on the new master
-      NEW_MERGE_BASE=$(grep "^$MERGE_BASE " "commit-map.txt" | awk '{print $2}')
-      if [ -z "$NEW_MERGE_BASE" ]; then
-        echo "Error: could not find new merge base for branch $BRANCH" >> $BRANCH_DIFF_LOG
-        continue
-      fi
-      git checkout -b ${BRANCH}_new $NEW_MERGE_BASE
-
-      # get the range of commits unique to this branch
-      COMMITS=$(git rev-list --reverse $MERGE_BASE..origin/${BRANCH})
-
-      HAS_ERROR=0
-
-      # simple delimiter
-      echo "BRANCH ${BRANCH}" >> commit-map.txt
-
-      for COMMIT in $COMMITS; do
-        # set environment variables to preserve author/committer and dates
-        export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
-        export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
-        export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
-        export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
-        export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
-        export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
-
-        # cherry-pick the commit
-        if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
-          # check if the failure is because of an empty commit
-          if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
-            echo "Empty commit detected. Skipping commit $COMMIT"
-            git cherry-pick --skip
-            # log it was empty to the mapping file
-            echo "$COMMIT EMPTY" >> commit-map.txt
-          else
-            # handle other errors or conflicts
-            echo "Cherry-pick of ${BRANCH} branch failed. Removing branch upstream..." >> $BRANCH_DIFF_LOG
-            echo "$GIT_OUTPUT" > "$LOGS_DIR/branch-${BRANCH}"
-            git cherry-pick --abort
-            git push --delete origin ${BRANCH}
-            HAS_ERROR=1
-            break
-          fi
-        else
-          # capture the new commit hash
-          NEW_COMMIT=$(git rev-parse HEAD)
-
-          # save the old and new commit hashes to the mapping file
-          echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
-
-          # append the old commit ID to the commit message
-          git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
-        fi
-      done
-
-      # force push the new branch
-      if [ $HAS_ERROR -eq 0 ]; then
-        # git lfs goes haywire here, so we need to install and uninstall
-        # git lfs install --skip-smudge --local
-        git lfs uninstall --local > /dev/null
-        git push -f origin ${BRANCH}_new:${BRANCH}
-      fi
-
-      # clean up local branch
-      git checkout master > /dev/null
-      git branch -D ${BRANCH}_new > /dev/null
-    else
-      echo "Deleting branch $BRANCH as it's not based on master history" >> $BRANCH_DIFF_LOG
-      git push --delete origin ${BRANCH}
-    fi
-  done
-fi
-
-# VALIDATE cherry-pick
-if [ ! -f "$SRC_CLONE/validation-done" ]; then
-  cd $SRC_CLONE
-  > validation-done
-
-  TOTAL_COMMITS=$(grep -cve '^\s*$' commit-map.txt)
-  CURRENT_COMMIT_NUMBER=0
-  COUNT_SAME=0
-  COUNT_DIFF=0
-
-  # empty file
-  > $COMMIT_DIFF_LOG
-
-  echo "Validating commits..."
-
-  # will store raw diffs here, if exist
-  mkdir -p differences
-
-  # read each line from commit-map.txt
-  while IFS=' ' read -r OLD_COMMIT NEW_COMMIT; do
-    if [ "$NEW_COMMIT" == "EMPTY" ]; then
-      continue
-    fi
-    if [ "$OLD_COMMIT" == "BRANCH" ]; then
-      echo "Branch ${NEW_COMMIT} below:" >> $COMMIT_DIFF_LOG
-      continue
-    fi
-    CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
-    # retrieve short hashes and dates for the old and new commits
-    OLD_COMMIT_SHORT=$(git rev-parse --short $OLD_COMMIT)
-    NEW_COMMIT_SHORT=$(git rev-parse --short $NEW_COMMIT)
-    OLD_DATE=$(git show -s --format='%cd' $OLD_COMMIT)
-    NEW_DATE=$(git show -s --format='%cd' $NEW_COMMIT)
-
-    # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"\\r
-    echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"
-
-    # generate lists of files and their hashes for the old and new commits, excluding ignored files
-    OLD_FILES=$(git ls-tree -r $OLD_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
-    NEW_FILES=$(git ls-tree -r $NEW_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
-
-    # Compare the diffs
-    if diff <(echo "$OLD_FILES") <(echo "$NEW_FILES") > /dev/null; then
-      # echo "Old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT are equivalent."
-      COUNT_SAME=$((COUNT_SAME + 1))
-    else
-      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Difference found between old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT" >> $COMMIT_DIFF_LOG
-      COUNT_DIFF=$((COUNT_DIFF + 1))
-      set +e
-      diff -u <(echo "$OLD_FILES") <(echo "$NEW_FILES") > "$LOGS_DIR/commit-$CURRENT_COMMIT_NUMBER-$OLD_COMMIT_SHORT-$NEW_COMMIT_SHORT"
-      set -e
-    fi
-  done < "commit-map.txt"
-
-  echo "Summary:" >> $COMMIT_DIFF_LOG
-  echo "Equivalent commits: $COUNT_SAME" >> $COMMIT_DIFF_LOG
-  echo "Different commits: $COUNT_DIFF" >> $COMMIT_DIFF_LOG
-fi
-
-if [ ! -d $OUT ]; then
-  cp -r $SRC $OUT
-
-  cd $OUT
-
-  # remove all non-master branches
-  # git branch | grep -v "^  master$" | grep -v "\*" | xargs git branch -D
-
-  # echo "cleaning up refs"
-  # delete pull request refs since we can't alter them anyway (https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#error-failed-to-push-some-refs)
-  # git for-each-ref --format='%(refname)' | grep '^refs/pull/' | xargs -I {} git update-ref -d {}
-
-  echo "importing new lfs files"
-  # import "almost" everything to lfs
-  BRANCHES=$(git for-each-ref --format='%(refname)' refs/heads/ | sed 's%refs/heads/%%g' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n') | tr '\n' ' ')
-  git lfs migrate import --include="*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,selfdrive/car/tests/test_models_segs.txt,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,flask/**/*,panda/**/*,board/**/*,messaging/**/*,opendbc/**/*,tools/cabana/chartswidget.cc,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,selfdrive/ui/paint.cc,werkzeug/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,selfdrive/locationd/laikad.py,selfdrive/locationd/test/test_laikad.py,tools/gpstest/test_laikad.py,selfdrive/locationd/laikad_helpers.py,tools/nui/**/*,jsonrpc/**/*,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,selfdrive/camerad/cameras/camera_qcom.cc,selfdrive/manager.py,selfdrive/modeld/models/driving.cc,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,gunicorn/**/*,*.qm,jinja2/**/*,click/**/*,dbcs/**/*,websocket/**/*" $BRANCHES
-
-  echo "reflog and gc"
-  # this is needed after lfs import
-  git reflog expire --expire=now --all
-  git gc --prune=now --aggressive
-
-  # check the git-filter-repo analysis again - can be found in the repo root/filter-repo/analysis
-  echo "Analyzing with git-filter-repo..."
-  /tmp/git-filter-repo --force --analyze
-
-  echo "New size is $(du -sh .)"
-fi
-
-cd $OUT
-
-# fetch all lfs files from https://github.com/commaai/openpilot.git
-# some lfs files are missing on gitlab, but they can be found on github
-git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
-git config lfs.pushurl ssh://git@github.com/commaai/openpilot.git
-git lfs fetch --all || true
-
-# also fetch all lfs files from https://gitlab.com/commaai/openpilot-lfs.git
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
-git lfs fetch --all || true
-
-# final push - will also push lfs
-# TODO: switch to git@github.com:commaai/openpilot.git when ready
-# git push --mirror git@github.com:commaai/openpilot-tiny.git
-# using this instead to ignore refs/pull/* - since this is also what --mirror does - https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
-git push --prune git@github.com:commaai/openpilot-tiny.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/rewrite.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-git clone --bare https://github.com/commaai/openpilot
-cp -r openpilot.git openpilot_backup
-cd openpilot.git
-
-# backup old repo
-git push git@github.com:commaai/openpilot-archive.git +refs/heads/master:refs/heads/master
-git push git@github.com:commaai/openpilot-archive.git +refs/heads/*:refs/heads/*
-git push git@github.com:commaai/openpilot-archive.git +refs/tags/*:refs/tags/*
-git push --mirror git@github.com:commaai/openpilot-archive.git
-
-# ignore all release branches
-git for-each-ref --format='delete %(refname)' | grep 'dashcam3\|devel\|master-ci\|nightly\|release2\|release3\|release3-staging' | git update-ref --stdin
-
-# re-tag old releases on master
-declare -A TAGS=( ["f8cb04e4a8b032b72a909f68b808a50936184bee"]="v0.9.7" ["0b4d08fab8e35a264bc7383e878538f8083c33e5"]="v0.9.6" ["3b1e9017c560499786d8a0e46aaaeea65037acac"]="v0.9.5" ["fa310d9e2542cf497d92f007baec8fd751ffa99c"]="v0.9.4" ["8704c1ff952b5c85a44f50143bbd1a4f7b4887e2"]="v0.9.3" ["c7d3b28b93faa6c955fb24bc64031512ee985ee9"]="v0.9.2" ["89f68bf0cbf53a81b0553d3816fdbe522f941fa1"]="v0.9.1" ["58b84fb401a804967aa0dd5ee66fafa90194fd30"]="v0.9.0" ["f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e"]="v0.8.16" ["5a7c2f90361e72e9c35e88abd2e11acdc4aba354"]="v0.8.15" ["71901c94dbbaa2f9f156a80c14cc7ea65219fc7c"]="v0.8.14" ["95da47079510afc91665263619e5939126da637c"]="v0.8.13" ["472177e2a8a1d002e56f9096326fd2dff62e54f9"]="v0.8.12" ["08078acbd0b4f7da469c7dff6159000e358974a9"]="v0.8.11" ["687925c775c375495f9827946138a724bde00b9d"]="v0.8.10" ["204e5a090735a059d69c29145a4cee49450da07e"]="v0.8.9" ["4be956f8861ecbb521ef9503a3c87b07c9d36721"]="v0.8.8" ["589f82c76627d634761a31a34b2488403556eb0b"]="v0.8.7" ["507cfc8910f74ddb8810039d68b880b426ff9ff9"]="v0.8.6" ["d47b00b45a866bef088f51d1ff31de5885ab04e9"]="v0.8.5" ["553e7d1cce314e7eb0587186b1764c3ff43bed62"]="v0.8.4" ["9896438d1511602a1ff87f7c4eb3c7172b30104a"]="v0.8.3" ["280192ed1443f112463417c2d815ea8ee2762fbd"]="v0.8.2" ["8039361567e4659eae2a084e6f39f34acadf4cac"]="v0.8.1" ["d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc"]="v0.8" ["3d456e5d0fbf0c9887d0499dee812f2b029edf6d"]="v0.7.10" ["81763a18b5d0e379b749e090ecce36a91fca7c43"]="v0.7.9" ["9bc0b350fd273bbb2deb3dcaef0312944e4f6cfd"]="v0.7.8" ["ede5b632b58c55e4ff003f948efae07fe03c2280"]="v0.7.7" ["775acd11ba2e0a8c2f5a5655338718d796491b36"]="v0.7.6.1" ["302417b4cf0dcf00d45e4995b5410e543ad121d1"]="v0.7.5" ["12ff088b42221dd17d9d97decb1fc61a7cb0a861"]="v0.7.4" ["9563f7730252451fdcba9bc3d9fe36dab9c86a26"]="v0.7.3" ["8321cf283abbc2ca3fda7e0c7a069a77a492fe0c"]="v0.7.2" ["1e1de64a1e59476b7b3d3558b92149246d5c3292"]="v0.7.1" ["a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3"]="v0.7" ["d4eb5a6eafdd4803d09e6f3963918216cca5a81f"]="v0.6.6" ["70d17cd69b80e7627dcad8fd5b6438f2309ac307"]="v0.6.5" ["58f376002e0c654fbc2de127765fa297cf694a33"]="v0.6.4" ["d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3"]="v0.6.3" ["095ef5f9f60fca1b269aabcc3cfd322b17b9e674"]="v0.6.2" ["cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22"]="v0.6.1" ["60a20537c5f3fcc7f11946d81aebc8f90c08c117"]="v0.6" ["dd34ccfe288ebda8e2568cf550994ae890379f45"]="v0.5.13" ["3f9059fea886f1fa3b0c19a62a981d891dcc84eb"]="v0.5.12" ["2f92d577f995ff6ae1945ef6b89df3cb69b92999"]="v0.5.11" ["5a9d89ed42ddcd209d001a10d7eb828ef0e6d9de"]="v0.5.10" ["0207a970400ee28d3e366f2e8f5c551281accf02"]="v0.5.9" ["b967da5fc1f7a07e3561db072dd714d325e857b0"]="v0.5.8" ["210db686bb89f8696aa040e6e16de65424b808c9"]="v0.5.7" ["860a48765d1016ba226fb2c64aea35a45fe40e4a"]="v0.5.6" ["8f3539a27b28851153454eb737da9624cccaed2d"]="v0.5.5" ["a422246dc30bce11e970514f13f7c110f4470cc3"]="v0.5.4" ["285c52eb693265a0a530543e9ca0aeb593a2a55e"]="v0.5.3" ["0129a8a4ff8da5314e8e4d4d3336e89667ff6d54"]="v0.5.2" ["6f3d10a4c475c4c4509f0b370805419acd13912d"]="v0.5.1" ["de33bc46452b1046387ee2b3a03191b2c71135fb"]="v0.5" ["ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9"]="v0.4.7" ["c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1"]="v0.4.6" ["37285038d3f91fa1b49159c4a35a8383168e644f"]="v0.4.5" ["9a9ff839a9b70cb2601d7696af743f5652395389"]="v0.4.4" ["28c0797d30175043bbfa31307b63aab4197cf996"]="v0.4.2" ["4474b9b3718653aeb0aee26422caefb90460cc0e"]="v0.4.1" ["da52d065a4c4f52d6017a537f3a80326f5af8bdc"]="v0.4.0.2" ["9d3963559ae7b15193057937ff3e72481899f40d"]="v0.3.5"  ["1b8c44b5067525a5d266b6e99799d8097da76a29"]="v0.3.4" ["5cf91d0496688fed4f2a6c7021349b1fc0e057a2"]="v0.3.3" ["7fe46f1e1df5dec08a940451ba0feefd5c039165"]="v0.3.2" ["41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24"]="v0.3.1" ["c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8"]="v0.3.0" ["693bcb0f83478f2651db6bac9be5ca5ad60d03f3"]="v0.2.9" ["95a349abcc050712c50d4d85a1c8a804eee7f6c2"]="v0.2.8" ["c6ba5dc5391d3ca6cda479bf1923b88ce45509a0"]="v0.2.7" ["6c3afeec0fb439070b2912978b8dbb659033b1d9"]="v0.2.6" ["29c58b45882ac79595356caf98580c1d2a626011"]="v0.2.5" ["ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0"]="v0.2.4" ["adaa4ed350acda4067fc0b455ad15b54cdf4c768"]="v0.2.3" ["a64b9aa9b8cb5863c917b6926516291a63c02fe5"]="v0.2.2" ["17d9becd3c673091b22f09aa02559a9ed9230f50"]="v0.2.1" ["449b482cc3236ccf31829830b4f6a44b2dcc06c2"]="v0.2" ["e94a30bec07e719c5a7b037ca1f4db8312702cce"]="v0.1" )
-for tag in "${!TAGS[@]}"; do git tag -f "${TAGS[$tag]}" "$tag" ; done
-
-# get master root commit
-ROOT_COMMIT=$(git rev-list --max-parents=0 HEAD | tail -n 1)
-
-# link master and devel
-git replace --graft $ROOT_COMMIT v0.7.1
-git-filter-repo --prune-empty never --force --commit-callback 'h=commit.original_id.decode("utf-8");m=commit.message.decode("utf-8");commit.message=str.encode(m + "\n" + "old-commit-hash: " + h)'
-
-# delete replace refs
-git for-each-ref --format='delete %(refname)' refs/replace | git update-ref --stdin
-
-# machine validation
-tail -n +2 "filter-repo/commit-map" | tr ' ' '\n' | xargs -P $(nproc) -n 2 bash -c 'H1=$(cd ../openpilot_backup && git ls-tree -r $0 | sha1sum) && H2=$(git ls-tree -r $1 | sha1sum) && echo "$H1 $H2" >> /tmp/GIT_HASHES && diff <(echo $H1) <(echo $H2) || exit 255'
-# human validation
-less /tmp/GIT_HASH
-
-# cleanup
-git reflog expire --expire=now --all
-git gc --prune=now --aggressive
-
-# get all lfs files
-set +e
-git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
-git lfs fetch --all
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git lfs fetch --all
-set -e
-
-# add new files to lfs
-git lfs migrate import --everything --include="*.ico,*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,pyextra/**/*,panda/board/**/inc/*.h,panda/board/obj/*.elf,board/inc/*.h,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,*.pro,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,poetry.lock,*.qm"
-
-# set new lfs endpoint
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
-
-# push all branch+tag (scary stuff...)
-git push -f --set-upstream git@github.com:commaai/openpilot.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/tag-commit-map.txt

@@ -1,82 +0,0 @@
-v0.1 e94a30bec07e719c5a7b037ca1f4db8312702cce
-v0.2 449b482cc3236ccf31829830b4f6a44b2dcc06c2
-v0.2.1 17d9becd3c673091b22f09aa02559a9ed9230f50
-v0.2.2 a64b9aa9b8cb5863c917b6926516291a63c02fe5
-v0.2.3 adaa4ed350acda4067fc0b455ad15b54cdf4c768
-v0.2.4 ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0
-v0.2.5 29c58b45882ac79595356caf98580c1d2a626011
-v0.2.6 6c3afeec0fb439070b2912978b8dbb659033b1d9
-v0.2.7 c6ba5dc5391d3ca6cda479bf1923b88ce45509a0
-v0.2.8 95a349abcc050712c50d4d85a1c8a804eee7f6c2
-v0.2.9 693bcb0f83478f2651db6bac9be5ca5ad60d03f3
-v0.3.0 c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8
-v0.3.1 41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24
-v0.3.2 7fe46f1e1df5dec08a940451ba0feefd5c039165
-v0.3.3 5cf91d0496688fed4f2a6c7021349b1fc0e057a2
-v0.3.4 1b8c44b5067525a5d266b6e99799d8097da76a29
-v0.3.5 b111277f464cf66fa34b67819a83ea683e0f64df
-v0.4.0.2 da52d065a4c4f52d6017a537f3a80326f5af8bdc
-v0.4.1 4474b9b3718653aeb0aee26422caefb90460cc0e
-v0.4.2 28c0797d30175043bbfa31307b63aab4197cf996
-v0.4.4 9a9ff839a9b70cb2601d7696af743f5652395389
-v0.4.5 37285038d3f91fa1b49159c4a35a8383168e644f
-v0.4.6 c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1
-v0.4.7 ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9
-v0.5 de33bc46452b1046387ee2b3a03191b2c71135fb
-v0.5.1 8f22f52235c48eada586795ac57edb22688e4d08
-v0.5.2 0129a8a4ff8da5314e8e4d4d3336e89667ff6d54
-v0.5.3 285c52eb693265a0a530543e9ca0aeb593a2a55e
-v0.5.4 a422246dc30bce11e970514f13f7c110f4470cc3
-v0.5.5 8f3539a27b28851153454eb737da9624cccaed2d
-v0.5.6 860a48765d1016ba226fb2c64aea35a45fe40e4a
-v0.5.7 9ce3045f139ee29bf0eea5ec59dfe7df9c3d2c51
-v0.5.8 2cee2e05ba0f3824fdbb8b957958800fa99071a1
-v0.5.9 ad145da3bcded0fe75306df02061d07a633963c3
-v0.5.10 ff4c1557d8358f158f4358788ff18ef93d2470ef
-v0.5.11 d1866845df423c6855e2b365ff230cf7d89a420b
-v0.5.12 f6e8ef27546e9a406724841e75f8df71cc4c2c97
-v0.5.13 dd34ccfe288ebda8e2568cf550994ae890379f45
-v0.6 60a20537c5f3fcc7f11946d81aebc8f90c08c117
-v0.6.1 cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22
-v0.6.2 095ef5f9f60fca1b269aabcc3cfd322b17b9e674
-v0.6.3 d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3
-v0.6.4 58f376002e0c654fbc2de127765fa297cf694a33
-v0.6.5 70d17cd69b80e7627dcad8fd5b6438f2309ac307
-v0.6.6 d4eb5a6eafdd4803d09e6f3963918216cca5a81f
-v0.7 a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3
-v0.7.1 1e1de64a1e59476b7b3d3558b92149246d5c3292
-v0.7.2 59bd58c940673b4c4a6a86f299022614bcf42b22
-v0.7.3 d7acd8b68f8131e0e714400cf124a3e228638643
-v0.7.4 e93649882c5e914eec4a8b8b593dc0587e497033
-v0.7.5 8abc0afe464626a461d2c7e192c912eeebeccc65
-v0.7.6 69aacd9d179fe6dd3110253a099c38b34cff7899
-v0.7.7 f1caed7299cdba5e45635d8377da6cc1e5fd7072
-v0.7.8 2189fe8741b635d8394d55dee28959425cfd5ad0
-v0.7.9 86dc54b836a973f132ed26db9f5a60b29f9b25b2
-v0.7.10 47a42ff432db8a2494e922ca5e767e58020f0446
-v0.7.11 f46ed718ba8d6bb4d42cd7b0f0150c406017c373
-v0.8 d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc
-v0.8.1 cd6f26664cb8d32a13847d6648567c47c580e248
-v0.8.2 7cc0999aebfe63b6bb6dd83c1dff62c3915c4820
-v0.8.3 986500fe2f10870018f1fba1e5465476b8915977
-v0.8.4 f0d0b82b8d6f5f450952113e234d0a5a49e80c48
-v0.8.5 f5d9ddc6c2a2802a61e5ce590c6b6688bf736a69
-v0.8.6 75904ed7452c6cbfb2a70cd379a899d8a75b97c2
-v0.8.7 4f9e568019492126e236da85b5ca0a059f292900
-v0.8.8 a949a49d5efaaf2d881143d23e9fb5ff9e28e88c
-v0.8.9 a034926264cd1025c69d6ceb3fe444965f960b75
-v0.8.10 59accdd814398b884167c0f41dbf46dcccf0c29c
-v0.8.11 d630ec9092f039cb5e51c5dd6d92fc47b91407e4
-v0.8.12 57871c99031cf597ffa0d819057ac1401e129f32
-v0.8.13 e43e6e876513450d235124fcb711f1724ed9814c
-v0.8.14 71901c94dbbaa2f9f156a80c14cc7ea65219fc7c
-v0.8.15 5a7c2f90361e72e9c35e88abd2e11acdc4aba354
-v0.8.16 f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e
-v0.9.0 58b84fb401a804967aa0dd5ee66fafa90194fd30
-v0.9.1 89f68bf0cbf53a81b0553d3816fdbe522f941fa1
-v0.9.2 c7d3b28b93faa6c955fb24bc64031512ee985ee9
-v0.9.3 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2
-v0.9.4 fa310d9e2542cf497d92f007baec8fd751ffa99c
-v0.9.5 3b1e9017c560499786d8a0e46aaaeea65037acac
-v0.9.6 0b4d08fab8e35a264bc7383e878538f8083c33e5
-v0.9.7 f8cb04e4a8b032b72a909f68b808a50936184bee

scripts/pyqt_demo.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-
-from PyQt5.QtWidgets import QApplication, QLabel
-from openpilot.selfdrive.ui.qt.python_helpers import set_main_window
-
-
-if __name__ == "__main__":
-  app = QApplication([])
-  label = QLabel('Hello World!')
-
-  # Set full screen and rotate
-  set_main_window(label)
-
-  app.exec_()