CI: Fix process replay to properly work with local routes (#469)

Updating test_processes.py to easily upload ref routes locally to git instead of azure

.gitattributes

@@ -9,6 +9,7 @@
 *.ttf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
+selfdrive/test/process_replay/fakedata/*.zst filter=lfs diff=lfs merge=lfs -text
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
 system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
 selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text

scripts/lint/lint.sh

@@ -13,7 +13,7 @@ cd $ROOT
 
 FAILED=0
 
-IGNORED_FILES="uv\.lock|docs\/CARS.md|LICENSE\.md"
+IGNORED_FILES="uv\.lock|docs\/CARS.md|LICENSE\.md|.*\.zst"
 IGNORED_DIRS="^third_party.*|^msgq.*|^msgq_repo.*|^opendbc.*|^opendbc_repo.*|^cereal.*|^panda.*|^rednose.*|^rednose_repo.*|^tinygrad.*|^tinygrad_repo.*|^teleoprtc.*|^teleoprtc_repo.*"
 
 function run() {

selfdrive/test/process_replay/.gitignore

@@ -1 +1 @@
-fakedata/
+!fakedata/*

selfdrive/test/process_replay/fakedata/regen1CA7A48E6F7|2024-08-30--02-45-08--0_controlsd_8ab4d7b987f55855ef64f16afbd3f67996e6935a.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec364c58a24d2b061da26db823ad9f9b022939acc69bf173c5e7692b369ffc2
+size 167913

selfdrive/test/process_replay/fakedata/regenC8F0D6ADC5C|2024-08-30--02-54-01--0_calibrationd_8ab4d7b987f55855ef64f16afbd3f67996e6935a.zst

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c34d0a968f789548d507be6e2aa42a308ea76fa29204f00287e07119b725bc90
+size 1863

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-255ceb08c75bc85379da5ec247e612be3716fb43
+8ab4d7b987f55855ef64f16afbd3f67996e6935a

selfdrive/test/process_replay/test_processes.py

@@ -16,6 +16,8 @@ from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC
 from openpilot.tools.lib.filereader import FileReader
 from openpilot.tools.lib.logreader import LogReader, save_log
 
+IS_AZURE_TOKEN_DEFINED = os.getenv("AZURE_TOKEN")
+
 source_segments = [
   ("BODY", "937ccb7243511b65|2022-05-24--16-03-09--1"),        # COMMA.COMMA_BODY
   ("HYUNDAI", "02c45f73a2e5c6e9|2021-01-01--19-08-22--1"),     # HYUNDAI.HYUNDAI_SONATA
@@ -67,6 +69,28 @@ REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit")
 EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"}
 
 
+def preserve_only_specified_files_from_ref_commit(*commits_to_keep):
+  """Keep only files in fakedata that contain any of the specified commit hashes."""
+  removed = 0
+  for f in os.listdir(FAKEDATA):
+    if not any(commit in f for commit in commits_to_keep):
+      os.remove(os.path.join(FAKEDATA, f))
+      removed += 1
+  if removed > 0:
+    print(f"Removed {removed} old files from {FAKEDATA}")
+
+
+def handle_output_file(cur_log_fn, local):
+  """Handle the output file based on whether we're using remote or local storage."""
+  assert os.path.exists(cur_log_fn), f"Cannot find log to upload: {cur_log_fn}"
+
+  if local:
+    os.system(f"git add '{os.path.realpath(cur_log_fn)}'")
+  else:
+    upload_file(cur_log_fn, os.path.basename(cur_log_fn))
+    os.remove(cur_log_fn)
+
+
 def run_test_process(data):
   segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data
   res = None
@@ -77,10 +101,9 @@ def run_test_process(data):
     save_log(cur_log_fn, log_msgs)
 
   if args.update_refs or args.upload_only:
-    print(f'Uploading: {os.path.basename(cur_log_fn)}')
-    assert os.path.exists(cur_log_fn), f"Cannot find log to upload: {cur_log_fn}"
-    upload_file(cur_log_fn, os.path.basename(cur_log_fn))
-    os.remove(cur_log_fn)
+    print(f'Processing: {os.path.basename(cur_log_fn)}')
+    handle_output_file(cur_log_fn, args.local)
+
   return (segment, cfg.proc_name, res)
 
 
@@ -119,6 +142,27 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non
     return str(e), log_msgs
 
 
+def finalize_git_updates(cur_commit, ref_commit_fn):
+  """Finalize git updates and create commit."""
+  try:
+    # Add all new files first
+    os.system(f"git add {os.path.realpath(ref_commit_fn)}")
+    os.system(f"git add {os.path.realpath(FAKEDATA)}/*.zst")
+
+    # Clean up old files - keep only new ref files since they're becoming the reference
+    preserve_only_specified_files_from_ref_commit(cur_commit)
+
+    # Add the deletions to git
+    os.system(f"git add -u {os.path.realpath(FAKEDATA)}")
+
+    # Create the commit
+    commit_msg = f"test_processes: update ref logs to {cur_commit[:7]}"
+    os.system(f'git commit -m "{commit_msg}"')
+    print("Successfully committed reference log updates")
+  except Exception as e:
+    print(f"Failed to commit changes: {e}")
+
+
 if __name__ == "__main__":
   all_cars = {car for car, _ in segments}
   all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS}
@@ -142,6 +186,8 @@ if __name__ == "__main__":
                       help="Updates reference logs using current commit")
   parser.add_argument("--upload-only", action="store_true",
                       help="Skips testing processes and uploads logs from previous test run")
+  parser.add_argument("--local", action="store_true",
+                      help="Use  local git/ storage instead of remote (Azure for Comma)")
   parser.add_argument("-j", "--jobs", type=int, default=max(cpu_count - 2, 1),
                       help="Max amount of parallel jobs")
   args = parser.parse_args()
@@ -168,6 +214,16 @@ if __name__ == "__main__":
   if not cur_commit:
     raise Exception("Couldn't get current commit")
 
+  # Could be set as default in args, but wanted to be more explicit on the flow.
+  if upload and not args.local and not IS_AZURE_TOKEN_DEFINED:
+    print("***** Warning: local/git run was used by default since AZURE_TOKEN was NOT found on the env variables! *****")
+    args.local = True
+
+  # Clean up old files before starting
+  if upload and args.local:
+    print("***** Cleaning up old fakedata for local/git tracked refs *****")
+    preserve_only_specified_files_from_ref_commit(cur_commit, ref_commit)
+
   print(f"***** testing against commit {ref_commit} *****")
 
   # check to make sure all car brands are tested
@@ -234,4 +290,8 @@ if __name__ == "__main__":
       f.write(cur_commit)
     print(f"\n\nUpdated reference logs for commit: {cur_commit}")
 
+    # Only do git operations if we're in local mode
+    if upload and args.local:
+      finalize_git_updates(cur_commit, REF_COMMIT_FN)
+
   sys.exit(int(failed))