add tool to create casync agnos updates from existing manifest (#32124)

* wip

* wip

* add agnos prefix

* add mkdir

* tool to create agnos updates
old-commit-hash: 44bb4eb521b3d0fcb00e9df5eb7c74741946ebe4

release/create_casync_agnos_release.py

@@ -0,0 +1,52 @@
+import argparse
+import json
+import pathlib
+import tempfile
+from openpilot.common.basedir import BASEDIR
+from openpilot.system.hardware.tici.agnos import StreamingDecompressor
+from openpilot.system.updated.casync.common import create_casync_from_file
+
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="creates a casync release")
+  parser.add_argument("output_dir", type=str, help="output directory for the channel")
+  parser.add_argument("version", type=str, help="version of agnos this is")
+  parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \
+                        default=str(pathlib.Path(BASEDIR) / "system/hardware/tici/agnos.json"))
+  args = parser.parse_args()
+
+  output_dir = pathlib.Path(args.output_dir)
+  output_dir.mkdir(parents=True, exist_ok=True)
+
+  manifest_file = pathlib.Path(args.manifest)
+
+  with tempfile.NamedTemporaryFile() as entry_file:
+    entry_path = pathlib.Path(entry_file.name)
+
+    with open(manifest_file) as f:
+      manifest = json.load(f)
+
+    for entry in manifest:
+      print(f"creating casync agnos build from {entry}")
+      downloader = StreamingDecompressor(entry['url'])
+
+      CHUNK_SIZE=16*1024*1024 # 16mb
+
+      size = entry["size"]
+
+      cur = 0
+      with open(entry_path, "wb") as f:
+        while True:
+          to_read = min((size - cur), CHUNK_SIZE)
+
+          print(f"{cur/size*100:06.2f}% {to_read}")
+
+          if not to_read:
+            break
+
+          f.write(downloader.read(to_read))
+
+          cur += to_read
+
+      create_casync_from_file(entry_path, output_dir, f"agnos-{args.version}-{entry['name']}")

system/updated/casync/common.py

@@ -45,11 +45,17 @@ def create_casync_tar_package(target_dir: pathlib.Path, output_path: pathlib.Pat
   tar.create_tar_archive(output_path, target_dir, is_not_git)
 
 
+def create_casync_from_file(file: pathlib.Path, output_dir: pathlib.Path, caibx_name: str):
+  caibx_file = output_dir / f"{caibx_name}.caibx"
+  run(["casync", "make", *CASYNC_ARGS, caibx_file, str(file)])
+
+  return caibx_file
+
+
 def create_casync_release(target_dir: pathlib.Path, output_dir: pathlib.Path, caibx_name: str):
   tar_file = output_dir / f"{caibx_name}.tar"
   create_casync_tar_package(target_dir, tar_file)
-  caibx_file = output_dir / f"{caibx_name}.caibx"
-  run(["casync", "make", *CASYNC_ARGS, caibx_file, str(tar_file)])
+  caibx_file = create_casync_from_file(tar_file, output_dir, caibx_name)
   tar_file.unlink()
   digest = run(["casync", "digest", *CASYNC_ARGS, target_dir]).decode("utf-8").strip()
   return digest, caibx_file