diff --git a/.gitignore b/.gitignore index 062801d78..1fef0a125 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ flycheck_* cppcheck_report.txt comma*.sh -selfdrive/modeld/models/*.pkl* +selfdrive/modeld/models/*.pkl # openpilot log files *.bz2 diff --git a/common/file_chunker.py b/common/file_chunker.py deleted file mode 100644 index 2a78963d1..000000000 --- a/common/file_chunker.py +++ /dev/null @@ -1,27 +0,0 @@ -import glob -import os -import sys -from pathlib import Path - -CHUNK_SIZE = 49 * 1024 * 1024 # 49MB, under GitHub's 50MB limit - - -def chunk_file(path): - with open(path, 'rb') as f: - data = f.read() - for i in range(0, len(data), CHUNK_SIZE): - with open(f"{path}.chunk{i // CHUNK_SIZE:02d}", 'wb') as f: - f.write(data[i:i + CHUNK_SIZE]) - os.remove(path) - - -def read_file_chunked(path): - files = sorted(glob.glob(f"{path}.chunk*")) or ([path] if os.path.isfile(path) else []) - if not files: - raise FileNotFoundError(path) - return b''.join(Path(f).read_bytes() for f in files) - - -if __name__ == "__main__": - for path in sys.argv[1:]: - chunk_file(path) diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index a48c79570..84e94df4a 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -8,8 +8,6 @@ tinygrad_root = env.Dir("#").abspath tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=tinygrad_root) if 'pycache' not in x and os.path.isfile(os.path.join(tinygrad_root, x))] -chunk_cmd = f'python3 {Dir("#common").abspath}/file_chunker.py' - # Get model metadata for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']: fn = File(f"models/{model_name}").abspath @@ -26,23 +24,21 @@ image_flag = { 'larch64': 'IMAGE=2', }.get(arch, 'IMAGE=0') script_files = [File(Dir("#selfdrive/modeld").File("compile_warp.py").abspath)] -compile_warp_cmd = f'{tg_flags} python3 {Dir("#selfdrive/modeld").abspath}/compile_warp.py ' +cmd = f'{tg_flags} python3 {Dir("#selfdrive/modeld").abspath}/compile_warp.py ' from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye warp_targets = [] for cam in [_ar_ox_fisheye, _os_fisheye]: w, h = cam.width, cam.height warp_targets += [File(f"models/warp_{w}x{h}_tinygrad.pkl").abspath, File(f"models/dm_warp_{w}x{h}_tinygrad.pkl").abspath] -lenv.Command([t + ".chunk00" for t in warp_targets], tinygrad_files + script_files, - f'{compile_warp_cmd} && {chunk_cmd} {" ".join(warp_targets)}') +lenv.Command(warp_targets, tinygrad_files + script_files, cmd) def tg_compile(flags, model_name): pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"' fn = File(f"models/{model_name}").abspath - pkl = fn + "_tinygrad.pkl" return lenv.Command( - pkl + ".chunk00", + fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, - f'{pythonpath_string} {flags} {image_flag} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {pkl} && {chunk_cmd} {pkl}' + f'{pythonpath_string} {flags} {image_flag} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl' ) # Compile small models diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index 7cac17fa8..bc3adffff 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -16,7 +16,6 @@ from openpilot.common.realtime import config_realtime_process from openpilot.common.transformations.model import dmonitoringmodel_intrinsics from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye from openpilot.system.camerad.cameras.nv12_info import get_nv12_info -from openpilot.common.file_chunker import read_file_chunked from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid, safe_exp PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld" @@ -44,7 +43,8 @@ class ModelState: self.frame_buf_params = None self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()} self.image_warp = None - self.model_run = pickle.loads(read_file_chunked(str(MODEL_PKL_PATH))) + with open(MODEL_PKL_PATH, "rb") as f: + self.model_run = pickle.load(f) def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]: self.numpy_inputs['calib'][0,:] = calib @@ -54,7 +54,8 @@ class ModelState: if self.image_warp is None: self.frame_buf_params = get_nv12_info(buf.width, buf.height) warp_path = MODELS_DIR / f'dm_warp_{buf.width}x{buf.height}_tinygrad.pkl' - self.image_warp = pickle.loads(read_file_chunked(str(warp_path))) + with open(warp_path, "rb") as f: + self.image_warp = pickle.load(f) self.warp_inputs['frame'] = Tensor.from_blob(buf.data.ctypes.data, (self.frame_buf_params[3],), dtype='uint8').realize() self.warp_inputs_np['transform'][:] = transform[:] diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 01052840a..6f3c481ea 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -27,7 +27,6 @@ from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value, get_curvature_from_plan from openpilot.selfdrive.modeld.parse_model_outputs import Parser from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState -from openpilot.common.file_chunker import read_file_chunked from openpilot.selfdrive.modeld.constants import ModelConstants, Plan @@ -178,8 +177,11 @@ class ModelState: self.frame_buf_params : dict[str, tuple[int, int, int, int]] = {} self.update_imgs = None - self.vision_run = pickle.loads(read_file_chunked(str(VISION_PKL_PATH))) - self.policy_run = pickle.loads(read_file_chunked(str(POLICY_PKL_PATH))) + with open(VISION_PKL_PATH, "rb") as f: + self.vision_run = pickle.load(f) + + with open(POLICY_PKL_PATH, "rb") as f: + self.policy_run = pickle.load(f) def slice_outputs(self, model_outputs: np.ndarray, output_slices: dict[str, slice]) -> dict[str, np.ndarray]: parsed_model_outputs = {k: model_outputs[np.newaxis, v] for k,v in output_slices.items()} @@ -196,7 +198,8 @@ class ModelState: w, h = bufs[key].width, bufs[key].height self.frame_buf_params[key] = get_nv12_info(w, h) warp_path = MODELS_DIR / f'warp_{w}x{h}_tinygrad.pkl' - self.update_imgs = pickle.loads(read_file_chunked(str(warp_path))) + with open(warp_path, "rb") as f: + self.update_imgs = pickle.load(f) for key in bufs.keys():