From cc6477015ae698057f04d041730240871e466e7a Mon Sep 17 00:00:00 2001 From: George Hotz Date: Fri, 24 Apr 2020 12:10:27 -0700 Subject: [PATCH] delete unused code old-commit-hash: 8b448a7c16d832567f0d3644cc54828ab951b8db --- common/column_store.py | 210 ------------------ .../internal/run_segment_through_visiond.py | 2 +- 2 files changed, 1 insertion(+), 211 deletions(-) delete mode 100644 common/column_store.py diff --git a/common/column_store.py b/common/column_store.py deleted file mode 100644 index 14d7accc3d..0000000000 --- a/common/column_store.py +++ /dev/null @@ -1,210 +0,0 @@ -import os -import numpy as np -import collections -from contextlib import closing - -from common.file_helpers import mkdirs_exists_ok - -class ColumnStoreReader(): - def __init__(self, path, mmap=False, allow_pickle=False, direct_io=False, prefix="", np_data=None): - if not (path and os.path.isdir(path)): - raise ValueError("Not a column store: {}".format(path)) - - self._path = os.path.realpath(path) - self._keys = os.listdir(self._path) - self._mmap = mmap - self._allow_pickle = allow_pickle - self._direct_io = direct_io - self._is_dict = 'columnstore' in self._keys - self._prefix = prefix - self._np_data = np_data - - @property - def path(self): - return self._path - - def close(self): - pass - - def get(self, key): - try: - return self[key] - except KeyError: - return None - - def keys(self): - if self._is_dict: - if not self._np_data: - self._np_data = self._load(os.path.join(self._path, 'columnstore')) - # return top level keys by matching on prefix and splitting on '/' - return { - k[len(self._prefix):].split('/')[0] - for k in self._np_data.keys() - if k.startswith(self._prefix) - } - - return list(self._keys) - - def iteritems(self): - for k in self: - yield (k, self[k]) - - def itervalues(self): - for k in self: - yield self[k] - - def get_npy_path(self, key): - """Gets a filesystem path for an npy file containing the specified array, - or none if the column store does not contain key. - """ - if key in self: - return os.path.join(self._path, key) - else: - return None - - def _load(self, path): - if self._mmap: - # note that direct i/o does nothing for mmap since file read/write interface is not used - return np.load(path, mmap_mode='r', allow_pickle=self._allow_pickle, fix_imports=False) - - if self._direct_io: - opener = lambda path, flags: os.open(path, os.O_RDONLY | os.O_DIRECT) - with open(path, 'rb', buffering=0, opener=opener) as f: - return np.load(f, allow_pickle=self._allow_pickle, fix_imports=False) - - return np.load(path, allow_pickle=self._allow_pickle, fix_imports=False) - - def __getitem__(self, key): - try: - if self._is_dict: - path = os.path.join(self._path, 'columnstore') - else: - path = os.path.join(self._path, key) - - # TODO(mgraczyk): This implementation will need to change for zip. - if not self._is_dict and os.path.isdir(path): - return ColumnStoreReader(path) - else: - if self._is_dict and self._np_data: - ret = self._np_data - else: - ret = self._load(path) - - if type(ret) == np.lib.npyio.NpzFile: - if self._is_dict: - if self._prefix+key in ret.keys(): - return ret[self._prefix+key] - if any(k.startswith(self._prefix+key+"/") for k in ret.keys()): - return ColumnStoreReader(self._path, mmap=self._mmap, allow_pickle=self._allow_pickle, direct_io=self._direct_io, prefix=self._prefix+key+"/", np_data=ret) - raise KeyError(self._prefix+key) - - # if it's saved as compressed, it has arr_0 only in the file. deref this - return ret['arr_0'] - else: - return ret - except IOError: - raise KeyError(key) - - def __contains__(self, item): - try: - self[item] - return True - except KeyError: - return False - - def __len__(self): - return len(self._keys) - - def __bool__(self): - return bool(self._keys) - - def __iter__(self): - return iter(self._keys) - - def __str__(self): - return "ColumnStoreReader({})".format(str({k: "..." for k in self._keys})) - - def __enter__(self): return self - def __exit__(self, type, value, traceback): self.close() - -class ColumnStoreWriter(): - def __init__(self, path, allow_pickle=False): - self._path = path - self._allow_pickle = allow_pickle - mkdirs_exists_ok(self._path) - - def map_column(self, path, dtype, shape): - npy_path = os.path.join(self._path, path) - mkdirs_exists_ok(os.path.dirname(npy_path)) - return np.lib.format.open_memmap(npy_path, mode='w+', dtype=dtype, shape=shape) - - def add_column(self, path, data, dtype=None, compression=False, overwrite=False): - npy_path = os.path.join(self._path, path) - mkdirs_exists_ok(os.path.dirname(npy_path)) - - if overwrite: - f = open(npy_path, "wb") - else: - f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb") - - with closing(f) as f: - data2 = np.array(data, copy=False, dtype=dtype) - if compression: - np.savez_compressed(f, data2) - else: - np.save(f, data2, allow_pickle=self._allow_pickle, fix_imports=False) - - def add_group(self, group_name): - # TODO(mgraczyk): This implementation will need to change if we add zip or compression. - return ColumnStoreWriter(os.path.join(self._path, group_name)) - - def add_dict(self, data, dtypes=None, compression=False, overwrite=False): - # default name exists to have backward compatibility with equivalent directory structure - npy_path = os.path.join(self._path, "columnstore") - mkdirs_exists_ok(os.path.dirname(npy_path)) - - flat_dict = dict() - _flatten_dict(flat_dict, "", data) - for k, v in flat_dict.items(): - dtype = dtypes[k] if dtypes is not None and k in dtypes else None - flat_dict[k] = np.array(v, copy=False, dtype=dtype) - - if overwrite: - f = open(npy_path, "wb") - else: - f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb") - - with closing(f) as f: - if compression: - np.savez_compressed(f, **flat_dict) - else: - np.savez(f, **flat_dict) - - def close(self): - pass - - def __enter__(self): return self - def __exit__(self, type, value, traceback): self.close() - - -def _flatten_dict(flat, ns, d): - for k, v in d.items(): - p = (ns + "/" if len(ns) else "") + k - if isinstance(v, collections.Mapping): - _flatten_dict(flat, p, v) - else: - flat[p] = v - - -def _save_dict_as_column_store(values, writer, compression): - for k, v in values.items(): - if isinstance(v, collections.Mapping): - _save_dict_as_column_store(v, writer.add_group(k), compression) - else: - writer.add_column(k, v, compression=compression) - - -def save_dict_as_column_store(values, output_path, compression=False): - with ColumnStoreWriter(output_path) as writer: - _save_dict_as_column_store(values, writer, compression) - diff --git a/selfdrive/debug/internal/run_segment_through_visiond.py b/selfdrive/debug/internal/run_segment_through_visiond.py index c2470ec117..e10ddcfe6a 100755 --- a/selfdrive/debug/internal/run_segment_through_visiond.py +++ b/selfdrive/debug/internal/run_segment_through_visiond.py @@ -11,7 +11,7 @@ from cereal.services import service_list from tools.lib.logreader import LogReader from xx.chffr.lib.route import Route, RouteSegment from tools.lib.route_framereader import RouteFrameReader -from common.column_store import save_dict_as_column_store +from xx.uncommon.column_store import save_dict_as_column_store from xx.pipeline.lib.log_time_series import append_dict from selfdrive.test.process_replay.compare_logs import save_log