mawei
/
openpilot0
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ColumnStoreWriter.add_dict() support for multiple types 

old-commit-hash: e23f1fcf0a
This commit is contained in:
Greg Hogan 2020-04-20 16:56:38 -07:00
parent f7f8e976b7
commit 915a7c4195
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
common/column_store.py
View File

@ -158,7 +158,7 @@ class ColumnStoreWriter():
# TODO(mgraczyk): This implementation will need to change if we add zip or compression. # TODO(mgraczyk): This implementation will need to change if we add zip or compression.
return ColumnStoreWriter(os.path.join(self._path, group_name)) return ColumnStoreWriter(os.path.join(self._path, group_name))
def add_dict(self, data, dtype=None, compression=False, overwrite=False): def add_dict(self, data, dtypes=None, compression=False, overwrite=False):
# default name exists to have backward compatibility with equivalent directory structure # default name exists to have backward compatibility with equivalent directory structure
npy_path = os.path.join(self._path, "columnstore") npy_path = os.path.join(self._path, "columnstore")
mkdirs_exists_ok(os.path.dirname(npy_path)) mkdirs_exists_ok(os.path.dirname(npy_path))
@ -166,6 +166,7 @@ class ColumnStoreWriter():
flat_dict = dict() flat_dict = dict()
_flatten_dict(flat_dict, "", data) _flatten_dict(flat_dict, "", data)
for k, v in flat_dict.items(): for k, v in flat_dict.items():
dtype = dtypes[k] if dtypes is not None and k in dtypes else None
flat_dict[k] = np.array(v, copy=False, dtype=dtype) flat_dict[k] = np.array(v, copy=False, dtype=dtype)
if overwrite: if overwrite: