From b5d700adae080bb08b2f392e8a4f90b196585423 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 26 Aug 2023 16:16:08 -0700 Subject: [PATCH] update openpilot supercombo.onnx to 0.9.4 (#1681) * update openpilot supercombo.onnx to 0.9.4 * update tests for the new model * comment out comma models from external_model_benchmark --- .github/workflows/test.yml | 2 +- openpilot/compile.py | 2 +- test/external/external_model_benchmark.py | 6 ++++-- test/models/test_onnx.py | 17 ++++++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9494b9b0..4f2f5467 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,7 +145,7 @@ jobs: - if: ${{ matrix.task == 'openpilot' }} name: Test openpilot model compile and size run: | - DEBUG=2 ALLOWED_KERNEL_COUNT=199 FLOAT16=1 DEBUGCL=1 GPU=1 IMAGE=2 python openpilot/compile.py + DEBUG=2 ALLOWED_KERNEL_COUNT=209 FLOAT16=1 DEBUGCL=1 GPU=1 IMAGE=2 python openpilot/compile.py python -c 'import os; assert os.path.getsize("/tmp/output.thneed") < 100_000_000' - if: ${{ matrix.task == 'openpilot' }} name: Test openpilot model correctness (float32) diff --git a/openpilot/compile.py b/openpilot/compile.py index 1a60bd8c..6e9f804b 100644 --- a/openpilot/compile.py +++ b/openpilot/compile.py @@ -25,7 +25,7 @@ from extra.utils import fetch from extra.onnx import get_run_onnx from tinygrad.tensor import Tensor -OPENPILOT_MODEL = "https://github.com/commaai/openpilot/raw/6c5693e965b9c63f8678f52b9e9b5abe35f23feb/selfdrive/modeld/models/supercombo.onnx" +OPENPILOT_MODEL = "https://github.com/commaai/openpilot/raw/v0.9.4/selfdrive/modeld/models/supercombo.onnx" np.random.seed(1337) def get_random_input_tensors(input_shapes): diff --git a/test/external/external_model_benchmark.py b/test/external/external_model_benchmark.py index d6a57eca..becfe237 100644 --- a/test/external/external_model_benchmark.py +++ b/test/external/external_model_benchmark.py @@ -14,10 +14,12 @@ from tinygrad.ops import Device MODELS = { "resnet50": "https://github.com/onnx/models/raw/main/vision/classification/resnet/model/resnet50-caffe2-v1-9.onnx", - "openpilot": "https://github.com/commaai/openpilot/raw/7da48ebdba5e3cf4c0b8078c934bee9a199f0280/selfdrive/modeld/models/supercombo.onnx", + # numerical issue with v0.9.4 + # "openpilot": "https://github.com/commaai/openpilot/raw/v0.9.4/selfdrive/modeld/models/supercombo.onnx", "efficientnet": "https://github.com/onnx/models/raw/main/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11.onnx", "shufflenet": "https://github.com/onnx/models/raw/main/vision/classification/shufflenet/model/shufflenet-9.onnx", - "commavq": "https://github.com/commaai/commavq/raw/master/models/gpt2m.onnx", + # cannot download the model from github + # "commavq": "https://github.com/commaai/commavq/raw/master/models/gpt2m.onnx", # broken in torch MPS #"zfnet": "https://github.com/onnx/models/raw/main/vision/classification/zfnet-512/model/zfnet512-9.onnx", diff --git a/test/models/test_onnx.py b/test/models/test_onnx.py index 156dd521..feffdabc 100644 --- a/test/models/test_onnx.py +++ b/test/models/test_onnx.py @@ -21,8 +21,7 @@ def run_onnx_torch(onnx_model, inputs): torch_out = torch_model(*[torch.tensor(x) for x in inputs.values()]) return torch_out -OPENPILOT_MODEL = "https://github.com/commaai/openpilot/raw/7da48ebdba5e3cf4c0b8078c934bee9a199f0280/selfdrive/modeld/models/supercombo.onnx" -#OPENPILOT_MODEL = "https://github.com/commaai/openpilot/raw/1f2f9ea9c9dc37bdea9c6e32e4cb8f88ea0a34bf/selfdrive/modeld/models/supercombo.onnx" +OPENPILOT_MODEL = "https://github.com/commaai/openpilot/raw/v0.9.4/selfdrive/modeld/models/supercombo.onnx" np.random.seed(1337) @@ -35,11 +34,11 @@ class TestOnnxModel(unittest.TestCase): np_inputs = { "input_imgs": np.random.randn(*(1, 12, 128, 256)), "big_input_imgs": np.random.randn(*(1, 12, 128, 256)), - "desire": np.zeros((1, 8)), + "desire": np.zeros((1, 100, 8)), "traffic_convention": np.array([[1., 0.]]), - "initial_state": np.zeros((1, 512)) - #"initial_state": np.zeros((1, 768)) - } + "nav_features": np.zeros((1, 256)), + "features_buffer": np.zeros((1, 99, 128)), + } inputs = {k:Tensor(v.astype(np.float32), requires_grad=False) for k,v in np_inputs.items()} return inputs @@ -79,10 +78,10 @@ class TestOnnxModel(unittest.TestCase): inputs = { "input_imgs": np.random.randn(*(1, 12, 128, 256)), "big_input_imgs": np.random.randn(*(1, 12, 128, 256)), - "desire": np.zeros((1, 8)), + "desire": np.zeros((1, 100, 8)), "traffic_convention": np.array([[1., 0.]]), - "initial_state": np.zeros((1, 512)) - #"initial_state": np.zeros((1, 768)) + "nav_features": np.zeros((1, 256)), + "features_buffer": np.zeros((1, 99, 128)), } inputs = {k:v.astype(np.float32) for k,v in inputs.items()}