No more magic for the can chunks (#1191)

* no more magic

* bump version

* no need for this

* comment cleanup

* add comms_can_reset tests

* we can't ensure this for compatibility reasons

* add CANPacket_t checksum

python/__init__.py

@@ -29,50 +29,47 @@ BASEDIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
 
 DEBUG = os.getenv("PANDADEBUG") is not None
 
-CAN_TRANSACTION_MAGIC = struct.pack("<I", 0x43414E2F)
 USBPACKET_MAX_SIZE = 0x40
-CANPACKET_HEAD_SIZE = 0x5
+CANPACKET_HEAD_SIZE = 0x6
 DLC_TO_LEN = [0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20, 24, 32, 48, 64]
 LEN_TO_DLC = {length: dlc for (dlc, length) in enumerate(DLC_TO_LEN)}
 
+def calculate_checksum(data):
+  res = 0
+  for b in data:
+    res ^= b
+  return res
+
 def pack_can_buffer(arr):
-  snds = [CAN_TRANSACTION_MAGIC]
+  snds = [b'']
   for address, _, dat, bus in arr:
     assert len(dat) in LEN_TO_DLC
     #logging.debug("  W 0x%x: 0x%s", address, dat.hex())
 
     extended = 1 if address >= 0x800 else 0
     data_len_code = LEN_TO_DLC[len(dat)]
-    header = bytearray(5)
+    header = bytearray(CANPACKET_HEAD_SIZE)
     word_4b = address << 3 | extended << 2
     header[0] = (data_len_code << 4) | (bus << 1)
     header[1] = word_4b & 0xFF
     header[2] = (word_4b >> 8) & 0xFF
     header[3] = (word_4b >> 16) & 0xFF
     header[4] = (word_4b >> 24) & 0xFF
+    header[5] = calculate_checksum(header[:5] + dat)
 
     snds[-1] += header + dat
     if len(snds[-1]) > 256: # Limit chunks to 256 bytes
-      snds.append(CAN_TRANSACTION_MAGIC)
+      snds.append(b'')
 
   return snds
 
 def unpack_can_buffer(dat):
   ret = []
-  if len(dat) < len(CAN_TRANSACTION_MAGIC):
-    return ret
-
-  if dat[:len(CAN_TRANSACTION_MAGIC)] != CAN_TRANSACTION_MAGIC:
-    logging.error("CAN: recv didn't start with magic")
-    return ret
-
-  dat = dat[len(CAN_TRANSACTION_MAGIC):]
 
   while len(dat) >= CANPACKET_HEAD_SIZE:
     data_len = DLC_TO_LEN[(dat[0]>>4)]
 
     header = dat[:CANPACKET_HEAD_SIZE]
-    dat = dat[CANPACKET_HEAD_SIZE:]
 
     bus = (header[0] >> 1) & 0x7
     address = (header[4] << 24 | header[3] << 16 | header[2] << 8 | header[1]) >> 3
@@ -84,17 +81,18 @@ def unpack_can_buffer(dat):
       # rejected
       bus += 192
 
-    data = dat[:data_len]
-    dat = dat[data_len:]
+    # we need more from the next transfer
+    if data_len > len(dat) - CANPACKET_HEAD_SIZE:
+      break
 
-    #logging.debug("  R 0x%x: 0x%s", address, data.hex())
+    assert calculate_checksum(dat[:(CANPACKET_HEAD_SIZE+data_len)]) == 0, "CAN packet checksum incorrect"
+
+    data = dat[CANPACKET_HEAD_SIZE:(CANPACKET_HEAD_SIZE+data_len)]
+    dat = dat[(CANPACKET_HEAD_SIZE+data_len):]
 
     ret.append((address, 0, data, bus))
 
-  if len(dat) > 0:
-    logging.error("CAN: malformed packet. leftover data")
-
-  return ret
+  return (ret, dat)
 
 def ensure_health_packet_version(fn):
   @wraps(fn)
@@ -185,11 +183,11 @@ class Panda:
   HW_TYPE_RED_PANDA_V2 = b'\x08'
   HW_TYPE_TRES = b'\x09'
 
-  CAN_PACKET_VERSION = 3
+  CAN_PACKET_VERSION = 4
   HEALTH_PACKET_VERSION = 11
-  CAN_HEALTH_PACKET_VERSION = 3
+  CAN_HEALTH_PACKET_VERSION = 4
   HEALTH_STRUCT = struct.Struct("<IIIIIIIIIBBBBBBHBBBHfBB")
-  CAN_HEALTH_STRUCT = struct.Struct("<BIBBBBBBBBIIIIIIHHBBB")
+  CAN_HEALTH_STRUCT = struct.Struct("<BIBBBBBBBBIIIIIIIHHBBB")
 
   F2_DEVICES = (HW_TYPE_PEDAL, )
   F4_DEVICES = (HW_TYPE_WHITE_PANDA, HW_TYPE_GREY_PANDA, HW_TYPE_BLACK_PANDA, HW_TYPE_UNO, HW_TYPE_DOS)
@@ -235,10 +233,15 @@ class Panda:
     self._handle = None
     self._bcd_device = None
 
+    self.can_rx_overflow_buffer = b''
+
     # connect and set mcu type
     self._spi = spi
     self.connect(claim)
 
+    # reset comms
+    self.can_reset_communications()
+
   def close(self):
     self._handle.close()
     self._handle = None
@@ -513,11 +516,12 @@ class Panda:
       "total_tx_cnt": a[13],
       "total_rx_cnt": a[14],
       "total_fwd_cnt": a[15],
-      "can_speed": a[16],
-      "can_data_speed": a[17],
-      "canfd_enabled": a[18],
-      "brs_enabled": a[19],
-      "canfd_non_iso": a[20],
+      "total_tx_checksum_error_cnt": a[16],
+      "can_speed": a[17],
+      "can_data_speed": a[18],
+      "canfd_enabled": a[19],
+      "brs_enabled": a[20],
+      "canfd_non_iso": a[21],
     }
 
   # ******************* control *******************
@@ -652,6 +656,9 @@ class Panda:
   # Timeout is in ms. If set to 0, the timeout is infinite.
   CAN_SEND_TIMEOUT_MS = 10
 
+  def can_reset_communications(self):
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xc0, 0, 0, b'')
+
   @ensure_can_packet_version
   def can_send_many(self, arr, timeout=CAN_SEND_TIMEOUT_MS):
     snds = pack_can_buffer(arr)
@@ -681,7 +688,8 @@ class Panda:
       except (usb1.USBErrorIO, usb1.USBErrorOverflow):
         print("CAN: BAD RECV, RETRYING")
         time.sleep(0.1)
-    return unpack_can_buffer(dat)
+    msgs, self.can_rx_overflow_buffer = unpack_can_buffer(self.can_rx_overflow_buffer + dat)
+    return msgs
 
   def can_clear(self, bus):
     """Clears all messages from the specified internal CAN ringbuffer as