Make updated more resilient (#1853)

* Add sleep at updated startup

* Mitigate symlink-related FS damage
old-commit-hash: c43df81214

launch_chffrplus.sh

@@ -41,9 +41,12 @@ function launch {
 
           mv $BASEDIR /data/safe_staging/old_openpilot
           mv "${STAGING_ROOT}/finalized" $BASEDIR
+          cd $BASEDIR
 
-          # The mv changed our working directory to /data/safe_staging/old_openpilot
-          cd "${BASEDIR}"
+          # Partial mitigation for symlink-related filesystem corruption
+          # Ensure all files match the repo versions after update
+          git reset --hard
+          git submodule foreach --recursive git reset --hard
 
           echo "Restarting launch script ${LAUNCHER_LOCATION}"
           exec "${LAUNCHER_LOCATION}"

selfdrive/updated.py

@@ -32,6 +32,7 @@ import signal
 from pathlib import Path
 import fcntl
 import threading
+import time
 from cffi import FFI
 
 from common.basedir import BASEDIR
@@ -336,6 +337,10 @@ def main():
   except IOError:
     raise RuntimeError("couldn't get overlay lock; is another updated running?")
 
+  # Wait a short time before our first update attempt
+  # Avoids race with IsOffroad not being set, reduces manager startup load
+  time.sleep(30)
+
   while True:
     update_failed_count += 1
     time_wrong = datetime.datetime.utcnow().year < 2019