jenkins: improve overall robustness (#23117)

* test onroad: more reliable log finding

* Update selfdrive/test/test_onroad.py

* write continue.sh in device setup

* clone

* try different delimeter

* try this instead

* fix keys path

* executable

* remove orphaned lock

Jenkinsfile

@@ -1,7 +1,7 @@
 def phone(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
-ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'EOF'
+ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END'
 
 set -e
 
@@ -29,7 +29,7 @@ cd ${env.TEST_DIR} || true
 ${cmd}
 exit 0
 
-EOF"""
+END"""
 
     sh script: ssh_cmd, label: step_label
   }

selfdrive/test/setup_device_ci.sh

@@ -1,4 +1,6 @@
-#!/usr/bin/bash -e
+#!/usr/bin/bash
+
+set -e
 
 if [ -z "$SOURCE_DIR" ]; then
   echo "SOURCE_DIR must be set"
@@ -24,8 +26,35 @@ if [ -f "/EON" ]; then
   rm -rf /data/safe_staging
 fi
 
+export KEYS_PATH="/usr/comma/setup_keys"
+export CONTINUE_PATH="/data/continue.sh"
+if [ -f "/EON" ]; then
+  export KEYS_PATH="/data/data/com.termux/files/home/setup_keys"
+  export CONTINUE_PATH="/data/data/com.termux/files/continue.sh"
+fi
+tee $CONTINUE_PATH << EOF
+#!/usr/bin/bash
+
+PARAMS_ROOT="/data/params/d"
+
+while true; do
+  mkdir -p \$PARAMS_ROOT
+  cp $KEYS_PATH \$PARAMS_ROOT/GithubSshKeys
+  echo -n 1 > \$PARAMS_ROOT/SshEnabled
+  sleep 1m
+done
+
+sleep infinity
+EOF
+chmod +x $CONTINUE_PATH
+
 # set up environment
+if [ ! -d "$SOURCE_DIR" ]; then
+  git clone https://github.com/commaai/openpilot.git $SOURCE_DIR
+fi
 cd $SOURCE_DIR
+
+rm -f .git/index.lock
 git reset --hard
 git fetch
 find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;

selfdrive/test/test_onroad.py

@@ -138,21 +138,18 @@ class TestOnroad(unittest.TestCase):
       cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2")))
       return
 
+    # setup env
     os.environ['REPLAY'] = "1"
     os.environ['SKIP_FW_QUERY'] = "1"
     os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
+
+    params = Params()
+    params.clear_all()
     set_params_enabled()
 
     # Make sure athena isn't running
-    Params().delete("DongleId")
-    Params().delete("AthenadPid")
     os.system("pkill -9 -f athena")
 
-    logger_root = Path(ROOT)
-    initial_segments = set()
-    if logger_root.exists():
-      initial_segments = set(Path(ROOT).iterdir())
-
     # start manager and run openpilot for a minute
     try:
       manager_path = os.path.join(BASEDIR, "selfdrive/manager/manager.py")
@@ -164,15 +161,19 @@ class TestOnroad(unittest.TestCase):
           sm.update(1000)
 
       # make sure we get at least two full segments
+      route = None
       cls.segments = []
       with Timeout(300, "timed out waiting for logs"):
+        while route is None:
+          route = params.get("CurrentRoute", encoding="utf-8")
+          time.sleep(0.1)
+
         while len(cls.segments) < 3:
-          new_paths = set()
-          if logger_root.exists():
-            new_paths = set(logger_root.iterdir()) - initial_segments
-          segs = [p for p in new_paths if "--" in str(p)]
+          segs = set()
+          if Path(ROOT).exists():
+            segs = set(Path(ROOT).glob(f"{route}--*"))
           cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1]))
-          time.sleep(5)
+          time.sleep(2)
 
     finally:
       proc.terminate()