Ship non-sparse system image in OTA (#275)

* removed system-skip-chunks image

* removed sparsed system.img

* clean github workflow

* copy image to output

* fake system-skip-chunks [upload]

* sparse key must exist otherwise KeyError: 'sparse' in openpilot

* reduce image size to 5G otherwise MemoryError: Unable to allocate output buffer. in openpilot

* [upload]

* Revert "fake system-skip-chunks"

This reverts commit bcd11e7e32.

* recreate 10G image [upload]

* resize2fs

* resize2fs shrink image

* revert skip-chunks image

* cleanup

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

.github/workflows/push-and-comment.yaml

@@ -49,7 +49,7 @@ jobs:
       - name: Push boot, system and agnos.json
         working-directory: ${{ github.workspace }}/ci-artifacts
         run: |
-          mv ota.json agnos.json && rm ota-staging.json system-skip-chunks-*.img.xz
+          mv ota.json agnos.json && rm ota-staging.json
           git checkout -b agnos-builder/pr-${{ env.PR_NUMBER }}
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"

Dockerfile.builder

@@ -17,8 +17,9 @@ RUN apt-get update && \
     python-is-python2 \
     openssl \
     ccache \
-    android-sdk-libsparse-utils \
     libcap2-bin \
+    android-sdk-libsparse-utils \
+    e2fsprogs \
     && rm -rf /var/lib/apt/lists/*
 
 RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \

build_system.sh

@@ -14,11 +14,14 @@ BUILD_DIR="$DIR/build"
 OUTPUT_DIR="$DIR/output"
 
 ROOTFS_DIR="$BUILD_DIR/agnos-rootfs"
-ROOTFS_IMAGE="$BUILD_DIR/system.img.raw"
-ROOTFS_IMAGE_SIZE=10G
-SPARSE_IMAGE="$OUTPUT_DIR/system.img"
+ROOTFS_IMAGE="$BUILD_DIR/system.img"
 SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img"
 
+# the partition is 10G, but openpilot's updater didn't always handle the full size
+# - the size will also get shrunk with "resize2fs -M"
+# - openpilot fix, shipped in 0.9.8 (8/18/24): https://github.com/commaai/openpilot/pull/33320
+ROOTFS_IMAGE_SIZE=5G
+
 # Create temp dir if non-existent
 mkdir -p $BUILD_DIR $OUTPUT_DIR
 
@@ -129,18 +132,20 @@ exec_as_root bash -c "set -e; export ROOTFS_DIR=$ROOTFS_DIR GIT_HASH=$GIT_HASH;
 echo "Unmount filesystem"
 exec_as_root umount -l $ROOTFS_DIR
 
-# Sparsify
-echo "Sparsify image $(basename $SPARSE_IMAGE)"
-exec_as_user bash -c "\
-TMP_SPARSE=\$(mktemp); \
-img2simg $ROOTFS_IMAGE \$TMP_SPARSE; \
-mv \$TMP_SPARSE $SPARSE_IMAGE"
-
 # Make image with skipped chunks
 echo "Sparsify image $(basename $SKIP_CHUNKS_IMAGE)"
 exec_as_user bash -c "\
+TMP_SPARSE=\$(mktemp); \
+img2simg $ROOTFS_IMAGE \$TMP_SPARSE; \
 TMP_SKIP=\$(mktemp); \
-$DIR/tools/simg2dontcare.py $SPARSE_IMAGE \$TMP_SKIP; \
+$DIR/tools/simg2dontcare.py $TMP_SPARSE \$TMP_SKIP; \
 mv \$TMP_SKIP $SKIP_CHUNKS_IMAGE"
 
+# Reduce system image to the minimum size
+exec_as_user e2fsck -fy $ROOTFS_IMAGE
+exec_as_user resize2fs -M $ROOTFS_IMAGE
+
+# Copy system image to output
+cp $ROOTFS_IMAGE $OUTPUT_DIR
+
 echo "Done!"

load_kernel.sh

@@ -9,6 +9,7 @@ sudo dd if=/data/tmp/boot.img of=/dev/disk/by-partlabel/boot_a
 sudo dd if=/data/tmp/boot.img of=/dev/disk/by-partlabel/boot_b
 
 sudo mount -o rw,remount /
+sudo resize2fs $(findmnt -n -o SOURCE /)
 sudo mv /data/tmp/wlan.ko /usr/comma/wlan.ko
 rm -rf /data/tmp/*
 sudo mount -o ro,remount / || true

scripts/check-space.sh

@@ -2,6 +2,6 @@
 
 # sudo apt install ncdu
 
-sudo mount build/system.img.raw build/agnos-rootfs
+sudo mount build/system.img build/agnos-rootfs
 sudo ncdu build/agnos-rootfs/ || true
 sudo umount build/agnos-rootfs

scripts/package_ota.py

@@ -25,23 +25,12 @@ def compress(fin, fout) -> None:
   subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True)
 
 
-def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None):
+def process_file(fn, name, full_check=True, has_ab=True, alt=None):
   print(name)
   hash_raw = hash = checksum(fn)
   size = fn.stat().st_size
   print(f"  {size} bytes, hash {hash}")
 
-  if sparse:
-    raw_img = BUILD_DIR / "system.img.raw"
-    if raw_img.exists():
-      print("  using existing raw image")
-      hash_raw = checksum(raw_img)
-      size = raw_img.stat().st_size
-    else:
-      print("Error: existing raw image not found")
-      exit(1)
-    print(f"  {size} bytes, hash {hash_raw} (raw)")
-
   print("  compressing")
   xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz"
   compress(fn, xz_fn)
@@ -52,7 +41,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None)
     "hash": hash,
     "hash_raw": hash_raw,
     "size": size,
-    "sparse": sparse,
+    "sparse": False,
     "full_check": full_check,
     "has_ab": has_ab,
   }
@@ -81,7 +70,7 @@ if __name__ == "__main__":
 
   files = [
     process_file(OUTPUT_DIR / "boot.img", "boot"),
-    process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"),
+    process_file(OUTPUT_DIR / "system.img", "system", full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"),
   ]
   configs = [
     (AGNOS_UPDATE_URL, "ota.json"),

scripts/setup-on-device.sh

@@ -10,6 +10,7 @@ if [ ! -f /AGNOS ]; then
 fi
 
 sudo mount -o rw,remount /
+sudo resize2fs $(findmnt -n -o SOURCE /)
 
 echo "symlink /usr/comma"
 sudo rm -rf /usr/comma

userspace/usr/comma/apt_setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 sudo mount -o rw,remount /
+sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /)
 sudo mount -o remount,size=1500M /var
 sudo cp -r /usr/default/var/lib/dpkg /var/lib/
 sudo sed -i '/bionic/s/^/#/' /etc/apt/sources.list

userspace/usr/comma/shims/pip

@@ -7,6 +7,7 @@ export TMPDIR=/tmp/pip-tmp
 mkdir -p $TMPDIR
 sudo mount -o remount,size=2G /tmp
 sudo mount -o rw,remount /
+sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /)
 
 # run command
 sudo TMPDIR=$TMPDIR PIP_NO_CACHE_DIR=1 $PIP_PATH "$@"