Fix /var/ filling up once and for all (#451)

* fix filling up /var/ with journal logs

* Revert "fix filling up /var/ with journal logs"

This reverts commit 5485630fb3.

* can't believe this is still an issue

* and another time bug

* one more

* fix

* revert that

* cleanup

Dockerfile.agnos

@@ -166,6 +166,7 @@ COPY ./userspace/files/allow-modem-control.pkla /etc/polkit-1/localauthority/50-
 COPY ./userspace/files/*.rules /etc/udev/rules.d/
 COPY ./userspace/files/ssh*_config /etc/ssh/
 COPY ./userspace/files/logrotate.conf /etc/
+COPY ./userspace/files/rsyslog /etc/logrotate.d/
 RUN chmod 644 /etc/logrotate.conf
 RUN touch -r /lib/systemd/systemd /etc/fstab
 

userspace/files/rsyslog

@@ -0,0 +1,18 @@
+/var/log/syslog
+/var/log/mail.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/cron.log
+{
+	rotate 4
+	size 10M
+	missingok
+	notifempty
+	compress
+	delaycompress
+	sharedscripts
+	postrotate
+		/usr/lib/rsyslog/rsyslog-rotate
+	endscript
+}

userspace/files/varwatch.service

@@ -0,0 +1,9 @@
+[Unit]
+After=multi-user.target
+
+[Service]
+Restart=always
+ExecStart=/usr/local/venv/bin/python -u /usr/comma/varwatch.py
+
+[Install]
+WantedBy=multi-user.target

userspace/services.sh

@@ -16,6 +16,7 @@ systemctl enable sound.service
 systemctl enable weston.service
 systemctl enable weston-ready.service
 systemctl enable init-qcom.service
+systemctl enable varwatch.service
 systemctl enable power_drop_monitor.service
 systemctl enable brightnessd.service
 systemctl enable ssh-param-watcher.path
@@ -61,6 +62,7 @@ systemctl disable update-notifier-download.timer
 systemctl disable update-notifier-download.service
 systemctl disable update-notifier-motd.timer
 systemctl disable update-notifier-motd.service
+systemctl disable man-db.timer
 
 # Disable NFS stuff by default
 systemctl disable rpcbind

userspace/usr/comma/fs_setup.sh

@@ -6,6 +6,11 @@
 # tmpfiles
 systemd-tmpfiles --create /usr/comma/tmpfiles.conf
 
+# /var/log/ tmpfs
+mkdir -p /var/log/
+chown root:syslog /var/log
+mount -t tmpfs -o rw,nosuid,nodev,size=128M,mode=755 tmpfs /var/log
+
 # setup /home
 mkdir -p /rwtmp/home_work
 mkdir -p /rwtmp/home_upper

userspace/usr/comma/tests/varspam.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+RATE=${1:-1}
+
+log_message() {
+  # /var/log/syslog
+  cat /usr/include/sqlite3.h | systemd-cat -t SPAM_TEST
+
+  # /var/log/kern.log
+  #cat /usr/include/sqlite3.h | sudo tee /dev/kmsg > /dev/null
+}
+
+#minutely verify config is good
+sudo logrotate -d /etc/logrotate.conf
+
+sudo rm -rf /var/log/*
+sudo systemctl restart rsyslog
+sudo systemctl restart systemd-journald
+sudo systemctl restart logrotate-hourly.timer
+
+while true; do
+  for i in $(seq 1 $RATE); do
+    log_message
+  done
+
+  echo
+  df -h /var/
+  sudo du -hs /var/log/* || true
+
+  sleep 1
+done

userspace/usr/comma/varspam.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+RATE=${1:-1}
+
+log_message() {
+  # /var/log/syslog
+  cat /usr/include/sqlite3.h | systemd-cat -t SPAM_TEST
+
+  # /var/log/kern.log
+  #cat /usr/include/sqlite3.h | sudo tee /dev/kmsg > /dev/null
+}
+
+# verify config is good
+sudo logrotate -d /etc/logrotate.conf
+
+sudo rm -rf /var/log/*
+sudo systemctl daemon-reload
+sudo systemctl restart rsyslog
+sudo systemctl restart systemd-journald
+sudo systemctl restart logrotate-hourly.timer
+
+while true; do
+  for i in $(seq 1 $RATE); do
+    log_message
+  done
+
+  echo
+  df -h /var/
+  sudo du -hs /var/log/* || true
+
+  sleep 1
+done

userspace/usr/comma/varwatch.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+import os
+import time
+
+if __name__ == "__main__":
+  # we are the last line of defense for /var/log filling up
+  while True:
+    usage = os.statvfs('/var/log')
+    percent = (usage.f_blocks - usage.f_bavail) / usage.f_blocks * 100
+    if percent > 70:
+      for fn in os.listdir('/var/log'):
+        file_path = os.path.join('/var/log', fn)
+        if os.path.isfile(file_path):
+          with open(file_path, 'w'):
+            pass
+    time.sleep(1)