Add support for KVM, monitoring bbot diagnostics

1 files changed, 41 insertions, 1 deletions

diff --git a/buildos b/buildos
index 0461e68..6c3e008 100755
--- a/buildos
+++ b/buildos
@@ -77,6 +77,14 @@ for v in "${cmdline[@]}"; do
   fi
 done
 
+# Make sure /dev/kvm has correct permissions (while all the udev scripts seem
+# to be there, it does not work for some reason).
+#
+if sudo test -c /dev/kvm; then
+  sudo chgrp kvm /dev/kvm
+  sudo chmod 0660 /dev/kvm
+fi
+
 hname="$(hostname)"
 
 # Get the build id.
@@ -721,6 +729,7 @@ function bbot_start () # <toolchain-name> <toolchain-index>
 # start their bbot agents.
 #
 toolchain_boots=()
+declare -A toolchain_cursors # Latest systemd journal cursor.
 
 # Monitoring loop.
 #
@@ -844,7 +853,38 @@ EOF
 	    sudo systemctl reset-failed "bbot-agent@$tn" 2>&1 | \
 	      tee -a "$tr/bbot-$count.log" 1>&2
 	  else
-	    continue # Nothing to do.
+	    # See if there is any diagnostics in the systemd journal. We
+	    # notify about warning and up.
+	    #
+	    # The old versions journalctl behavior is to not output anything
+	    # (not even the cursor) if there are no new entries. The new
+	    # versions output the old cursor.
+	    #
+	    c=(sudo journalctl --unit "bbot-agent@$tn")
+
+	    # Get the last cursor if any.
+	    #
+	    oc="${toolchain_cursors["$tn"]}"
+	    if [ -n "$oc" ]; then
+	      c+=("--after-cursor" "$oc")
+	    fi
+
+	    nc="$("${c[@]}" --no-pager --quiet --priority 4 --show-cursor | \
+sed -n -re 's/^-- cursor: (.+)$/\1/p')"
+
+	    # If we have no new entries, then nothing to do.
+	    #
+	    if [ -n "$nc" -a "$nc" != "$oc" ]; then
+	      s="bbot-agent@$tn service issued new diagnostics"
+
+	      info "$s"
+	      email "$s" <<EOF
+$tn.bbot_cmd: ssh build@$hname ${c[@]}
+EOF
+	      toolchain_cursors["$tn"]="$nc"
+	    fi
+
+	    continue
 	  fi
 	  ;;
 	1)