From c82efc3e5ee4619e6ffc638911659b3dc184fdaf Mon Sep 17 00:00:00 2001
From: Boris Kolpackov <boris@codesynthesis.com>
Date: Wed, 29 Mar 2017 14:33:31 +0200
Subject: Implement toolchain fetching and monitoring

---
 buildos | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 181 insertions(+), 10 deletions(-)

(limited to 'buildos')

diff --git a/buildos b/buildos
index 9a0224a..c91c554 100755
--- a/buildos
+++ b/buildos
@@ -9,10 +9,6 @@
 # 4. Monitor for OS and toolchain changes and reboot if detected.
 #
 
-# @@ What do we do in case or error, trap? Send email? Reboot? Probably
-#    reboot bad idea since someone may need to login to examine what's
-#    going on.
-#
 # @@ What will systemd do if we fail? Perhaps configure it to restart
 #    us? Or not since we may hose the logs.
 #
@@ -81,20 +77,192 @@ function restart ()
   sudo systemctl reboot
 }
 
+email "starting buildos monitor" <<EOF
+buildid:       $buildid
+buildid_url:   $buildid_url
+toolchain_url: $toolchain_url
+EOF
+
 if [ -z "$buildid_url" ]; then
   info "no buildos.buildid_url specified, not monitoring for new os builds"
 fi
 
-email "starting buildos monitor" <<EOF
-buildid:     $buildid
-buildid_url: $buildid_url
-EOF
+if [ -z "$toolchain_url" ]; then
+  info "no buildos.toolchain_url specified, not bootstrapping"
+else
+  # The toolchain "sums" file (a list of SHA sums and relative file names, as
+  # produced by shaNNNsum). The first entry should always be build2-toolchain
+  # tar archive itself (which we use to figure out the version). Blank lines
+  # and lines that start with '#' are ignored.
+  #
+  tc_url="$toolchain_url"
+  tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")"
+  tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")"
+  tc_root="/build/tftp"
+  tc_path="$tc_root/$tc_file"
+  tc_ver=
+fi
+
+# Calculate the file checksum using the shaNNNsum utility.
+#
+function tc_checksum () # <file>
+{
+  "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p'
+}
+
+# Fetch a file from the sums file into $tc_root, verify its checksum, and make
+# a predictable name (without version) symlink.
+#
+function tc_fetch () # <line>
+{
+  local s p f u l
+
+  s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")"         # Checksum.
+  p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")"    # File path (rel).
+  f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")"      # File name.
+  u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL.
+
+  if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then
+    info "invalid sum line '$1'"
+    return 1
+  fi
+
+  # Extract the version.
+  #
+  if [ -z "$tc_ver" ]; then
+    tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+
+    if [ -z "$tc_ver" ]; then
+      info "unable to extract toolchain version from '$f'"
+      return 1
+    fi
+
+    info "toolchain version is $tc_ver"
+    echo "$tc_ver" >"$tc_root/toolchain-version"
+  fi
+
+  # Derive a predictable name link.
+  #
+  l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")"
+
+  if [ -z "$l" ]; then
+    info "unable to derive predicatable name from '$f', '$tc_ver'"
+    return 1
+  fi
+
+  # Fetch the file.
+  #
+  info "fetching $u [$l]"
+
+  if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then
+    info "unable to fetch $u"
+    return 1
+  fi
+
+  # Verify the checksum.
+  #
+  info "verifying checksum for $f"
+
+  local n
+  n="$(tc_checksum "$tc_root/$f")"
+
+  if [ "$n" != "$s" ]; then
+    info "$tc_sum checksum mismatch for $u"
+    info "  expected:   $s"
+    info "  calculated: $n"
+    return 1
+  fi
 
+  # Make the link.
+  #
+  ln -s "$f" "$tc_root/$l"
+}
+
+# Bootstrap the toolchain.
+#
+function tc_bootstrap ()
+{
+  local l ls=()
+
+  # Fetch files according to the sums file. Skip empty line and those that
+  # start with '#'.
+  #
+  readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path")
+
+  for l in "${ls[@]}"; do
+    if ! tc_fetch "$l"; then
+      return 1 # Diagnostics has already been issued.
+    fi
+  done
+}
+
+# Monitoring loop.
+#
 while true; do
-  info "monitoring..."
 
-  sleep 5
+  # Check for toolchain changes. If this is the first run, bootstrap it.
+  #
+  if [ -n "$tc_url" ]; then
+
+    # Fetch the toolchain sums either to $tc_path if this is the first time
+    # or to $tc_path.new if we are checking for changes.
+    #
+    if [ -e "$tc_path" ]; then
+      f="$tc_path.new"
+    else
+      f="$tc_path"
+    fi
+
+    if curl -f -L -s -S -o "$f" "$tc_url"; then
 
+      # Take care of change detection.
+      #
+      if [ "$f" != "$tc_path" ]; then
+
+	n="$(tc_checksum "$f")"
+
+	if [ "$tc_file_sum" != "$n" ]; then
+	  email "rebooting because of new toolchain" <<EOF
+old_toolchain: $tc_file_sum
+new_toolchain: $n
+EOF
+	  info "new toolchain ($n), rebooting..."
+	  restart
+	fi
+      else
+	# This is the first run, bootstrap the toolchain.
+	#
+	tc_file_sum="$(tc_checksum "$f")"
+
+	# If we fail, we simply wait for a new toolchain to be uploaded (or
+	# some manual intervention).
+	#
+	# Note that because of the pipe tc_bootstrap() will run in subshell
+	# and any variables it sets (like tc_ver) won't be visible to us.
+	#
+	info "bootstrapping toolchain..."
+
+	tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2
+
+	if [ "${PIPESTATUS[0]}" -eq 0 ]; then
+	  tc_ver="$(cat $tc_root/toolchain-version)"
+	  s="bootstrapped toolchain $tc_ver"
+	else
+	  s="failed to bootstrap toolchain, waiting for new version"
+	fi
+
+	email "$s" <<EOF
+toolchain_log: tftp://$hname/toolchain.log
+EOF
+      fi
+    else
+      info "unable to fetch $tc_url, will try again"
+      rm -f "$f"
+    fi
+  fi
+
+  # Check for OS changes.
+  #
   if [ -n "$buildid_url" ]; then
     # Fetch the current id. While normally it will be a TFTP URL, it could also
     # be HTTP(S) so we configure sensible behavior for that.
@@ -112,4 +280,7 @@ EOF
       info "unable to fetch $buildid_url, will try again"
     fi
   fi
+
+  info "monitoring..."
+  sleep 10
 done
-- 
cgit v1.1