#!/bin/bash # Build OS monitor. It starts as a systemd service and performs the following # steps: # # 1. Bootstrap the build2 toolchain. # 2. Build and start bbot. # 3. Build and start bslave. # 4. Monitor for OS and toolchain changes and reboot if detected. # # @@ What will systemd do if we fail? Perhaps configure it to restart # us? Or not since we may hose the logs. # trap "exit 1" ERR set -o errtrace # Trap in functions. # Note: diagnostics goes to stdout. # function info () { echo "$*" 1>&2; } function error () { if [ "$#" -gt 0 ]; then info "$*"; fi exit 1 } info "starting buildos monitor..." # Parse the kernel command line. This is complicated by the fact that the # values can be quoted, for example: # # foo='foo fox' # bar="bar 'box'" # # First we separete quoted variables and arguments with newlines (giving # priority to assignments). Then we replace whitespaces with newline on # lines that don't contain quites. Finally, clean up by removing blank # lines. # # Note: the same code as in init. # readarray -t cmdline < <(cat /proc/cmdline | \ sed -r -e "s/([^ ]+=)?('[^']*'|\"[^\"]*\")/\n\1\2\n/g" | \ sed -r -e "/['\"]/!s/ /\n/g" | sed -r -e '/^\s*$/d') # Enter all buildos variables as bash variables. # for v in "${cmdline[@]}"; do var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. if [ -n "$var" ]; then val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. declare "$var=$val" fi done hname="$(hostname)" # Get the build id. # buildid="$(sed -n -re 's/^BUILD_ID="(.+)"$/\1/p' /etc/os-release)" function email () # < { (echo -e "Subject: [$hname] $1\n"; cat -) | sendmail -i "$admin_email" } function restart () { sendmail -q # Flush mail queue. sleep 10 # Give any remaining mail chance to go through. sudo systemctl reboot } email "starting buildos monitor" < { "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p' } # Fetch a file from the sums file into $tc_root, verify its checksum, and make # a predictable name (without version) symlink. # function tc_fetch () # { local s p f u l s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")" # Checksum. p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")" # File path (rel). f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL. if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then info "invalid sum line '$1'" return 1 fi # Extract the version. # if [ -z "$tc_ver" ]; then tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" if [ -z "$tc_ver" ]; then info "unable to extract toolchain version from '$f'" return 1 fi info "toolchain version is $tc_ver" echo "$tc_ver" >"$tc_root/toolchain-version" fi # Derive a predictable name link. # l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")" if [ -z "$l" ]; then info "unable to derive predicatable name from '$f', '$tc_ver'" return 1 fi # Fetch the file. # info "fetching $u [$l]" if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then info "unable to fetch $u" return 1 fi # Verify the checksum. # info "verifying checksum for $f" local n n="$(tc_checksum "$tc_root/$f")" if [ "$n" != "$s" ]; then info "$tc_sum checksum mismatch for $u" info " expected: $s" info " calculated: $n" return 1 fi # Make the link. # ln -s "$f" "$tc_root/$l" } # Bootstrap the toolchain. # function tc_bootstrap () { local l ls=() # Fetch files according to the sums file. Skip empty line and those that # start with '#'. # readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path") for l in "${ls[@]}"; do if ! tc_fetch "$l"; then return 1 # Diagnostics has already been issued. fi done } # Monitoring loop. # while true; do # Check for toolchain changes. If this is the first run, bootstrap it. # if [ -n "$tc_url" ]; then # Fetch the toolchain sums either to $tc_path if this is the first time # or to $tc_path.new if we are checking for changes. # if [ -e "$tc_path" ]; then f="$tc_path.new" else f="$tc_path" fi if curl -f -L -s -S -o "$f" "$tc_url"; then # Take care of change detection. # if [ "$f" != "$tc_path" ]; then n="$(tc_checksum "$f")" if [ "$tc_file_sum" != "$n" ]; then email "rebooting because of new toolchain" <&1 | tee "$tc_root/toolchain.log" 1>&2 if [ "${PIPESTATUS[0]}" -eq 0 ]; then tc_ver="$(cat $tc_root/toolchain-version)" s="bootstrapped toolchain $tc_ver" else s="failed to bootstrap toolchain, waiting for new version" fi email "$s" <