#!/bin/bash # Build OS monitor. It starts as a systemd service and performs the following # steps: # # 1. Bootstrap the build2 toolchain. # 2. Build and start bbot. # 3. Build and start bslave. # 4. Monitor for OS and toolchain changes and reboot if detected. # # @@ What will systemd do if we fail? Perhaps configure it to restart # us? Or not since we may hose the logs. # owd="$(pwd)" trap "{ cd '$owd'; exit 1; }" ERR set -o errtrace # Trap in functions. # Note: diagnostics goes to stdout. # function info () { echo "$*" 1>&2; } function error () { if [ "$#" -gt 0 ]; then info "$*"; fi exit 1 } info "starting buildos monitor..." # Parse the kernel command line. This is complicated by the fact that the # values can be quoted, for example: # # foo='foo fox' # bar="bar 'box'" # # First we separete quoted variables and arguments with newlines (giving # priority to assignments). Then we replace whitespaces with newline on # lines that don't contain quites. Finally, clean up by removing blank # lines. # # Note: the same code as in init. # readarray -t cmdline < <(cat /proc/cmdline | \ sed -r -e "s/([^ ]+=)?('[^']*'|\"[^\"]*\")/\n\1\2\n/g" | \ sed -r -e "/['\"]/!s/ /\n/g" | sed -r -e '/^\s*$/d') # Enter all buildos variables as bash variables. # # Map of toolchain names (as specified in buildos..) to the # corresponding bash variable prefix. # declare -A toolchains toolchains["default"]="" for v in "${cmdline[@]}"; do var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. if [ -n "$var" ]; then val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. # If the variable contains a dot, then it is a toolchain variable. # if [[ "$var" == *.* ]]; then tn="$(sed -re 's/^[^.]+\.(.+)$/\1/' <<<"$var")" var="${tn}_$(sed -re 's/^([^.]+)\..+$/\1/' <<<"$var")" toolchains["$tn"]="${tn}_" fi declare "$var=$val" fi done hname="$(hostname)" # Get the build id. # buildid="$(sed -n -re 's/^BUILD_ID="(.+)"$/\1/p' /etc/os-release)" function email () # < { (echo -e "Subject: [$hname] $1\n"; cat -) | sendmail -i "$admin_email" } function restart () { sendmail -q # Flush mail queue. sleep 10 # Give any remaining mail chance to go through. sudo systemctl reboot } # Toolchain-related funtions. # # Return the value of one of the toolchain_* variables for this toolchain. # function tc_value () # { local n="${1}${2}" echo "${!n}" } # Calculate the file checksum using the shaNNNsum utility. # function tc_checksum () # { "$(tc_value "$1" toolchain_csum)sum" -b "$2" | \ sed -n -re 's/^([^ ]+) .+$/\1/p' } # Fetch a file from the sums file into $toolchain_root, verify its checksum, # and make a predictable name (without version) symlink. # function tc_fetch () # { local s p f u l tp tu tr tv tp="$1" tu="$(tc_value "$tp" toolchain_url)" tr="$(tc_value "$tp" toolchain_root)" s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$2")" # Checksum. p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$2")" # File path (relative). f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tu")/$p" # File URL. if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then info "invalid sum line '$2'" return 1 fi # Extract the version. # tv="$(tc_value "$tp" toolchain_ver)" if [ -z "$tv" ]; then tv="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" if [ -z "$tv" ]; then info "unable to extract toolchain version from '$f'" return 1 fi declare -g "${tp}toolchain_ver=$tv" info "toolchain version $tv" echo "$tv" >"$tr/version" fi # Derive a predictable name link. # l="$(sed -n -re "s/^(.+)-$tv(.*)$/\1\2/p" <<<"$f")" if [ -z "$l" ]; then info "unable to derive predicatable name from '$f' and '$tv'" return 1 fi # Fetch the file. # info "fetching $u [$l]" if ! curl -f -L -s -S -o "$tr/$f" "$u"; then info "unable to fetch $u" return 1 fi # Verify the checksum. # info "verifying checksum for $f" local cs cs="$(tc_checksum "$tp" "$tr/$f")" if [ "$cs" != "$s" ]; then info "checksum mismatch for $u" info " expected: $s" info " calculated: $cs" return 1 fi # Make the link. # ln -s "$f" "$tr/$l" } # Bootstrap the toolchain. # function tc_bootstrap () # { local tn="$1" local tp="${toolchains["$tn"]}" local tr="$(tc_value "$tp" toolchain_root)" local tf="$(tc_value "$tp" toolchain_file)" # Fetch files according to the sums file. Skip empty lines and those that # start with '#'. # local l ls=() readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tr/$tf") for l in "${ls[@]}"; do if ! tc_fetch "$tp" "$l"; then return 1 # Diagnostics has already been issued. fi done local tv="$(tc_value "$tp" toolchain_ver)" # Should be set by tc_fetch(). local tt="$(tc_value "$tp" toolchain_trust)" # Bootstrap in /tmp/toolchains/$tn/, install to /build/toolchains/$tn/. # local wd="/tmp/toolchains/$tn" local id="/build/toolchains/$tn" mkdir -p "$wd" mkdir -p "$id" local r=1 cd "$wd" while true; do # The "breakout loop". # Extract the toolchain. # if ! tar -xf "$tr/build2-toolchain.tar.xz"; then info "unable to extract $tr/build2-toolchain.tar.xz" break fi cd "build2-toolchain-$tv" # Bootstrap, stage, and install using the provided build.sh script. # if ! ./build.sh --install-dir "$id" --trust "$tt" g++; then info "failed to build $(pwd)" break fi r=0 break done cd "$owd" # Clean up. # rm -r "$wd" return "$r" } # Print monitor configuration as email body. # function print () { echo "buildid: $buildid" echo "buildid_url: $buildid_url" echo for tn in "${!toolchains[@]}"; do tp="${toolchains["$tn"]}" tu="$(tc_value "$tp" toolchain_url)" if [ -z "$tu" ]; then continue fi tt="$(tc_value "$tp" toolchain_trust)" echo "$tn.toolchain_url: $tu" echo "$tn.toolchain_trust: $tt" echo done } print | email "starting buildos monitor" if [ -z "$buildid_url" ]; then info "no buildos.buildid_url specified, not monitoring for new os builds" fi tc= for tn in "${!toolchains[@]}"; do tp="${toolchains["$tn"]}" tu="$(tc_value "$tp" toolchain_url)" if [ -z "$tu" ]; then continue fi tc="true" # The toolchain "sums" file (a list of SHA sums and relative file names, as # produced by shaNNNsum). The first entry should always be build2-toolchain # tar archive itself (which we use to figure out the version). Blank lines # and lines that start with '#' are ignored. # tf="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tu")" declare "${tp}toolchain_file=$tf" declare "${tp}toolchain_csum=$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tf")" declare "${tp}toolchain_root=/build/tftp/toolchains/$tn" declare "${tp}toolchain_ver=" # If buildos.toolchain_trust was not specified, set it to "no" so that # we don't prompt if the repository happens to be signed. # if [ -z "$(tc_value "$tp" toolchain_trust)" ]; then declare "${tp}toolchain_trust=no" fi done if [ -z "$tc" ]; then info "no buildos.toolchain_url specified, not bootstrapping" fi # Monitoring loop. # while true; do # Check for toolchain changes. If this is the first run, bootstrap them. # for tn in "${!toolchains[@]}"; do tp="${toolchains["$tn"]}" tu="$(tc_value "$tp" toolchain_url)" if [ -z "$tu" ]; then continue fi tr="$(tc_value "$tp" toolchain_root)" tf="$(tc_value "$tp" toolchain_file)" p="$tr/$tf" mkdir -p "$tr" # Fetch the toolchain sums either to $p if this is the first time or to # $p.new if we are checking for changes. # if [ -e "$p" ]; then f="$p.new" else f="$p" fi if curl -f -L -s -S -o "$f" "$tu"; then # Take care of change detection. # if [ "$f" != "$p" ]; then ts="$(tc_value "$tp" toolchain_file_csum)" cs="$(tc_checksum "$tp" "$f")" if [ "$ts" != "$cs" ]; then email "rebooting because of new $tn toolchain" <&1 | tee "$tr/bootstrap.log" 1>&2 if [ "${PIPESTATUS[0]}" -eq 0 ]; then v="$(cat $tr/version)" declare "${tp}toolchain_ver=$v" s="bootstrapped $tn toolchain $v" else s="failed to bootstrap $tn toolchain, waiting for new version" fi info "$s" email "$s" <