diff options
Diffstat (limited to 'buildos')
-rwxr-xr-x | buildos | 191 |
1 files changed, 181 insertions, 10 deletions
@@ -9,10 +9,6 @@ # 4. Monitor for OS and toolchain changes and reboot if detected. # -# @@ What do we do in case or error, trap? Send email? Reboot? Probably -# reboot bad idea since someone may need to login to examine what's -# going on. -# # @@ What will systemd do if we fail? Perhaps configure it to restart # us? Or not since we may hose the logs. # @@ -81,20 +77,192 @@ function restart () sudo systemctl reboot } +email "starting buildos monitor" <<EOF +buildid: $buildid +buildid_url: $buildid_url +toolchain_url: $toolchain_url +EOF + if [ -z "$buildid_url" ]; then info "no buildos.buildid_url specified, not monitoring for new os builds" fi -email "starting buildos monitor" <<EOF -buildid: $buildid -buildid_url: $buildid_url -EOF +if [ -z "$toolchain_url" ]; then + info "no buildos.toolchain_url specified, not bootstrapping" +else + # The toolchain "sums" file (a list of SHA sums and relative file names, as + # produced by shaNNNsum). The first entry should always be build2-toolchain + # tar archive itself (which we use to figure out the version). Blank lines + # and lines that start with '#' are ignored. + # + tc_url="$toolchain_url" + tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")" + tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")" + tc_root="/build/tftp" + tc_path="$tc_root/$tc_file" + tc_ver= +fi + +# Calculate the file checksum using the shaNNNsum utility. +# +function tc_checksum () # <file> +{ + "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p' +} + +# Fetch a file from the sums file into $tc_root, verify its checksum, and make +# a predictable name (without version) symlink. +# +function tc_fetch () # <line> +{ + local s p f u l + + s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")" # Checksum. + p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")" # File path (rel). + f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. + u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL. + + if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then + info "invalid sum line '$1'" + return 1 + fi + + # Extract the version. + # + if [ -z "$tc_ver" ]; then + tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" + + if [ -z "$tc_ver" ]; then + info "unable to extract toolchain version from '$f'" + return 1 + fi + + info "toolchain version is $tc_ver" + echo "$tc_ver" >"$tc_root/toolchain-version" + fi + + # Derive a predictable name link. + # + l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")" + + if [ -z "$l" ]; then + info "unable to derive predicatable name from '$f', '$tc_ver'" + return 1 + fi + + # Fetch the file. + # + info "fetching $u [$l]" + + if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then + info "unable to fetch $u" + return 1 + fi + + # Verify the checksum. + # + info "verifying checksum for $f" + + local n + n="$(tc_checksum "$tc_root/$f")" + + if [ "$n" != "$s" ]; then + info "$tc_sum checksum mismatch for $u" + info " expected: $s" + info " calculated: $n" + return 1 + fi + # Make the link. + # + ln -s "$f" "$tc_root/$l" +} + +# Bootstrap the toolchain. +# +function tc_bootstrap () +{ + local l ls=() + + # Fetch files according to the sums file. Skip empty line and those that + # start with '#'. + # + readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path") + + for l in "${ls[@]}"; do + if ! tc_fetch "$l"; then + return 1 # Diagnostics has already been issued. + fi + done +} + +# Monitoring loop. +# while true; do - info "monitoring..." - sleep 5 + # Check for toolchain changes. If this is the first run, bootstrap it. + # + if [ -n "$tc_url" ]; then + + # Fetch the toolchain sums either to $tc_path if this is the first time + # or to $tc_path.new if we are checking for changes. + # + if [ -e "$tc_path" ]; then + f="$tc_path.new" + else + f="$tc_path" + fi + + if curl -f -L -s -S -o "$f" "$tc_url"; then + # Take care of change detection. + # + if [ "$f" != "$tc_path" ]; then + + n="$(tc_checksum "$f")" + + if [ "$tc_file_sum" != "$n" ]; then + email "rebooting because of new toolchain" <<EOF +old_toolchain: $tc_file_sum +new_toolchain: $n +EOF + info "new toolchain ($n), rebooting..." + restart + fi + else + # This is the first run, bootstrap the toolchain. + # + tc_file_sum="$(tc_checksum "$f")" + + # If we fail, we simply wait for a new toolchain to be uploaded (or + # some manual intervention). + # + # Note that because of the pipe tc_bootstrap() will run in subshell + # and any variables it sets (like tc_ver) won't be visible to us. + # + info "bootstrapping toolchain..." + + tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2 + + if [ "${PIPESTATUS[0]}" -eq 0 ]; then + tc_ver="$(cat $tc_root/toolchain-version)" + s="bootstrapped toolchain $tc_ver" + else + s="failed to bootstrap toolchain, waiting for new version" + fi + + email "$s" <<EOF +toolchain_log: tftp://$hname/toolchain.log +EOF + fi + else + info "unable to fetch $tc_url, will try again" + rm -f "$f" + fi + fi + + # Check for OS changes. + # if [ -n "$buildid_url" ]; then # Fetch the current id. While normally it will be a TFTP URL, it could also # be HTTP(S) so we configure sensible behavior for that. @@ -112,4 +280,7 @@ EOF info "unable to fetch $buildid_url, will try again" fi fi + + info "monitoring..." + sleep 10 done |