diff options
| -rwxr-xr-x | buildos | 266 | ||||
| -rw-r--r-- | doc/manual.cli | 5 | ||||
| -rwxr-xr-x | init | 14 |
3 files changed, 191 insertions, 94 deletions
@@ -49,12 +49,28 @@ readarray -t cmdline < <(cat /proc/cmdline | \ # Enter all buildos variables as bash variables. # + +# Map of toolchain names (as specified in buildos.<var>.<name>) to the +# corresponding bash variable prefix. +# +declare -A toolchains +toolchains["default"]="" + for v in "${cmdline[@]}"; do - var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. + var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. if [ -n "$var" ]; then - val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. - val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. + val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. + val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. + + # If the variable contains a dot, then it is a toolchain variable. + # + if [[ "$var" == *.* ]]; then + tn="$(sed -re 's/^[^.]+\.(.+)$/\1/' <<<"$var")" + var="${tn}_$(sed -re 's/^([^.]+)\..+$/\1/' <<<"$var")" + toolchains["$tn"]="${tn}_" + fi + declare "$var=$val" fi done @@ -77,85 +93,70 @@ function restart () sudo systemctl reboot } -email "starting buildos monitor" <<EOF -buildid: $buildid -buildid_url: $buildid_url -toolchain_url: $toolchain_url -EOF - -if [ -z "$buildid_url" ]; then - info "no buildos.buildid_url specified, not monitoring for new os builds" -fi - -if [ -z "$toolchain_url" ]; then - info "no buildos.toolchain_url specified, not bootstrapping" -else - # The toolchain "sums" file (a list of SHA sums and relative file names, as - # produced by shaNNNsum). The first entry should always be build2-toolchain - # tar archive itself (which we use to figure out the version). Blank lines - # and lines that start with '#' are ignored. - # - tc_url="$toolchain_url" - tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")" - tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")" - tc_root="/build/tftp" - tc_path="$tc_root/$tc_file" - tc_ver= +# Toolchain-related funtions. +# - # If buildos.toolchain_trust was not specified, set it to "no" so that - # we don't prompt if the repository happens to be signed. - # - if [ -n "$toolchain_trust" ]; then - tc_trust="$toolchain_trust" - else - tc_trust="no" - fi -fi +# Return the value of one of the toolchain_* variables for this toolchain. +# +function tc_value () # <toolchain-prefix> <variable> +{ + local n="${1}${2}" + echo "${!n}" +} # Calculate the file checksum using the shaNNNsum utility. # -function tc_checksum () # <file> +function tc_checksum () # <toolchain-prefix> <file> { - "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p' + "$(tc_value "$1" toolchain_csum)sum" -b "$2" | \ + sed -n -re 's/^([^ ]+) .+$/\1/p' } -# Fetch a file from the sums file into $tc_root, verify its checksum, and make -# a predictable name (without version) symlink. +# Fetch a file from the sums file into $toolchain_root, verify its checksum, +# and make a predictable name (without version) symlink. # -function tc_fetch () # <line> +function tc_fetch () # <toolchain-prefix> <line> { - local s p f u l + local s p f u l tp tu tr tv + + tp="$1" + tu="$(tc_value "$tp" toolchain_url)" + tr="$(tc_value "$tp" toolchain_root)" - s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")" # Checksum. - p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")" # File path (rel). - f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. - u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL. + s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$2")" # Checksum. + p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$2")" # File path (relative). + f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. + u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tu")/$p" # File URL. if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then - info "invalid sum line '$1'" + info "invalid sum line '$2'" return 1 fi # Extract the version. # - if [ -z "$tc_ver" ]; then - tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" + tv="$(tc_value "$tp" toolchain_ver)" - if [ -z "$tc_ver" ]; then + if [ -z "$tv" ]; then + tv="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" + + if [ -z "$tv" ]; then info "unable to extract toolchain version from '$f'" return 1 fi - info "toolchain version is $tc_ver" - echo "$tc_ver" >"$tc_root/toolchain-version" + declare "${tp}toolchain_ver=$tv" + + info "toolchain version $tv" + echo "$tv" >"$tr/version" fi # Derive a predictable name link. # - l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")" + l="$(sed -n -re "s/^(.+)-$tv(.*)$/\1\2/p" <<<"$f")" if [ -z "$l" ]; then - info "unable to derive predicatable name from '$f', '$tc_ver'" + info "unable to derive predicatable name from '$f' and '$tv'" return 1 fi @@ -163,7 +164,7 @@ function tc_fetch () # <line> # info "fetching $u [$l]" - if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then + if ! curl -f -L -s -S -o "$tr/$f" "$u"; then info "unable to fetch $u" return 1 fi @@ -172,103 +173,188 @@ function tc_fetch () # <line> # info "verifying checksum for $f" - local n - n="$(tc_checksum "$tc_root/$f")" + local cs + cs="$(tc_checksum "$tp" "$tr/$f")" - if [ "$n" != "$s" ]; then - info "$tc_sum checksum mismatch for $u" + if [ "$cs" != "$s" ]; then + info "checksum mismatch for $u" info " expected: $s" - info " calculated: $n" + info " calculated: $cs" return 1 fi # Make the link. # - ln -s "$f" "$tc_root/$l" + ln -s "$f" "$tr/$l" } # Bootstrap the toolchain. # -function tc_bootstrap () +function tc_bootstrap () # <toolchain-name> { - local l ls=() + local tp="${toolchains["$1"]}" + local tr="$(tc_value "$tp" toolchain_root)" + local tf="$(tc_value "$tp" toolchain_file)" # Fetch files according to the sums file. Skip empty line and those that # start with '#'. # - readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path") + local l ls=() + + readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tr/$tf") for l in "${ls[@]}"; do - if ! tc_fetch "$l"; then + if ! tc_fetch "$tp" "$l"; then return 1 # Diagnostics has already been issued. fi done } +# Print monitor configuration as email body. +# +function print () +{ + echo "buildid: $buildid" + echo "buildid_url: $buildid_url" + echo + + for tn in "${!toolchains[@]}"; do + tp="${toolchains["$tn"]}" + tu="$(tc_value "$tp" toolchain_url)" + + if [ -z "$tu" ]; then + continue + fi + + tt="$(tc_value "$tp" toolchain_trust)" + + echo "$tn.toolchain_url: $tu" + echo "$tn.toolchain_trust: $tt" + echo + done +} + +print | email "starting buildos monitor" + +if [ -z "$buildid_url" ]; then + info "no buildos.buildid_url specified, not monitoring for new os builds" +fi + +tc= +for tn in "${!toolchains[@]}"; do + tp="${toolchains["$tn"]}" + tu="$(tc_value "$tp" toolchain_url)" + + if [ -z "$tu" ]; then + continue + fi + + tc="true" + + # The toolchain "sums" file (a list of SHA sums and relative file names, as + # produced by shaNNNsum). The first entry should always be build2-toolchain + # tar archive itself (which we use to figure out the version). Blank lines + # and lines that start with '#' are ignored. + # + tf="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tu")" + + declare "${tp}toolchain_file=$tf" + declare "${tp}toolchain_csum=$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tf")" + declare "${tp}toolchain_root=/build/tftp/toolchains/$tn" + declare "${tp}toolchain_ver=" + + # If buildos.toolchain_trust was not specified, set it to "no" so that + # we don't prompt if the repository happens to be signed. + # + if [ -z "$(tc_value "$tp" toolchain_trust)" ]; then + declare "${tp}toolchain_trust=no" + fi +done + +if [ -z "$tc" ]; then + info "no buildos.toolchain_url specified, not bootstrapping" +fi + # Monitoring loop. # while true; do - # Check for toolchain changes. If this is the first run, bootstrap it. + # Check for toolchain changes. If this is the first run, bootstrap them. # - if [ -n "$tc_url" ]; then + for tn in "${!toolchains[@]}"; do + tp="${toolchains["$tn"]}" + tu="$(tc_value "$tp" toolchain_url)" + + if [ -z "$tu" ]; then + continue + fi + + tr="$(tc_value "$tp" toolchain_root)" + tf="$(tc_value "$tp" toolchain_file)" + p="$tr/$tf" - # Fetch the toolchain sums either to $tc_path if this is the first time - # or to $tc_path.new if we are checking for changes. + mkdir -p "$tr" + + # Fetch the toolchain sums either to $p if this is the first time or to + # $p.new if we are checking for changes. # - if [ -e "$tc_path" ]; then - f="$tc_path.new" + if [ -e "$p" ]; then + f="$p.new" else - f="$tc_path" + f="$p" fi - if curl -f -L -s -S -o "$f" "$tc_url"; then + if curl -f -L -s -S -o "$f" "$tu"; then # Take care of change detection. # - if [ "$f" != "$tc_path" ]; then + if [ "$f" != "$p" ]; then - n="$(tc_checksum "$f")" + ts="$(tc_value "$tp" toolchain_file_csum)" + cs="$(tc_checksum "$tp" "$f")" - if [ "$tc_file_sum" != "$n" ]; then - email "rebooting because of new toolchain" <<EOF -old_toolchain: $tc_file_sum -new_toolchain: $n + if [ "$ts" != "$cs" ]; then + email "rebooting because of new $tn toolchain" <<EOF +old_checksum: $ts +new_checksum: $cs EOF - info "new toolchain ($n), rebooting..." + info "new $tn toolchain ($cs), rebooting..." restart fi else # This is the first run, bootstrap the toolchain. # - tc_file_sum="$(tc_checksum "$f")" + declare "${tp}toolchain_file_csum=$(tc_checksum "$tp" "$f")" # If we fail, we simply wait for a new toolchain to be uploaded (or # some manual intervention). # # Note that because of the pipe tc_bootstrap() will run in subshell - # and any variables it sets (like tc_ver) won't be visible to us. + # and any variables it sets (like toolchain_ver) won't be visible to + # us. # - info "bootstrapping toolchain..." + info "bootstrapping $tn toolchain..." - tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2 + tc_bootstrap "$tn" 2>&1 | tee "$tr/bootstrap.log" 1>&2 if [ "${PIPESTATUS[0]}" -eq 0 ]; then - tc_ver="$(cat $tc_root/toolchain-version)" - s="bootstrapped toolchain $tc_ver" + v="$(cat $tr/version)" + declare "${tp}toolchain_ver=$v" + + s="bootstrapped $tn toolchain $v" else - s="failed to bootstrap toolchain, waiting for new version" + s="failed to bootstrap $tn toolchain, waiting for new version" fi email "$s" <<EOF -toolchain_log: tftp://$hname/toolchain.log +$tn.bootstrap_log: tftp://$hname/toolchains/$tn/bootstrap.log EOF fi else - info "unable to fetch $tc_url, will try again" + info "unable to fetch $tu, will try again" rm -f "$f" fi - fi + done # Check for OS changes. # diff --git a/doc/manual.cli b/doc/manual.cli index b7aca34..b168675 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -303,6 +303,11 @@ bootstrap process (both on the build host and inside build machines) uses the default toolchain repository location embedded into the build scripts in the \c{build2-toolchain} package. +It is also possible to use multiple toolchains on a single Build OS instance. +In this case a toolchain name can be appended after \c{buildos.toolchain_*}, +for example, \c{buildos.toolchain_url.stage} (values without the toolchain +name use the toolchain name \c{default}). + Each line in the checksums file is the output of the \c{shaNNNsum(1)} utility, that is, the SHANNN sum following by space, an asterisk (\c{*}) which signals the binary mode), and the relative file path. Blank lines and lines that start @@ -88,13 +88,19 @@ readarray -t cmdline < <(cat /proc/cmdline | \ # info "command line:" for v in "${cmdline[@]}"; do - var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. + var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. if [ -n "$var" ]; then - val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. - val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. + val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. + val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. info " $var=$val" - declare "$var=$val" + + # If the variable contains a dot, then it is a toolchain variable and we + # don't care about those in init. + # + if [[ "$var" != *.* ]]; then + declare "$var=$val" + fi fi done |