Add support for having multiple toolchains on single OS instance

3 files changed, 191 insertions, 94 deletions

diff --git a/buildos b/buildos
index 872a5a0..af39cd3 100755
--- a/buildos
+++ b/buildos
@@ -49,12 +49,28 @@ readarray -t cmdline < <(cat /proc/cmdline | \
 
 # Enter all buildos variables as bash variables.
 #
+
+# Map of toolchain names (as specified in buildos.<var>.<name>) to the
+# corresponding bash variable prefix.
+#
+declare -A toolchains
+toolchains["default"]=""
+
 for v in "${cmdline[@]}"; do
-  var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
+  var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
 
   if [ -n "$var" ]; then
-    val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")"            # Extract value.
-    val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+    val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")"            # Extract value.
+    val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+
+    # If the variable contains a dot, then it is a toolchain variable.
+    #
+    if [[ "$var" == *.* ]]; then
+      tn="$(sed -re 's/^[^.]+\.(.+)$/\1/' <<<"$var")"
+      var="${tn}_$(sed -re 's/^([^.]+)\..+$/\1/' <<<"$var")"
+      toolchains["$tn"]="${tn}_"
+    fi
+
     declare "$var=$val"
   fi
 done
@@ -77,85 +93,70 @@ function restart ()
   sudo systemctl reboot
 }
 
-email "starting buildos monitor" <<EOF
-buildid:       $buildid
-buildid_url:   $buildid_url
-toolchain_url: $toolchain_url
-EOF
-
-if [ -z "$buildid_url" ]; then
-  info "no buildos.buildid_url specified, not monitoring for new os builds"
-fi
-
-if [ -z "$toolchain_url" ]; then
-  info "no buildos.toolchain_url specified, not bootstrapping"
-else
-  # The toolchain "sums" file (a list of SHA sums and relative file names, as
-  # produced by shaNNNsum). The first entry should always be build2-toolchain
-  # tar archive itself (which we use to figure out the version). Blank lines
-  # and lines that start with '#' are ignored.
-  #
-  tc_url="$toolchain_url"
-  tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")"
-  tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")"
-  tc_root="/build/tftp"
-  tc_path="$tc_root/$tc_file"
-  tc_ver=
+# Toolchain-related funtions.
+#
 
-  # If buildos.toolchain_trust was not specified, set it to "no" so that
-  # we don't prompt if the repository happens to be signed.
-  #
-  if [ -n "$toolchain_trust" ]; then
-    tc_trust="$toolchain_trust"
-  else
-    tc_trust="no"
-  fi
-fi
+# Return the value of one of the toolchain_* variables for this toolchain.
+#
+function tc_value () # <toolchain-prefix> <variable>
+{
+  local n="${1}${2}"
+  echo "${!n}"
+}
 
 # Calculate the file checksum using the shaNNNsum utility.
 #
-function tc_checksum () # <file>
+function tc_checksum () # <toolchain-prefix> <file>
 {
-  "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p'
+  "$(tc_value "$1" toolchain_csum)sum" -b "$2" | \
+    sed -n -re 's/^([^ ]+) .+$/\1/p'
 }
 
-# Fetch a file from the sums file into $tc_root, verify its checksum, and make
-# a predictable name (without version) symlink.
+# Fetch a file from the sums file into $toolchain_root, verify its checksum,
+# and make a predictable name (without version) symlink.
 #
-function tc_fetch () # <line>
+function tc_fetch () # <toolchain-prefix> <line>
 {
-  local s p f u l
+  local s p f u l tp tu tr tv
+
+  tp="$1"
+  tu="$(tc_value "$tp" toolchain_url)"
+  tr="$(tc_value "$tp" toolchain_root)"
 
-  s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")"         # Checksum.
-  p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")"    # File path (rel).
-  f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")"      # File name.
-  u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL.
+  s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$2")"       # Checksum.
+  p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$2")"  # File path (relative).
+  f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")"    # File name.
+  u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tu")/$p"   # File URL.
 
   if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then
-    info "invalid sum line '$1'"
+    info "invalid sum line '$2'"
     return 1
   fi
 
   # Extract the version.
   #
-  if [ -z "$tc_ver" ]; then
-    tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+  tv="$(tc_value "$tp" toolchain_ver)"
 
-    if [ -z "$tc_ver" ]; then
+  if [ -z "$tv" ]; then
+    tv="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+
+    if [ -z "$tv" ]; then
       info "unable to extract toolchain version from '$f'"
       return 1
     fi
 
-    info "toolchain version is $tc_ver"
-    echo "$tc_ver" >"$tc_root/toolchain-version"
+    declare "${tp}toolchain_ver=$tv"
+
+    info "toolchain version $tv"
+    echo "$tv" >"$tr/version"
   fi
 
   # Derive a predictable name link.
   #
-  l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")"
+  l="$(sed -n -re "s/^(.+)-$tv(.*)$/\1\2/p" <<<"$f")"
 
   if [ -z "$l" ]; then
-    info "unable to derive predicatable name from '$f', '$tc_ver'"
+    info "unable to derive predicatable name from '$f' and '$tv'"
     return 1
   fi
 
@@ -163,7 +164,7 @@ function tc_fetch () # <line>
   #
   info "fetching $u [$l]"
 
-  if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then
+  if ! curl -f -L -s -S -o "$tr/$f" "$u"; then
     info "unable to fetch $u"
     return 1
   fi
@@ -172,103 +173,188 @@ function tc_fetch () # <line>
   #
   info "verifying checksum for $f"
 
-  local n
-  n="$(tc_checksum "$tc_root/$f")"
+  local cs
+  cs="$(tc_checksum "$tp" "$tr/$f")"
 
-  if [ "$n" != "$s" ]; then
-    info "$tc_sum checksum mismatch for $u"
+  if [ "$cs" != "$s" ]; then
+    info "checksum mismatch for $u"
     info "  expected:   $s"
-    info "  calculated: $n"
+    info "  calculated: $cs"
     return 1
   fi
 
   # Make the link.
   #
-  ln -s "$f" "$tc_root/$l"
+  ln -s "$f" "$tr/$l"
 }
 
 # Bootstrap the toolchain.
 #
-function tc_bootstrap ()
+function tc_bootstrap () # <toolchain-name>
 {
-  local l ls=()
+  local tp="${toolchains["$1"]}"
+  local tr="$(tc_value "$tp" toolchain_root)"
+  local tf="$(tc_value "$tp" toolchain_file)"
 
   # Fetch files according to the sums file. Skip empty line and those that
   # start with '#'.
   #
-  readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path")
+  local l ls=()
+
+  readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tr/$tf")
 
   for l in "${ls[@]}"; do
-    if ! tc_fetch "$l"; then
+    if ! tc_fetch "$tp" "$l"; then
       return 1 # Diagnostics has already been issued.
     fi
   done
 }
 
+# Print monitor configuration as email body.
+#
+function print ()
+{
+  echo "buildid:       $buildid"
+  echo "buildid_url:   $buildid_url"
+  echo
+
+  for tn in "${!toolchains[@]}"; do
+    tp="${toolchains["$tn"]}"
+    tu="$(tc_value "$tp" toolchain_url)"
+
+    if [ -z "$tu" ]; then
+      continue
+    fi
+
+    tt="$(tc_value "$tp" toolchain_trust)"
+
+    echo "$tn.toolchain_url:   $tu"
+    echo "$tn.toolchain_trust: $tt"
+    echo
+  done
+}
+
+print | email "starting buildos monitor"
+
+if [ -z "$buildid_url" ]; then
+  info "no buildos.buildid_url specified, not monitoring for new os builds"
+fi
+
+tc=
+for tn in "${!toolchains[@]}"; do
+  tp="${toolchains["$tn"]}"
+  tu="$(tc_value "$tp" toolchain_url)"
+
+  if [ -z "$tu" ]; then
+    continue
+  fi
+
+  tc="true"
+
+  # The toolchain "sums" file (a list of SHA sums and relative file names, as
+  # produced by shaNNNsum). The first entry should always be build2-toolchain
+  # tar archive itself (which we use to figure out the version). Blank lines
+  # and lines that start with '#' are ignored.
+  #
+  tf="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tu")"
+
+  declare "${tp}toolchain_file=$tf"
+  declare "${tp}toolchain_csum=$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tf")"
+  declare "${tp}toolchain_root=/build/tftp/toolchains/$tn"
+  declare "${tp}toolchain_ver="
+
+  # If buildos.toolchain_trust was not specified, set it to "no" so that
+  # we don't prompt if the repository happens to be signed.
+  #
+  if [ -z "$(tc_value "$tp" toolchain_trust)" ]; then
+    declare "${tp}toolchain_trust=no"
+  fi
+done
+
+if [ -z "$tc" ]; then
+  info "no buildos.toolchain_url specified, not bootstrapping"
+fi
+
 # Monitoring loop.
 #
 while true; do
 
-  # Check for toolchain changes. If this is the first run, bootstrap it.
+  # Check for toolchain changes. If this is the first run, bootstrap them.
   #
-  if [ -n "$tc_url" ]; then
+  for tn in "${!toolchains[@]}"; do
+    tp="${toolchains["$tn"]}"
+    tu="$(tc_value "$tp" toolchain_url)"
+
+    if [ -z "$tu" ]; then
+      continue
+    fi
+
+    tr="$(tc_value "$tp" toolchain_root)"
+    tf="$(tc_value "$tp" toolchain_file)"
+    p="$tr/$tf"
 
-    # Fetch the toolchain sums either to $tc_path if this is the first time
-    # or to $tc_path.new if we are checking for changes.
+    mkdir -p "$tr"
+
+    # Fetch the toolchain sums either to $p if this is the first time or to
+    # $p.new if we are checking for changes.
     #
-    if [ -e "$tc_path" ]; then
-      f="$tc_path.new"
+    if [ -e "$p" ]; then
+      f="$p.new"
     else
-      f="$tc_path"
+      f="$p"
     fi
 
-    if curl -f -L -s -S -o "$f" "$tc_url"; then
+    if curl -f -L -s -S -o "$f" "$tu"; then
 
       # Take care of change detection.
       #
-      if [ "$f" != "$tc_path" ]; then
+      if [ "$f" != "$p" ]; then
 
-	n="$(tc_checksum "$f")"
+	ts="$(tc_value "$tp" toolchain_file_csum)"
+	cs="$(tc_checksum "$tp" "$f")"
 
-	if [ "$tc_file_sum" != "$n" ]; then
-	  email "rebooting because of new toolchain" <<EOF
-old_toolchain: $tc_file_sum
-new_toolchain: $n
+	if [ "$ts" != "$cs" ]; then
+	  email "rebooting because of new $tn toolchain" <<EOF
+old_checksum: $ts
+new_checksum: $cs
 EOF
-	  info "new toolchain ($n), rebooting..."
+	  info "new $tn toolchain ($cs), rebooting..."
 	  restart
 	fi
       else
 	# This is the first run, bootstrap the toolchain.
 	#
-	tc_file_sum="$(tc_checksum "$f")"
+	declare "${tp}toolchain_file_csum=$(tc_checksum "$tp" "$f")"
 
 	# If we fail, we simply wait for a new toolchain to be uploaded (or
 	# some manual intervention).
 	#
 	# Note that because of the pipe tc_bootstrap() will run in subshell
-	# and any variables it sets (like tc_ver) won't be visible to us.
+	# and any variables it sets (like toolchain_ver) won't be visible to
+	# us.
 	#
-	info "bootstrapping toolchain..."
+	info "bootstrapping $tn toolchain..."
 
-	tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2
+	tc_bootstrap "$tn" 2>&1 | tee "$tr/bootstrap.log" 1>&2
 
 	if [ "${PIPESTATUS[0]}" -eq 0 ]; then
-	  tc_ver="$(cat $tc_root/toolchain-version)"
-	  s="bootstrapped toolchain $tc_ver"
+	  v="$(cat $tr/version)"
+	  declare "${tp}toolchain_ver=$v"
+
+	  s="bootstrapped $tn toolchain $v"
 	else
-	  s="failed to bootstrap toolchain, waiting for new version"
+	  s="failed to bootstrap $tn toolchain, waiting for new version"
 	fi
 
 	email "$s" <<EOF
-toolchain_log: tftp://$hname/toolchain.log
+$tn.bootstrap_log: tftp://$hname/toolchains/$tn/bootstrap.log
 EOF
       fi
     else
-      info "unable to fetch $tc_url, will try again"
+      info "unable to fetch $tu, will try again"
       rm -f "$f"
     fi
-  fi
+  done
 
   # Check for OS changes.
   #
diff --git a/doc/manual.cli b/doc/manual.cli
index b7aca34..b168675 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -303,6 +303,11 @@ bootstrap process (both on the build host and inside build machines) uses the
 default toolchain repository location embedded into the build scripts in the
 \c{build2-toolchain} package.
 
+It is also possible to use multiple toolchains on a single Build OS instance.
+In this case a toolchain name can be appended after \c{buildos.toolchain_*},
+for example, \c{buildos.toolchain_url.stage} (values without the toolchain
+name use the toolchain name \c{default}).
+
 Each line in the checksums file is the output of the \c{shaNNNsum(1)} utility,
 that is, the SHANNN sum following by space, an asterisk (\c{*}) which signals
 the binary mode), and the relative file path. Blank lines and lines that start
diff --git a/init b/init
index 64016a1..89191de 100755
--- a/init
+++ b/init
@@ -88,13 +88,19 @@ readarray -t cmdline < <(cat /proc/cmdline | \
 #
 info "command line:"
 for v in "${cmdline[@]}"; do
-  var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
+  var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
 
   if [ -n "$var" ]; then
-    val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")"            # Extract value.
-    val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+    val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")"            # Extract value.
+    val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
     info "  $var=$val"
-    declare "$var=$val"
+
+    # If the variable contains a dot, then it is a toolchain variable and we
+    # don't care about those in init.
+    #
+    if [[ "$var" != *.* ]]; then
+      declare "$var=$val"
+    fi
   fi
 done