aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2017-03-30 11:27:54 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2017-03-30 11:27:54 +0200
commita222b03d3942d380e24042cc8c14f6215cd45749 (patch)
tree5601a13c55e3f863007726a9f616af160afc5e2a
parent73dfb006998bfca6410411c698ac52595264bb9c (diff)
Add support for having multiple toolchains on single OS instance
-rwxr-xr-xbuildos266
-rw-r--r--doc/manual.cli5
-rwxr-xr-xinit14
3 files changed, 191 insertions, 94 deletions
diff --git a/buildos b/buildos
index 872a5a0..af39cd3 100755
--- a/buildos
+++ b/buildos
@@ -49,12 +49,28 @@ readarray -t cmdline < <(cat /proc/cmdline | \
# Enter all buildos variables as bash variables.
#
+
+# Map of toolchain names (as specified in buildos.<var>.<name>) to the
+# corresponding bash variable prefix.
+#
+declare -A toolchains
+toolchains["default"]=""
+
for v in "${cmdline[@]}"; do
- var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
+ var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
if [ -n "$var" ]; then
- val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value.
- val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+ val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value.
+ val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+
+ # If the variable contains a dot, then it is a toolchain variable.
+ #
+ if [[ "$var" == *.* ]]; then
+ tn="$(sed -re 's/^[^.]+\.(.+)$/\1/' <<<"$var")"
+ var="${tn}_$(sed -re 's/^([^.]+)\..+$/\1/' <<<"$var")"
+ toolchains["$tn"]="${tn}_"
+ fi
+
declare "$var=$val"
fi
done
@@ -77,85 +93,70 @@ function restart ()
sudo systemctl reboot
}
-email "starting buildos monitor" <<EOF
-buildid: $buildid
-buildid_url: $buildid_url
-toolchain_url: $toolchain_url
-EOF
-
-if [ -z "$buildid_url" ]; then
- info "no buildos.buildid_url specified, not monitoring for new os builds"
-fi
-
-if [ -z "$toolchain_url" ]; then
- info "no buildos.toolchain_url specified, not bootstrapping"
-else
- # The toolchain "sums" file (a list of SHA sums and relative file names, as
- # produced by shaNNNsum). The first entry should always be build2-toolchain
- # tar archive itself (which we use to figure out the version). Blank lines
- # and lines that start with '#' are ignored.
- #
- tc_url="$toolchain_url"
- tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")"
- tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")"
- tc_root="/build/tftp"
- tc_path="$tc_root/$tc_file"
- tc_ver=
+# Toolchain-related funtions.
+#
- # If buildos.toolchain_trust was not specified, set it to "no" so that
- # we don't prompt if the repository happens to be signed.
- #
- if [ -n "$toolchain_trust" ]; then
- tc_trust="$toolchain_trust"
- else
- tc_trust="no"
- fi
-fi
+# Return the value of one of the toolchain_* variables for this toolchain.
+#
+function tc_value () # <toolchain-prefix> <variable>
+{
+ local n="${1}${2}"
+ echo "${!n}"
+}
# Calculate the file checksum using the shaNNNsum utility.
#
-function tc_checksum () # <file>
+function tc_checksum () # <toolchain-prefix> <file>
{
- "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p'
+ "$(tc_value "$1" toolchain_csum)sum" -b "$2" | \
+ sed -n -re 's/^([^ ]+) .+$/\1/p'
}
-# Fetch a file from the sums file into $tc_root, verify its checksum, and make
-# a predictable name (without version) symlink.
+# Fetch a file from the sums file into $toolchain_root, verify its checksum,
+# and make a predictable name (without version) symlink.
#
-function tc_fetch () # <line>
+function tc_fetch () # <toolchain-prefix> <line>
{
- local s p f u l
+ local s p f u l tp tu tr tv
+
+ tp="$1"
+ tu="$(tc_value "$tp" toolchain_url)"
+ tr="$(tc_value "$tp" toolchain_root)"
- s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")" # Checksum.
- p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")" # File path (rel).
- f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name.
- u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL.
+ s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$2")" # Checksum.
+ p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$2")" # File path (relative).
+ f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name.
+ u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tu")/$p" # File URL.
if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then
- info "invalid sum line '$1'"
+ info "invalid sum line '$2'"
return 1
fi
# Extract the version.
#
- if [ -z "$tc_ver" ]; then
- tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+ tv="$(tc_value "$tp" toolchain_ver)"
- if [ -z "$tc_ver" ]; then
+ if [ -z "$tv" ]; then
+ tv="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+
+ if [ -z "$tv" ]; then
info "unable to extract toolchain version from '$f'"
return 1
fi
- info "toolchain version is $tc_ver"
- echo "$tc_ver" >"$tc_root/toolchain-version"
+ declare "${tp}toolchain_ver=$tv"
+
+ info "toolchain version $tv"
+ echo "$tv" >"$tr/version"
fi
# Derive a predictable name link.
#
- l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")"
+ l="$(sed -n -re "s/^(.+)-$tv(.*)$/\1\2/p" <<<"$f")"
if [ -z "$l" ]; then
- info "unable to derive predicatable name from '$f', '$tc_ver'"
+ info "unable to derive predicatable name from '$f' and '$tv'"
return 1
fi
@@ -163,7 +164,7 @@ function tc_fetch () # <line>
#
info "fetching $u [$l]"
- if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then
+ if ! curl -f -L -s -S -o "$tr/$f" "$u"; then
info "unable to fetch $u"
return 1
fi
@@ -172,103 +173,188 @@ function tc_fetch () # <line>
#
info "verifying checksum for $f"
- local n
- n="$(tc_checksum "$tc_root/$f")"
+ local cs
+ cs="$(tc_checksum "$tp" "$tr/$f")"
- if [ "$n" != "$s" ]; then
- info "$tc_sum checksum mismatch for $u"
+ if [ "$cs" != "$s" ]; then
+ info "checksum mismatch for $u"
info " expected: $s"
- info " calculated: $n"
+ info " calculated: $cs"
return 1
fi
# Make the link.
#
- ln -s "$f" "$tc_root/$l"
+ ln -s "$f" "$tr/$l"
}
# Bootstrap the toolchain.
#
-function tc_bootstrap ()
+function tc_bootstrap () # <toolchain-name>
{
- local l ls=()
+ local tp="${toolchains["$1"]}"
+ local tr="$(tc_value "$tp" toolchain_root)"
+ local tf="$(tc_value "$tp" toolchain_file)"
# Fetch files according to the sums file. Skip empty line and those that
# start with '#'.
#
- readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path")
+ local l ls=()
+
+ readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tr/$tf")
for l in "${ls[@]}"; do
- if ! tc_fetch "$l"; then
+ if ! tc_fetch "$tp" "$l"; then
return 1 # Diagnostics has already been issued.
fi
done
}
+# Print monitor configuration as email body.
+#
+function print ()
+{
+ echo "buildid: $buildid"
+ echo "buildid_url: $buildid_url"
+ echo
+
+ for tn in "${!toolchains[@]}"; do
+ tp="${toolchains["$tn"]}"
+ tu="$(tc_value "$tp" toolchain_url)"
+
+ if [ -z "$tu" ]; then
+ continue
+ fi
+
+ tt="$(tc_value "$tp" toolchain_trust)"
+
+ echo "$tn.toolchain_url: $tu"
+ echo "$tn.toolchain_trust: $tt"
+ echo
+ done
+}
+
+print | email "starting buildos monitor"
+
+if [ -z "$buildid_url" ]; then
+ info "no buildos.buildid_url specified, not monitoring for new os builds"
+fi
+
+tc=
+for tn in "${!toolchains[@]}"; do
+ tp="${toolchains["$tn"]}"
+ tu="$(tc_value "$tp" toolchain_url)"
+
+ if [ -z "$tu" ]; then
+ continue
+ fi
+
+ tc="true"
+
+ # The toolchain "sums" file (a list of SHA sums and relative file names, as
+ # produced by shaNNNsum). The first entry should always be build2-toolchain
+ # tar archive itself (which we use to figure out the version). Blank lines
+ # and lines that start with '#' are ignored.
+ #
+ tf="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tu")"
+
+ declare "${tp}toolchain_file=$tf"
+ declare "${tp}toolchain_csum=$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tf")"
+ declare "${tp}toolchain_root=/build/tftp/toolchains/$tn"
+ declare "${tp}toolchain_ver="
+
+ # If buildos.toolchain_trust was not specified, set it to "no" so that
+ # we don't prompt if the repository happens to be signed.
+ #
+ if [ -z "$(tc_value "$tp" toolchain_trust)" ]; then
+ declare "${tp}toolchain_trust=no"
+ fi
+done
+
+if [ -z "$tc" ]; then
+ info "no buildos.toolchain_url specified, not bootstrapping"
+fi
+
# Monitoring loop.
#
while true; do
- # Check for toolchain changes. If this is the first run, bootstrap it.
+ # Check for toolchain changes. If this is the first run, bootstrap them.
#
- if [ -n "$tc_url" ]; then
+ for tn in "${!toolchains[@]}"; do
+ tp="${toolchains["$tn"]}"
+ tu="$(tc_value "$tp" toolchain_url)"
+
+ if [ -z "$tu" ]; then
+ continue
+ fi
+
+ tr="$(tc_value "$tp" toolchain_root)"
+ tf="$(tc_value "$tp" toolchain_file)"
+ p="$tr/$tf"
- # Fetch the toolchain sums either to $tc_path if this is the first time
- # or to $tc_path.new if we are checking for changes.
+ mkdir -p "$tr"
+
+ # Fetch the toolchain sums either to $p if this is the first time or to
+ # $p.new if we are checking for changes.
#
- if [ -e "$tc_path" ]; then
- f="$tc_path.new"
+ if [ -e "$p" ]; then
+ f="$p.new"
else
- f="$tc_path"
+ f="$p"
fi
- if curl -f -L -s -S -o "$f" "$tc_url"; then
+ if curl -f -L -s -S -o "$f" "$tu"; then
# Take care of change detection.
#
- if [ "$f" != "$tc_path" ]; then
+ if [ "$f" != "$p" ]; then
- n="$(tc_checksum "$f")"
+ ts="$(tc_value "$tp" toolchain_file_csum)"
+ cs="$(tc_checksum "$tp" "$f")"
- if [ "$tc_file_sum" != "$n" ]; then
- email "rebooting because of new toolchain" <<EOF
-old_toolchain: $tc_file_sum
-new_toolchain: $n
+ if [ "$ts" != "$cs" ]; then
+ email "rebooting because of new $tn toolchain" <<EOF
+old_checksum: $ts
+new_checksum: $cs
EOF
- info "new toolchain ($n), rebooting..."
+ info "new $tn toolchain ($cs), rebooting..."
restart
fi
else
# This is the first run, bootstrap the toolchain.
#
- tc_file_sum="$(tc_checksum "$f")"
+ declare "${tp}toolchain_file_csum=$(tc_checksum "$tp" "$f")"
# If we fail, we simply wait for a new toolchain to be uploaded (or
# some manual intervention).
#
# Note that because of the pipe tc_bootstrap() will run in subshell
- # and any variables it sets (like tc_ver) won't be visible to us.
+ # and any variables it sets (like toolchain_ver) won't be visible to
+ # us.
#
- info "bootstrapping toolchain..."
+ info "bootstrapping $tn toolchain..."
- tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2
+ tc_bootstrap "$tn" 2>&1 | tee "$tr/bootstrap.log" 1>&2
if [ "${PIPESTATUS[0]}" -eq 0 ]; then
- tc_ver="$(cat $tc_root/toolchain-version)"
- s="bootstrapped toolchain $tc_ver"
+ v="$(cat $tr/version)"
+ declare "${tp}toolchain_ver=$v"
+
+ s="bootstrapped $tn toolchain $v"
else
- s="failed to bootstrap toolchain, waiting for new version"
+ s="failed to bootstrap $tn toolchain, waiting for new version"
fi
email "$s" <<EOF
-toolchain_log: tftp://$hname/toolchain.log
+$tn.bootstrap_log: tftp://$hname/toolchains/$tn/bootstrap.log
EOF
fi
else
- info "unable to fetch $tc_url, will try again"
+ info "unable to fetch $tu, will try again"
rm -f "$f"
fi
- fi
+ done
# Check for OS changes.
#
diff --git a/doc/manual.cli b/doc/manual.cli
index b7aca34..b168675 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -303,6 +303,11 @@ bootstrap process (both on the build host and inside build machines) uses the
default toolchain repository location embedded into the build scripts in the
\c{build2-toolchain} package.
+It is also possible to use multiple toolchains on a single Build OS instance.
+In this case a toolchain name can be appended after \c{buildos.toolchain_*},
+for example, \c{buildos.toolchain_url.stage} (values without the toolchain
+name use the toolchain name \c{default}).
+
Each line in the checksums file is the output of the \c{shaNNNsum(1)} utility,
that is, the SHANNN sum following by space, an asterisk (\c{*}) which signals
the binary mode), and the relative file path. Blank lines and lines that start
diff --git a/init b/init
index 64016a1..89191de 100755
--- a/init
+++ b/init
@@ -88,13 +88,19 @@ readarray -t cmdline < <(cat /proc/cmdline | \
#
info "command line:"
for v in "${cmdline[@]}"; do
- var="$(sed -r -n -e 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
+ var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name.
if [ -n "$var" ]; then
- val="$(sed -r -e 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value.
- val="$(sed -r -e "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
+ val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value.
+ val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted.
info " $var=$val"
- declare "$var=$val"
+
+ # If the variable contains a dot, then it is a toolchain variable and we
+ # don't care about those in init.
+ #
+ if [[ "$var" != *.* ]]; then
+ declare "$var=$val"
+ fi
fi
done