#!/bin/bash # Build OS monitor. It starts as a systemd service and performs the following # steps: # # 1. Bootstrap the build2 toolchain. # 2. Build and start bbot. # 3. Build and start bslave. # 4. Monitor for OS and toolchain changes and reboot if detected. # # @@ What will systemd do if we fail? Perhaps configure it to restart # us? Or not since we may hose the logs. # owd="$(pwd)" trap "{ cd '$owd'; exit 1; }" ERR set -o errtrace # Trap in functions. shopt -s nullglob # Expand patterns than don't match to empty. # Note: diagnostics goes to stdout. # function info () { echo "$*" 1>&2; } function error () { if [ "$#" -gt 0 ]; then info "$*"; fi exit 1 } # Network timeouts: 60 seconds to connect, 10 minutes to complete, 4 retries # (5 attempts total). These are similar to bbot timeouts. Note that the # toolchain archives can be quite sizable. # timeout=600 curl=(curl -f -L -s -S \ --retry 4 \ --retry-max-time "$timeout" \ --max-time "$timeout" \ --connect-timeout 60) info "starting build os monitor..." # Parse the kernel command line. This is complicated by the fact that the # values can be quoted, for example: # # foo='foo fox' # bar="bar 'box'" # # First we separete quoted variables and arguments with newlines (giving # priority to assignments). Then we replace whitespaces with newline on # lines that don't contain quites. Finally, clean up by removing blank # lines. # # Note: the same code as in init. # readarray -t cmdline < <(cat /proc/cmdline | \ sed -r -e "s/([^ ]+=)?('[^']*'|\"[^\"]*\")/\n\1\2\n/g" | \ sed -r -e "/['\"]/!s/ /\n/g" | sed -r -e '/^\s*$/d') # Enter all buildos variables as bash variables. # # Map of toolchain names (as specified in buildos..) to the # corresponding bash variable prefix. # declare -A toolchains toolchains["default"]="" for v in "${cmdline[@]}"; do var="$(sed -n -re 's/^buildos\.([^=]+)=.*$/\1/p' <<<"$v")" # Extract name. if [ -n "$var" ]; then val="$(sed -re 's/^[^=]+=(.*)$/\1/' <<<"$v")" # Extract value. val="$(sed -re "s/^('(.*)'|\"(.*)\")$/\2\3/" <<<"$val")" # Strip quoted. # Recognize some variables as arrays. # a= # If the variable contains a dot, then it is a toolchain name-specific # variable. # if [[ "$var" == *.* ]]; then tn="$(sed -re 's/^[^.]+\.(.+)$/\1/' <<<"$var")" var="$(sed -re 's/^([^.]+)\..+$/\1/' <<<"$var")" if [ "$var" = "controller_url" -o "$var" = "controller_trust" ]; then a=true fi var="${tn}_$var" toolchains["$tn"]="${tn}_" fi if [ -n "$a" ]; then declare -a "$var+=('$val')" else declare "$var=$val" fi fi done hname="$(hostname)" # Get the build id. # buildid="$(sed -n -re 's/^BUILD_ID="(.+)"$/\1/p' /etc/os-release)" function email () # < { (echo -e "Subject: [$hname] $1\n"; cat -) | sendmail -i "$admin_email" } function restart () { sendmail -q # Flush mail queue. sleep 10 # Give any remaining mail chance to go through. sudo systemctl reboot } if [ -z "$buildid_url" ]; then info "no buildos.buildid_url specified, not monitoring for new os builds" fi # Process toolchains. # # Return the value of one of the toolchain_* variables for this toolchain. # function toolchain_value () # { local n="${1}${2}" echo "${!n}" } toolchain_names=() for tn in "${!toolchains[@]}"; do tp="${toolchains["$tn"]}" tu="$(toolchain_value "$tp" toolchain_url)" if [ -z "$tu" ]; then continue fi toolchain_names+=("$tn") # The toolchain "sums" file (a list of SHA sums and relative file names, as # produced by shaNNNsum). The first entry should always be build2-toolchain # tar archive itself (which we use to figure out the version). Blank lines # and lines that start with '#' are ignored. # tf="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tu")" declare "${tp}toolchain_file=$tf" declare "${tp}toolchain_csum=$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tf")" declare "${tp}toolchain_root=/build/tftp/toolchains/$tn" declare "${tp}toolchain_ver=" declare "${tp}toolchain_fver=" # Full version (with snapshot). # If buildos.toolchain_trust was not specified, set it to "no" so that # we don't prompt if the repository happens to be signed. # if [ -z "$(toolchain_value "$tp" toolchain_trust)" ]; then declare "${tp}toolchain_trust=no" fi # Warn if we have no controller URLs for this toolchain. # n="${tp}controller_url[0]" if [ -z "${!n}" ]; then info "no buildos.controller_url.$tn specified, not starting bbot agent" fi done if [ "${#toolchain_names[@]}" -eq 0 ]; then info "no buildos.toolchain_url specified, not bootstrapping" fi # Divide CPUs and RAM (in kB) among the toolchains. # # Reserve 4G of RAM for ourselves (rootfs, tmpfs). # ram_total="$(sed -n -re 's/^MemTotal: *([0-9]+) *kB$/\1/p' /dev/null } print | email "starting build os monitor" # Machines cleanup (/build/machines/). # diag=() fail= function print_diag () { local p for p in "${diag[@]}"; do echo " $p" done } # Iterate over all the machines and call a function (one of the below # machines_clean_*()) for each. # function machines_for () # ... { local f="$1" shift diag=() fail= local v m for v in /build/machines/*; do if [ ! -d "$v" ]; then diag+=("$v: error: invalid volume") fail="true" continue fi cd "$v" for m in *; do if [ ! -d "$m" ]; then diag+=("$v/$m: error: invalid machine") fail="true" continue fi "$f" "$v" "$m" "$@" done cd "$owd" done } function machines_clean_subvolume () # { if ! btrfs property set -ts "$1" ro false; then diag+=("$1: error: unable to change subvolume property") fail="true" return 1 fi if ! btrfs subvolume delete "$1"; then diag+=("$1: error: unable to delete subvolume") fail="true" return 1 fi } # Cleanup the -- entries for the specified toolchain # called before starting each toolchain. # function machines_clean_toolchain () # { local v="$1" local m="$2" local tn="$3" cd "$m" local s for s in "$m"-"$tn"-*; do if [ ! -d "$s" ]; then diag+=("$v/$m/$s: error: invalid machine subvolume") fail="true" continue fi if machines_clean_subvolume "$v/$m/$s"; then diag+=("$v/$m/$s: info: deleted stray toolchain working subvolume") fi done cd "$v" } # Cleanup stray snapshots or deleted machines. Called once during startup. # function machines_clean_stray () # { local v="$1" local m="$2" cd "$m" # Collect current machine symlink's bootstrap protocol numbers. If there # are no current machine symlinks, then we delete the whole thing. # local s ps=() for s in "$m"-*; do if [[ "$s" =~ ^"$m"-[0-9]+$ ]]; then if [ ! -L "$s" ]; then diag+=("$v/$m/$s: error: not a symlink") fail="true" fi # Treat it as if it were a symlink even if its not. Failed that we # may try to delete the whole thing. # ps+=("$(sed -n -re 's/^.+-([0-9]+)$/\1/p' <<<"$s")") fi done # Examine each machine subvolume. # for s in "$m"-*; do # - (current machine symlink) # if [[ "$s" =~ ^"$m"-[0-9]+$ ]]; then continue fi if [ ! -d "$s" ]; then diag+=("$v/$m/$s: error: invalid machine subvolume") fail="true" continue fi # Unless we are deleting the whole thing, keep initial and bootstrapped # (for known toolchains) subvolumes. # if [ "${#ps[@]}" -gt 0 ]; then # - . (initial image) # local p f= for p in "${ps[@]}"; do if [[ "$s" =~ ^"$m"-"$p"\.[0-9]+$ ]]; then f="true" break fi done if [ -n "$f" ]; then continue fi # - (bootstrapped image) # f= local tn for tn in "${toolchain_names[@]}"; do if [[ "$s" =~ ^"$m"-"$tn"$ ]]; then f="true" break fi done if [ -n "$f" ]; then continue fi fi # This is either a stray working submodule or a bootsrapped subvolume # for a toolchain that was deleted (or we are deleting everything). # if machines_clean_subvolume "$v/$m/$s"; then diag+=("$v/$m/$s: info: deleted subvolume") fi done cd "$v" # Delete the machine directory (which we expect to be now empty). # if [ "${#ps[@]}" -eq 0 ]; then if rmdir "$m"; then diag+=("$v/$m: info: deleted machine directory") else diag+=("$v/$m: error: unable to delete machine directory") fail="true" fi fi } # Do the initial cleanup. # machines_for machines_clean_stray if [ "${#diag[@]}" -gt 0 ]; then if [ -z "$fail" ]; then s="cleaned up entries in /build/machines/" else s="invalid entries in /build/machines/, halting" fi print_diag | email "$s" info "$s" && print_diag 1>&2 if [ -n "$fail" ]; then info "correct and restart the monitor (systemctl restart buildos)" exit 1 fi fi # Toolchain-related funtions. # # Calculate the file checksum using the shaNNNsum utility. # function toolchain_checksum () # { "$(toolchain_value "$1" toolchain_csum)sum" -b "$2" | \ sed -n -re 's/^([^ ]+) .+$/\1/p' } # Fetch a file from the sums file into $toolchain_root, verify its checksum, # and make a predictable name (without version) symlink. # function toolchain_fetch () # { local s p f u l tn tp tu tr tv tn="$1" tp="${toolchains["$tn"]}" tu="$(toolchain_value "$tp" toolchain_url)" tr="$(toolchain_value "$tp" toolchain_root)" s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$2")" # Checksum. p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$2")" # File path (relative). f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tu")/$p" # File URL. if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then info "invalid sum line '$2'" return 1 fi # Extract the version and derive a predictable name link. # tv="$(toolchain_value "$tp" toolchain_ver)" if [ -z "$tv" ]; then tv="$(sed -n -re 's/^build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" if [ -z "$tv" ]; then info "unable to extract toolchain version from '$f'" return 1 fi info "toolchain '$tn' version $tv" declare -g "${tp}toolchain_fver=$tv" # Full version. echo "$tv" >"$tr/version-full" l="$(sed -n -re "s/^(.+)-$tv(.*)$/\1\2/p" <<<"$f")" # Use full version. # Strip snapshot. # tv="$(sed -n -re 's/^([^.]+\.[^.]+\.[^-]+(-[ab]\.[^.+]+)?).*/\1/p' <<<"$tv")" declare -g "${tp}toolchain_ver=$tv" echo "$tv" >"$tr/version" else # For files other that build2-toolchain we expect the version component to # be in the [-] form, for example 1.2.3-stage. # l="$(sed -n -re "s/^(.+)-$tv(-$tn)?(.*)$/\1\3/p" <<<"$f")" fi if [ -z "$l" ]; then info "unable to derive predicatable name from '$f' and '$tv'" return 1 fi # Fetch the file. # info "fetching $u [$l]" if ! "${curl[@]}" -o "$tr/$f" "$u"; then info "unable to fetch $u" return 1 fi # Verify the checksum. # info "verifying checksum for $f" local cs cs="$(toolchain_checksum "$tp" "$tr/$f")" if [ "$cs" != "$s" ]; then info "checksum mismatch for $u" info " expected: $s" info " calculated: $cs" return 1 fi # Make the link. # # Note that the target must be just the file for TFTP chroot to work. # ln -s "$f" "$tr/$l" } # Bootstrap the toolchain. # # Return 0 on success, 1 if the toolchain is disabled, and 2 in case of # an error. # function toolchain_bootstrap () # { local tn="$1" local tp="${toolchains["$tn"]}" local tr="$(toolchain_value "$tp" toolchain_root)" local tf="$(toolchain_value "$tp" toolchain_file)" # Fetch files according to the sums file. Skip empty lines and those that # start with '#'. # local l ls=() readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tr/$tf") if [ "${#ls[@]}" -eq 0 ]; then info "empty $tr/$tf" return 2 fi # Check if this toolchain is disabled. # if [ "${ls[0]}" = "disabled" ]; then return 1 fi for l in "${ls[@]}"; do if ! toolchain_fetch "$tn" "$l"; then return 2 # Diagnostics has already been issued. fi done local tv="$(toolchain_value "$tp" toolchain_fver)" # Set by fetch(). local tt="$(toolchain_value "$tp" toolchain_trust)" # Save the repository certificate fingerprint into the trust file (used # by machine bootstrap). # echo "$tt" >"$tr/trust" # Bootstrap in /tmp/toolchains/$tn/, install to /build/toolchains/$tn/. # local wd="/tmp/toolchains/$tn" local id="/build/toolchains/$tn" mkdir -p "$wd" mkdir -p "$id" local r=2 cd "$wd" while true; do # The "breakout loop". # Extract the toolchain. # if ! tar -xf "$tr/build2-toolchain.tar.xz"; then info "unable to extract $tr/build2-toolchain.tar.xz" break fi cd "build2-toolchain-$tv"*/ # Bootstrap, stage, and install using the provided build.sh script. Do # parallel bootstrap using make. # if ! ./build.sh --make make \ --make "-j$cpu_total" \ --timeout "$timeout" \ --install-dir "$id" \ --trust "$tt" \ g++; then info "failed to build $(pwd)" break fi cd "$wd" rm -r "build2-toolchain-$tv"*/ mv -T build2-toolchain-* build2-toolchain # Strip version. r=0 break done cd "$owd" return "$r" } # Check if we need to build/start or rebuild/restart the bbot agent. Return # 0 if nothing to do, 1 for upgrades, 2 for first build, and 3 for failure. # function bbot_check () # { local tn="$1" export PATH="/build/toolchains/$tn/bin:$PATH" # Running in subshell. cd "/tmp/toolchains/$tn/build2-toolchain" local r=3 local l_stat b_stat while true; do # The "breakout loop". l_stat="$(bpkg status libbbot)" b_stat="$(bpkg status bbot)" if ! bpkg --fetch-timeout "$timeout" fetch -q; then info "failed to fetch package information" break fi # See if this is the first time or if we need to upgrade. # if [ "$(cut -d ' ' -f 2 <<<"$b_stat")" = "configured" ]; then # We assume that if anything has changed in the status line, then we # have a new version. # if [ "$b_stat" = "$(bpkg status bbot)" -a \ "$l_stat" = "$(bpkg status libbbot)" ]; then r=0 break fi r=1 break fi r=2 break done cd "$owd" return "$r" } # Build and start bbot agent using the bpkg configuration created by # toolchain_bootstrap(). # function bbot_start () # { local tn="$1" local ti="$2" local tp="${toolchains["$tn"]}" local tv="$(toolchain_value "$tp" toolchain_fver)" local ts="$(toolchain_value "$tp" toolchain_file_csum)" local id="/build/bots/$tn" mkdir -p "$id" # Install/uninstall vars. # local vars=(config.install.root="$id" config.bin.rpath="$id/lib") export PATH="/build/toolchains/$tn/bin:$PATH" # Running in subshell. cd "/tmp/toolchains/$tn/build2-toolchain" local r=1 local i n b_word while true; do # The "breakout loop". b_word="$(bpkg status bbot | cut -d ' ' -f 2)" # If upgrading, stop the service and uninstall. # if [ "$b_word" = "configured" ]; then if ! sudo systemctl stop "bbot-agent@$tn"; then info "failed to stop bbot-agent@$tn service, assuming not running" fi # We may not be able to uninstall if we previously failed to build. # if ! bpkg uninstall "${vars[@]}" bbot; then info "failed to uninstall bbot agent, assuming not installed" fi fi # Build and install the bbot agent. Since other agents might already # be running, limit the number of jobs to our slice. # if ! bpkg --fetch-timeout "$timeout" \ --build-option --jobs --build-option "$cpu_slice" \ build --yes libbbot bbot; then info "failed to build bbot-agent@$tn" break fi if ! bpkg install "${vars[@]}" bbot; then info "failed to install bbot-agent@$tn" break fi # Post-process and install systemd .service file. Note that we cannot use # the systemd pattern machinery since each version of bbot can have its # own version of the .service file. # sed -i -r \ -e "s/%[iI]/$tn/g" \ -e "s#^(Environment=AUTH_KEY)=.*#\1=/state/etc/host-key.pem#" \ -e "s/^(Environment=CPU)=.*/\1=$cpu_slice/" \ -e "s/^(Environment=RAM)=.*/\1=$ram_slice/" \ -e "s/^(Environment=TOOLCHAIN_ID)=.*/\1=$ts/" \ -e "s/^(Environment=TOOLCHAIN_NUM)=.*/\1=$ti/" \ -e "s/^(Environment=TOOLCHAIN_VER)=.*/\1=$tv/" \ "$id/lib/systemd/system/bbot-agent@.service" # Patch in the controller URLs. These can contain special characters # like & so we have to escape them. # n="${tp}controller_url[@]" for i in "${!n}"; do i="$(sed -e 's/[&/\]/\\&/g' <<<"$i")" sed -i -r \ -e "s#^(Environment=\"CONTROLLER_URL=[^\"]*)\"\$#\1 $i\"#" \ "$id/lib/systemd/system/bbot-agent@.service" done # Patch in the controller trust fingerprints. # n="${tp}controller_trust[@]" for i in "${!n}"; do sed -i -r \ -e "s#^(Environment=\"CONTROLLER_TRUST=[^\"]*)\"\$#\1 --trust $i\"#" \ "$id/lib/systemd/system/bbot-agent@.service" done sudo ln -sf "$id/lib/systemd/system/bbot-agent@.service" \ "/usr/lib/systemd/system/bbot-agent@$tn.service" # Clean up any machine snapshots that might have been left behind. # machines_for machines_clean_toolchain "$tn" if [ "${#diag[@]}" -gt 0 ]; then if [ -z "$fail" ]; then info "cleaned up entries in /build/machines/" else info "invalid entries in /build/machines/, not starting" fi print_diag 1>&2 if [ -n "$fail" ]; then info "correct and start bbot-agent@$tn (systemctl start bbot-agent@$tn)" break fi fi # Start the service. With Type=simple start returns as soon as the process # has forked. To see if the service actually started is done as part of # service monitoring. # if ! sudo systemctl start "bbot-agent@$tn"; then info "failed to start bbot-agent@$tn service" break fi r=0 break done cd "$owd" return "$r" } # Array of bootstrapped toolchains. # # The idea is to collect them until we bootstrap all of them and only then # start their bbot agents. # toolchain_boots=() declare -A toolchain_cursors # Latest systemd journal cursor. # Monitoring loop. # count=0 while true; do count=$(($count + 1)) # Check for toolchain changes. If this is the first run, bootstrap them. # for tn in "${toolchain_names[@]}"; do tp="${toolchains["$tn"]}" tu="$(toolchain_value "$tp" toolchain_url)" tr="$(toolchain_value "$tp" toolchain_root)" tf="$(toolchain_value "$tp" toolchain_file)" p="$tr/$tf" mkdir -p "$tr" # Fetch the toolchain sums either to $p if this is the first time or to # $p.new if we are checking for changes. # if [ -e "$p" ]; then f="$p.new" else f="$p" fi if "${curl[@]}" -o "$f" "$tu"; then # Take care of change detection. # if [ "$f" != "$p" ]; then ts="$(toolchain_value "$tp" toolchain_file_csum)" cs="$(toolchain_checksum "$tp" "$f")" if [ "$ts" != "$cs" ]; then email "rebooting because of new '$tn' toolchain" <&1 | tee "$tr/toolchain-$count.log" 1>&2 case "${PIPESTATUS[0]}" in 0) tv="$(cat $tr/version)" declare "${tp}toolchain_ver=$tv" tv="$(cat $tr/version-full)" declare "${tp}toolchain_fver=$tv" s="bootstrapped '$tn' toolchain $tv" toolchain_boots+=("$tn") ;; 1) s="skipping disabled '$tn' toolchain, waiting for new version" toolchain_boots+=("") # Skip. ;; *) s="failed to bootstrap '$tn' toolchain, waiting for new version" toolchain_boots+=("") # Skip. ;; esac info "$s" email "$s" <&1 | tee "$tr/bbot-$count.log" 1>&2 case "${PIPESTATUS[0]}" in 0) rm -f "$tr/bbot-$count.log" # Check if the service has failed. # if sudo systemctl is-failed --quiet "bbot-agent@$tn"; then s="bbot-agent@$tn service has failed, stopping" # Note: ignore errors. # sudo systemctl status "bbot-agent@$tn" 2>&1 | \ tee "$tr/bbot-$count.log" 1>&2 # Reset it so that we don't keep sending the log on each # iteration. Note: ignore errors. # sudo systemctl reset-failed "bbot-agent@$tn" 2>&1 | \ tee -a "$tr/bbot-$count.log" 1>&2 else # See if there is any diagnostics in the systemd journal. We # notify about warning and up. # # The old versions journalctl behavior is to not output anything # (not even the cursor) if there are no new entries. The new # versions output the old cursor. # # Plus, it sometimes changes the cursor even without any errors in # it (journal rewind/truncation maybe?) so we have to detect that. # c=(sudo journalctl --no-pager --quiet --output short-full \ --unit "bbot-agent@$tn") # Get the last cursor if any. # oc="${toolchain_cursors["$tn"]}" if [ -n "$oc" ]; then c+=("--after-cursor" "$oc") fi # Get the "log range": the first line is the date of the first # error, the second line is the date of the last error, and the # third line is the end cursor. It can also be just one line in # which case it is the new cursor (that rewind stuff). # # Here is what's going on in that sed script: # # The first chunk matches the first line. We first put it into # the hold space (in case that's the only line) and then extract # and print the date. # # The second chunk matches the last line. We first handle the hold # space which by now should contain the last error line and then # the cursor. # # The last chunk matches every other line. We simply replace the # hold space with the next line so that at the end we have the # last line there. # lr="$("${c[@]}" --priority 4 --show-cursor | sed -n -r \ -e '1{h;s/^[MTWFS].. ([^ ]+ [^ ]+) .*$/\1/p;t}' \ -e '${x;s/^[MTWFS].. ([^ ]+ [^ ]+) .*$/\1/p;x;s/^-- cursor: (.+)$/\1/p;t}' \ -e 'h')" lc="$(wc -l <<<"$lr")" nc="$(sed -n -e "${lc}p" <<<"$lr")" # If we have no new entries, then nothing to do. # if [ "$nc" != "$oc" ]; then # We may have no actual entries (cursor rewind). # if [ "$lc" -ne 1 ]; then # Try to get some context before the first error and after the # last. This is unexpectedly hard in systemd. # # This can be a lot of output which makes it hard to spot the # error so we are going to print just the error summary first. # Quite a mess, I agree. # sd="$(sed -n -e '1p' <<<"$lr")" sd="$(date '+%s' -d "$sd")" # sec sd="$(date '+%Y-%m-%d %H:%M:%S' -d "@$(($sd - 10))")" # -10sec ed="$(sed -n -e '2p' <<<"$lr")" ed="$(date '+%s' -d "$ed")" # sec ed="$(date '+%Y-%m-%d %H:%M:%S' -d "@$(($ed + 10))")" # +10sec s="bbot-agent@$tn service issued new diagnostics" info "$s" { echo "$tn.bbot_cmd: ssh build@$hname ${c[@]}"; echo; echo "summary:"; echo; "${c[@]}" --priority 4 | head -n 200; echo; echo "context:"; echo; if [ -n "$oc" ]; then unset 'c[-1]' # Pop cursor (for --since/--until). unset 'c[-1]' fi; "${c[@]}" --since "$sd" --until "$ed" | head -n 200 } | email "$s" fi toolchain_cursors["$tn"]="$nc" fi continue fi ;; 1) s="re" ;& 2) info "${s}starting bbot-agent@$tn..." # Note: appending to the same log. # bbot_start "$tn" "$ti" 2>&1 | tee -a "$tr/bbot-$count.log" 1>&2 if [ "${PIPESTATUS[0]}" -eq 0 ]; then s="${s}started bbot-agent@$tn" else s="failed to ${s}start bbot-agent@$tn, waiting for new version" fi ;; *) s="failed to fetch package information for '$tn' toolchain, will try again" ;; esac info "$s" email "$s" <