Add machine test, bootstrap/environment scripts

9 files changed, 510 insertions, 1 deletions

diff --git a/buildfile b/buildfile
index 49305bf..53b4f37 100644
--- a/buildfile
+++ b/buildfile
@@ -2,7 +2,7 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : TBC; see accompanying LICENSE file
 
-./: bbot/ unit-tests/ doc{INSTALL LICENSE NEWS README version} file{manifest}
+./: bbot/ etc/ unit-tests/ doc{INSTALL LICENSE NEWS README version} file{manifest}
 
 # Don't install tests or the INSTALL file.
 #
diff --git a/etc/bootstrap/bbot-bootstrap-msvc.bat b/etc/bootstrap/bbot-bootstrap-msvc.bat
new file mode 100644
index 0000000..de8b297
--- /dev/null
+++ b/etc/bootstrap/bbot-bootstrap-msvc.bat
@@ -0,0 +1,164 @@
+@echo off
+
+rem file      : etc/bootstrap/bbot-bootstrap-msvc.bat
+rem copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+rem license   : TBC; see accompanying LICENSE file
+
+setlocal EnableExtensions EnableDelayedExpansion
+
+set "MSVC=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community"
+set "VCVARS=%MSVC%\VC\Auxiliary\Build\vcvars64.bat"
+
+set "BUILD=C:\tmp"
+set "INSTALL=C:\build2"
+set "BOOTSTRAP=C:\bootstrap"
+set "ENVIRONMENT=C:\environment"
+
+set "TFTP=196.254.111.222"
+rem set "TFTP=10.1.0.1:55123"
+set "VERBOSE=3"
+
+rem If we already have the bbot worker, assume we are bootstrapped.
+rem
+if exist %INSTALL%\bin\bbot-worker.exe (
+  set "PATH=%INSTALL%\bin;%PATH%"
+  bbot-worker.exe --startup --build %BUILD% --environment %ENVIRONMENT%^
+ --tftp-host %TFTP% --verbose %VERBOSE%
+  goto end
+)
+
+rem Setup the compiler for the toolchain.
+rem
+call "%VCVARS%"
+if errorlevel 1 goto error
+
+rem Use bootstrap tools.
+rem
+set "PATH=%BOOTSTRAP%\bin;%PATH%"
+
+rem Show the steps we are performing.
+rem
+@echo on
+
+@rem
+@rem Bootstrap the toolchain and then build bbot.
+@rem
+@if exist %BUILD%\bootstrap\ (
+  rmdir /S /Q %BUILD%\bootstrap
+  @if errorlevel 1 goto error
+)
+
+mkdir %BUILD%\bootstrap
+@if errorlevel 1 goto error
+
+@if exist %INSTALL% (
+  rmdir /S /Q %INSTALL%
+  @if errorlevel 1 goto error
+)
+
+cd %BUILD%\bootstrap
+
+@rem
+@rem Get the baseutils.
+@rem
+@rem We could be running on a new network which may take Windows some time
+@rem to digest. And if we start before that happens, we will be hanging
+@rem forever.
+@rem
+:restart
+curl -s -S -O --connect-timeout 5 --max-time 30^
+  "tftp://%tftp%/build2-baseutils-x86_64-windows.zip"
+@if errorlevel 1 goto restart
+
+unzip -q build2-baseutils-x86_64-windows.zip
+@if errorlevel 1 goto error
+
+del build2-baseutils-x86_64-windows.zip
+@if errorlevel 1 goto error
+
+move build2-baseutils-*-x86_64-windows %INSTALL%
+@if errorlevel 1 goto error
+
+
+@rem
+@rem Get the toolchain.
+@rem
+curl -s -S -O "tftp://%tftp%/build2-toolchain.tar.xz"
+@if errorlevel 1 goto error
+
+tar -xf build2-toolchain.tar.xz
+@if errorlevel 1 goto error
+
+del build2-toolchain.tar.xz
+@if errorlevel 1 goto error
+
+
+@rem
+@rem Get the repository certificate fingerprint.
+@rem
+curl -s -S -O "tftp://%tftp%/trust"
+@if errorlevel 1 goto error
+
+@set /P trust=<trust
+
+del trust
+@if errorlevel 1 goto error
+
+
+@rem
+@rem Bootstrap and install the toolchain.
+@rem
+@for /D %%d in (build2-toolchain-*) do set "bstrap=%%d"
+cd %bstrap%
+
+@rem Note: executing in a separate cmd.exe to preserve the echo mode.
+@rem
+cmd /C build-msvc.bat %INSTALL% %trust%
+@if errorlevel 1 goto error
+
+cd ..
+rmdir /S /Q %bstrap%
+@if errorlevel 1 goto error
+
+
+@rem
+@rem Switch to the bootstrapped utilities/toolchain.
+@rem
+@set "PATH=%INSTALL%\bin;%PATH%"
+
+
+@rem
+@rem Build and install the bbot worker.
+@rem
+@for /D %%d in (build2-toolchain-*) do set "config=%%d"
+cd %config%
+
+bpkg build --yes bbot
+@if errorlevel 1 goto error
+
+bpkg install bbot
+@if errorlevel 1 goto error
+
+cd ..
+rmdir /S /Q %config%
+@if errorlevel 1 goto error
+
+@rem
+@rem Finish off by uploading the result manifest produced by the bbot worker.
+@rem
+bbot-worker --bootstrap >manifest
+@if errorlevel 1 goto error
+
+curl -s -S --upload-file manifest "tftp://%tftp%/manifest"
+@if errorlevel 1 goto error
+
+@echo off
+goto end
+
+:error
+@echo off
+endlocal
+exit /b 1
+
+:end
+endlocal
diff --git a/etc/bootstrap/bbot-bootstrap.service b/etc/bootstrap/bbot-bootstrap.service
new file mode 100644
index 0000000..ecb0da4
--- /dev/null
+++ b/etc/bootstrap/bbot-bootstrap.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=bbot worker bootstrap
+After=default.target
+Conflicts=getty@tty1.service
+
+[Service]
+Type=idle
+# Old versions of systemd have no 'infinity'.
+TimeoutStartSec=6000min
+RemainAfterExit=true
+User=build
+Group=build
+# Old versions of systemd have no '~'.
+WorkingDirectory=/home/build
+
+Environment=CXX=g++
+Environment=BUILD=/tmp
+Environment=ENVIRONMENT=/home/build/environment
+
+ExecStart=/usr/local/bin/bbot-bootstrap.sh \
+  --cxx ${CXX} \
+  --build ${BUILD} \
+  --environment ${ENVIRONMENT}
+
+StandardInput=tty-force
+StandardOutput=inherit
+StandardError=inherit
+TTYPath=/dev/tty1
+TTYReset=yes
+TTYVHangup=yes
+
+[Install]
+WantedBy=default.target
diff --git a/etc/bootstrap/bbot-bootstrap.sh b/etc/bootstrap/bbot-bootstrap.sh
new file mode 100755
index 0000000..65deddc
--- /dev/null
+++ b/etc/bootstrap/bbot-bootstrap.sh
@@ -0,0 +1,131 @@
+#!/bin/sh
+
+# file      : etc/bootstrap/bbot-bootstrap.sh
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+usage="Usage: $0 [<options>]"
+
+set -e # Exit on errors.
+
+diag ()
+{
+  echo "$*" 1>&2
+}
+
+error ()
+{
+  diag "$*"
+  exit 1
+}
+
+# Note that this function will execute a command with arguments that contain
+# spaces but it will not print them as quoted (and neither does set -x).
+#
+run ()
+{
+  diag "+ $@"
+  "$@"
+  if test "$?" -ne "0"; then
+    exit 1;
+  fi
+}
+
+# Defaults that can be changed via command line.
+#
+cxx=g++
+build=/tmp
+environment="$HOME/environment"
+
+# Parse options.
+#
+while test $# -ne 0; do
+  case $1 in
+    --cxx)
+      shift
+      if test $# -eq 0; then
+	error "missing C++ compiler after --cxx"
+      fi
+      cxx="$1"
+      shift
+      ;;
+    --build)
+      shift
+      if test $# -eq 0; then
+	error "missing build directory after --build"
+      fi
+      build="$1"
+      shift
+      ;;
+    --environment)
+      shift
+      if test $# -eq 0; then
+	error "missing environment directory after --environment"
+      fi
+      environment="$1"
+      shift
+      ;;
+    *)
+      error "unexpected argument '$1'"
+      ;;
+  esac
+done
+
+# Defaults that can be changed for testing.
+#
+# Note: build_options is an array-like (expanded unquoted).
+#
+tftp="196.254.111.222"
+install="/usr/local"
+build_options=
+verbose=3
+
+#install="/tmp/bbot-install"
+#tftp="127.0.0.1:55123"
+#build_options="--install-dir $install"
+
+PATH="$install/bin:$PATH"
+export PATH
+
+# If we already have the bbot worker, assume we are bootstrapped.
+#
+if bbot-worker --version >/dev/null 2>&1; then
+  exec bbot-worker --startup --build "$build" --environment "$environment" \
+       --tftp-host "$tftp" --verbose "$verbose"
+fi
+
+# Bootstrap the toolchain and then build bbot.
+#
+run rm -rf "$build/bootstrap"
+run mkdir -p "$build/bootstrap"
+run cd "$build/bootstrap"
+
+run curl -s -S -O "tftp://$tftp/build2-toolchain.tar.xz"
+run tar -xf build2-toolchain.tar.xz
+run rm build2-toolchain.tar.xz
+
+run curl -s -S -O "tftp://$tftp/trust"
+trust="$(cat trust)"
+run rm trust
+
+# Bootstrap and install the toolchain (by default into /usr/local using sudo).
+#
+bstrap="$(echo build2-toolchain-*)"
+run cd "$bstrap"
+run ./build.sh --trust "$trust" $build_options "$cxx"
+run cd ..
+run rm -r "$bstrap"
+
+# Build and install the bbot worker.
+#
+config="$(echo build2-toolchain-*)"
+run cd "$config"
+run bpkg build --yes bbot
+run bpkg install bbot
+run cd ..
+run rm -r "$config"
+
+# Finish off by uploading the result manifest produced by the bbot worker.
+#
+run bbot-worker --bootstrap >manifest
+run curl -s -S --upload-file manifest "tftp://$tftp/manifest"
diff --git a/etc/buildfile b/etc/buildfile
new file mode 100644
index 0000000..0c3b57b
--- /dev/null
+++ b/etc/buildfile
@@ -0,0 +1,8 @@
+# file      : etc/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+./: file{**}
+
+*: install = data/etc/
+*: install.subdirs = true # Recreate subdirectories.
diff --git a/etc/environment/default b/etc/environment/default
new file mode 100755
index 0000000..82107f6
--- /dev/null
+++ b/etc/environment/default
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# file      : etc/environment/default
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+# Environment setup script for C/C++ compilation.
+
+c=gcc
+cxx=g++
+
+# $1  - target
+# $2  - bbot executable
+# $3+ - bbot options
+
+set -e # Exit on errors.
+
+t="$1"
+shift
+
+if test -n "$t"; then
+  echo "unknown target: $t" 1>&2
+  exit 1
+fi
+
+exec "$@" cc config.c="$c" config.cxx="$cxx"
diff --git a/etc/environment/default-msvc.bat b/etc/environment/default-msvc.bat
new file mode 100644
index 0000000..64c325d
--- /dev/null
+++ b/etc/environment/default-msvc.bat
@@ -0,0 +1,56 @@
+@echo off
+
+rem file      : etc/environment/default-msvc.bat
+rem copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+rem license   : TBC; see accompanying LICENSE file
+
+rem
+rem Environment setup script for C/C++ compilation with Visual Studio 15.
+rem
+
+rem %1  - target
+rem %2  - bbot executable
+rem %3+ - bbot options
+
+setlocal EnableExtensions EnableDelayedExpansion
+
+set "MSVC=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community"
+
+set "VCVARS32=%MSVC%\VC\Auxiliary\Build\vcvarsamd64_x86.bat"
+set "VCVARS64=%MSVC%\VC\Auxiliary\Build\vcvars64.bat"
+
+rem Based on target determine what we are building. If the target is
+rem not specified, then we build 64-bit by default.
+rem
+rem Note that an empty argument is passed as "" (literal quotes).
+rem
+if "_%1_" == "_x86_64-microsoft-win32-msvc14.1_" (
+  set "VCVARS=%VCVARS64%"
+) else (
+  if "_%1_" == "_i386-microsoft-win32-msvc14.1_" (
+    set "VCVARS=%VCVARS32%"
+  ) else (
+    if _%1_ == _""_ (
+      set "VCVARS=%VCVARS64%"
+    ) else (
+      echo error: unknown target %1
+      goto error
+    )
+  )
+)
+
+call "%VCVARS%"
+if errorlevel 1 goto error
+
+%2 %3 %4 %5 %6 %7 %8 %9 cc config.c=cl config.cxx=cl
+if errorlevel 1 goto error
+
+goto end
+
+:error
+@echo off
+endlocal
+exit /b 1
+
+:end
+endlocal
diff --git a/tests/machine/buildfile b/tests/machine/buildfile
new file mode 100644
index 0000000..a9f6ac3
--- /dev/null
+++ b/tests/machine/buildfile
@@ -0,0 +1,8 @@
+# file      : tests/machine/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+./: ../../bbot/exe{bbot-agent} test{testscript}
+dir{./}: test = ../../bbot/exe{bbot-agent}
+
+include ../../bbot/
diff --git a/tests/machine/testscript b/tests/machine/testscript
new file mode 100644
index 0000000..238ca91
--- /dev/null
+++ b/tests/machine/testscript
@@ -0,0 +1,83 @@
+# file      : tests/machine/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+test.options = --cpu 8 --ram 10485760 --verbose 3
+
+tftp = /build/tftp
+machines = /btrfs/boris/machines # @@ TODO
+
+toolchain_url   = https://stage.build2.org/0
+toolchain_trust = B8:52:AB:94:C5:FA:73:F3:53:D3:F1:EB:0F:E0:E4:06:32:3F:E2:46:22:FA:39:05:C8:FA:70:B7:21:E2:46:C5
+
+pkg = hello
+ver = 1.0.0
+rep = https://build2.org/pkg/1/stage/stable
+rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
+
+# Download the toolchain.
+#
+# Note: similar logic to what we have in Build OS.
+#
++mkdir -p $tftp/toolchain/default/
++curl -s -S -f -L $toolchain_url/toolchain.sha256 >=toolchain.sha256
+
++sed -n -e 's%^[0-9a-f]+ \*(.+)$%\1%p' toolchain.sha256 | set -e archives
++sed -n -e 's%^(.+/)?build2-toolchain-(.+)\.tar.*%\2%p' <"$archives" | set version
++sha256sum -b toolchain.sha256 | sed -n -e 's%^([0-9a-f]+) .*$%\1%p' | set checksum
+
++/bin/bash -c "while read i && test -n \"\$i\"; do \
+b=`basename \$i` ; \
+f=$tftp/toolchain/default/\$b; \
+echo \$b: 1>&2; \
+curl -# -f -L -z \$f -o \$f $toolchain_url/\$i; \
+ln -sf \$b `sed -n -re 's/^\(.+\)-$version\(.+\)/\\1\\2/p' <<<\$f`; \
+done" <"$archives" 2>|
+
++echo "$version"         >=$tftp/toolchain/default/version
++echo "$toolchain_trust" >=$tftp/toolchain/default/trust
+
+#\
+#
+# To test the bootstrap script locally, start the TFTP server in
+# $tftp/toolchain/default/.
+#
+
+sudo /usr/sbin/in.tftpd \
+  --foreground \
+  --address :55123 \
+  --user "$(whoami)" \
+  --permissive \
+  --create \
+  --secure \
+  "$(pwd)"
+
+# Then uncommen testing setting (after option parsing) and run:
+
+etc/bootstrap/bbot-bootstrap.sh # Should bootstrap (check manifest on TFTP).
+etc/bootstrap/bbot-bootstrap.sh # Should startup (fails with invalid manifest).
+
+# The same can be done for batch files but from the Windows VM.
+
+# Can also provide a test task manifest from below.
+
+#\
+
+# Boostrap and list the machines.
+#
+test.options += --machines $machines --toolchain-id $checksum
+
++$* --dump-machines 2>| | sed -n -e 's/^name: (.+)$/\1/p' | set -n machine
+
+# Perform a test build.
+#
++echo "building on $machine" >&2 2>|
+
++cat <<"EOI" | $* --fake-request - --dump-result >| 2>|
+  : 1
+  name: $pkg
+  version: $ver
+  repository: $rep
+  trust: $rfp
+  machine: $machine
+  EOI