Finish agent and worker logic

5 files changed, 357 insertions, 35 deletions

diff --git a/tests/agent/testscript b/tests/agent/testscript
index cf92ace..e864bf2 100644
--- a/tests/agent/testscript
+++ b/tests/agent/testscript
@@ -3,7 +3,7 @@
 # license   : TBC; see accompanying LICENSE file
 
 # The /build/machines directory should be on a btrfs filesystem and have the
-# following layout and contents:
+# following layout/contents:
 #
 # /build/machines/
 # └── default/
@@ -17,25 +17,24 @@
 #
 #  - The test must be run serially (@@ TODO: serial directive)
 
-test.options   = --verbose 2
-test.arguments = stage 1
+test.options = --verbose 3
 
 cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines
 rm = $src_base/btrfs-rmdir /build/machines
 
-#\
 : dump-machines
 :
 {
-  m = /build/machines/default/linux-gcc
-
   test.options += --dump-machines --fake-bootstrap
 
+  m = /build/machines/default/linux-gcc
+  u = https://example.org/?dummy
+
   +$cp
 
   : no-current-machine-symlink
   :
-  $* 123 2>>"EOE"
+  $* --toolchain-id 123 $u 2>>"EOE"
     trace: enumerate_machines: skipping $m/: no subvolume link
     EOE
 
@@ -43,91 +42,140 @@ rm = $src_base/btrfs-rmdir /build/machines
 
   : bootstrap
   :
-  $* 123 >>EOO 2>>~"%EOE%d"
+  $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d"
     : 1
     id: linux-gcc-1.0
     name: linux-gcc
     summary: Linux with GCC
     EOO
-    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+%
-    trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-default-\\.+%
+    trace: enumerate_machines: bootstrapping $m/linux-gcc-default/
+    trace: enumerate_machines: bootstrapped linux-gcc
     EOE
 
   ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1
 
   : re-bootstrap-machine
   :
-  $* 123 >>EOO 2>>~"%EOE%d"
+  $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d"
     : 1
     id: linux-gcc-1.1
     name: linux-gcc
     summary: Linux with GCC
     EOO
-    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
-    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine
-    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
-    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new machine
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+    trace: enumerate_machines: bootstrapped linux-gcc
     EOE
 
   : re-bootstrap-toolchain
   :
-  $* 124 >>EOO 2>>~"%EOE%d"
+  $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d"
     : 1
     id: linux-gcc-1.1
     name: linux-gcc
     summary: Linux with GCC
     EOO
-    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
-    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain
-    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
-    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new toolchain
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+    trace: enumerate_machines: bootstrapped linux-gcc
     EOE
 
-  sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest
+  sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-default/manifest
 
   : re-bootstrap-bbot
   :
-  $* 124 >>EOO 2>>~"%EOE%d"
+  $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d"
     : 1
     id: linux-gcc-1.1
     name: linux-gcc
     summary: Linux with GCC
     EOO
-    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
-    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot
-    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
-    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new bbot
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+    trace: enumerate_machines: bootstrapped linux-gcc
     EOE
 
-  sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest
+  sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-default/manifest
 
   : re-bootstrap-bbot-newer
   :
-  $* 124 2>>~"%EOE%d"
-    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
-    trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot
-    %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+%
+  $* --toolchain-id 124 $u 2>>~"%EOE%d"
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+    trace: enumerate_machines: ignoring $m/linux-gcc-default/: old bbot
+    %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default-\\.+%
     EOE
 
   -$rm
 }
 
-#\
-
-: bootstrap
+: dump-result
 :
 {
+  test.options += --fake-bootstrap --fake-build --fake-request -
+
   m = /build/machines/default/linux-gcc
+  u = https://example.org/?dummy
+
+  +$cp
+  +ln -T -s linux-gcc-1.0 $m/linux-gcc-1
+  +$* --dump-machines --verbose 1 $u >- # Bootstrap.
+
+  : no-machine
+  :
+  $* --dump-task $u <<EOI 2>>~"%EOE%d"
+    : 1
+    name: foo
+    version: 1.2.3
+    repository: https://example.org/1/
+    machine: windows-msvc
+    EOI
+    %trace: enumerate_machines:\\.*%*
+    error: task from $u for unknown machine windows-msvc
+    EOE
 
+  : result
+  :
+  $* --dump-result $u <<EOI >>EOO 2>>~"%EOE%d"
+    : 1
+    name: foo
+    version: 1.2.3
+    repository: https://example.org/1/
+    machine: linux-gcc
+    EOI
+    : 1
+    name: foo
+    version: 1.2.3
+    status: abort
+    EOO
+    %trace: enumerate_machines:\\.*%*
+    EOE
+
+  -$rm
+}
+
+#\
+: bootstrap
+:
+{
   test.options += --dump-machines
 
+  m = /build/machines/default/linux-gcc
+  u = https://example.org/?dummy
+
   +$cp
 
   ln -T -s linux-gcc-1.0 $m/linux-gcc-1
 
   : bootstrap
   :
-  $* 123 >>EOO 2>>EOE #2>>~"%EOE%d"
+  $* $u >>EOO 2>>EOE #2>>~"%EOE%d"
     : 1
     id: linux-gcc-1.0
     name: linux-gcc
@@ -137,3 +185,4 @@ rm = $src_base/btrfs-rmdir /build/machines
 
   #-$rm
 }
+#\
diff --git a/tests/worker/bootstrap.test b/tests/worker/bootstrap.test
new file mode 100644
index 0000000..91e4417
--- /dev/null
+++ b/tests/worker/bootstrap.test
@@ -0,0 +1,13 @@
+# file      : tests/worker/bootstrap.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+test.options = --bootstrap
+
+$* >>~%EOO%
+: 1
+%bbot-version: \d+%
+%libbbot-version: \d+%
+%libbpkg-version: \d+%
+%libbutl-version: \d+%
+EOO
diff --git a/tests/worker/build.test b/tests/worker/build.test
new file mode 100644
index 0000000..adedccc
--- /dev/null
+++ b/tests/worker/build.test
@@ -0,0 +1,133 @@
+# file      : tests/worker/build.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+# Note: requires TFTP server (see buildfile).
+
+tftp = 127.0.0.1:55123/test
+
+pkg = hello
+ver = 1.0.0
+
+#rep = /home/boris/work/build2/hello/repository/1/stable
+rep = https://build2.org/pkg/1/stage/stable
+#rep = https://build2.org/pkg/1/hello/stable
+
+rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
+
+: pass
+:
+cat <<"EOI" >=manifest;
+  : 1
+  name: $pkg
+  version: $ver
+  repository: $rep
+  trust: $rfp
+  machine: linux-gcc
+  target: x86_64-linux-gnu
+  config: config.cc.coptions=-O3
+  EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+  trace: build: bpkg -v create -d build --wipe config.cc.coptions=-O3 cc
+  trace: build: bpkg -v add $rep
+  trace: build: bpkg -v fetch --trust $rfp --trust-no
+  trace: build: bpkg -v build --configure-only --yes $pkg/$ver
+  trace: build: bpkg -v update $pkg
+  trace: build: bpkg -v test $pkg
+  trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+  EOE
+cat manifest >>~"%EOO%"
+  : 1
+  name: $pkg
+  version: $ver
+  status: success
+  configure-status: success
+  update-status: success
+  test-status: success
+  configure-log: \\
+  %.*%+
+  \\
+  update-log: \\
+  %.*%+
+  \\
+  test-log: \\
+  %.*%+
+  \\
+  EOO
+
+
+: fail-abnormal
+:
+cat <<"EOI" >=manifest;
+  : 1
+  foo: bar
+  EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc 2>>"EOE" != 0
+  error: invalid task manifest: manifest:3:1: no task package name specified
+  EOE
+
+
+: fail-configure
+:
+cat <<"EOI" >=manifest;
+  : 1
+  name: bogus
+  version: 1.2.3
+  repository: $rep
+  trust: $rfp
+  machine: linux-gcc
+  target: x86_64-linux-gnu
+  EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+  trace: build: bpkg -v create -d build --wipe cc
+  trace: build: bpkg -v add $rep
+  trace: build: bpkg -v fetch --trust $rfp --trust-no
+  trace: build: bpkg -v build --configure-only --yes bogus/1.2.3
+  trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+  EOE
+cat manifest >>~"%EOO%"
+  : 1
+  name: bogus
+  version: 1.2.3
+  status: error
+  configure-status: error
+  configure-log: \\
+  %.*%+
+  \\
+  EOO
+
+
+: fail-update
+:
+cat <<"EOI" >=manifest;
+  : 1
+  name: $pkg
+  version: $ver
+  repository: $rep
+  trust: $rfp
+  machine: linux-gcc
+  target: x86_64-linux-gnu
+  config: config.cc.loptions=-lbogus
+  EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+  trace: build: bpkg -v create -d build --wipe config.cc.loptions=-lbogus cc
+  trace: build: bpkg -v add $rep
+  trace: build: bpkg -v fetch --trust $rfp --trust-no
+  trace: build: bpkg -v build --configure-only --yes $pkg/$ver
+  trace: build: bpkg -v update $pkg
+  trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+  EOE
+cat manifest >>~"%EOO%"
+  : 1
+  name: $pkg
+  version: $ver
+  status: error
+  configure-status: success
+  update-status: error
+  configure-log: \\
+  %.*%+
+  \\
+  update-log: \\
+  %.*%+
+  \\
+  EOO
diff --git a/tests/worker/buildfile b/tests/worker/buildfile
new file mode 100644
index 0000000..7d0a206
--- /dev/null
+++ b/tests/worker/buildfile
@@ -0,0 +1,26 @@
+# file      : tests/worker/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+#\
+
+Some tests in this directory require a running TFTP server.
+
+TFTP server (tftp-hpa) setup: from the test out_base, run (sudo is required
+for --secure/chroot):
+
+sudo /usr/sbin/in.tftpd \
+  --foreground \
+  --address 127.0.0.1:55123 \
+  --user "$(whoami)" \
+  --permissive \
+  --create \
+  --secure \
+  "$(pwd)"
+
+#\
+
+./: ../../bbot/exe{bbot-worker} test{bootstrap startup build}
+dir{./}: test = ../../bbot/exe{bbot-worker}
+
+include ../../bbot/
diff --git a/tests/worker/startup.test b/tests/worker/startup.test
new file mode 100644
index 0000000..d13e6c5
--- /dev/null
+++ b/tests/worker/startup.test
@@ -0,0 +1,101 @@
+# file      : tests/worker/startup.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : TBC; see accompanying LICENSE file
+
+# Note: requires TFTP server (see buildfile).
+
+test.options = --startup
+tftp = 127.0.0.1:55123/test
+
+: fail-download
+:
+$* --tftp-host "$tftp/$@" 2>>"EOE" &manifest != 0;
+  curl: \(68\) TFTP: File Not Found
+  error: unable to download task manifest from tftp://$tftp/$@/manifest: input/output error
+  EOE
+diff -u - manifest <<EOO
+  : 1
+  name: unknown
+  version: 0
+  status: abnormal
+  EOO
+
+: fail-manifest
+:
+cat <': 1' >=manifest;
+$* --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+  error: invalid task manifest: manifest:2:1: no task package name specified
+  EOE
+diff -u - manifest <<EOO
+  : 1
+  name: unknown
+  version: 0
+  status: abnormal
+  EOO
+
+: fail-env-search
+:
+cat <<EOI >=manifest;
+  : 1
+  name: libhello
+  version: 1.2.3
+  repository: https://pkg.example.org/1/
+  machine: linux-gcc
+  EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+  error: no environment setup executable in $representation($~) for target ''
+  EOE
+diff -u - manifest <<EOO
+  : 1
+  name: libhello
+  version: 1.2.3
+  status: abnormal
+  EOO
+
+: fail-env-run
+:
+cat <<EOI >=x86_64-linux-gnu;
+  #!/bin/sh
+  echo "environment setup failed"
+  exit 1
+  EOI
+chmod ugo+x x86_64-linux-gnu;
+cat <<EOI >=manifest;
+  : 1
+  name: libhello
+  version: 1.2.3
+  repository: https://pkg.example.org/1/
+  machine: linux-gcc
+  target: x86_64-linux-gnu
+  EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+  environment setup failed
+  error: process $~/x86_64-linux-gnu terminated with non-zero exit code
+  EOE
+diff -u - manifest <<EOO
+  : 1
+  name: libhello
+  version: 1.2.3
+  status: abnormal
+  EOO
+
+: pass
+:
+cat <<EOI >=x86_64-linux-gnu;
+  #!/bin/sh
+  echo "$1"
+  echo "$2"
+  EOI
+chmod ugo+x x86_64-linux-gnu;
+cat <<EOI >=manifest;
+  : 1
+  name: libhello
+  version: 1.2.3
+  repository: https://pkg.example.org/1/
+  machine: linux-gcc
+  target: x86_64-linux-gnu
+  EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE"
+  x86_64-linux-gnu
+  $0
+  EOE