From efa53d6ad8797310b10d299408c2e3fa33978e27 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 19 Apr 2017 08:02:53 +0200 Subject: Add --fake-machine and machine-less agent/worker test --- bbot/agent | 2 - bbot/agent.cli | 7 +++ bbot/agent.cxx | 114 +++++++++++++++++++++++++++---------------- bbot/utility | 2 + bbot/utility.txx | 5 ++ bbot/worker.cli | 5 ++ bbot/worker.cxx | 23 ++++++++- doc/manual.cli | 40 +++++++++------ tests/agent/testscript | 29 +---------- tests/integration/buildfile | 8 +++ tests/integration/testscript | 98 +++++++++++++++++++++++++++++++++++++ tests/integration/tftp-map | 11 +++++ tests/worker/build.test | 12 ++--- 13 files changed, 261 insertions(+), 95 deletions(-) create mode 100644 tests/integration/buildfile create mode 100644 tests/integration/testscript create mode 100644 tests/integration/tftp-map diff --git a/bbot/agent b/bbot/agent index 14d2995..1271a87 100644 --- a/bbot/agent +++ b/bbot/agent @@ -22,8 +22,6 @@ namespace bbot extern size_t tc_num; // Toolchain number. extern string tc_id; // Toolchain id. - extern strings controllers; // Controller URLs. - extern string hname; // Our host name. extern uid_t uid; // Our effective user id. extern string uname; // Our effective user name. diff --git a/bbot/agent.cli b/bbot/agent.cli index c67535e..b71db4e 100644 --- a/bbot/agent.cli +++ b/bbot/agent.cli @@ -149,6 +149,13 @@ namespace bbot "Fake the package building process by creating the aborted build result." } + path --fake-machine + { + "", + "Fake the machine enumeration process by reading the machine header + manifest from (or \cb{stdin} if is '\cb{-}')." + } + path --fake-request { "", diff --git a/bbot/agent.cxx b/bbot/agent.cxx index 8847e62..60d59e6 100644 --- a/bbot/agent.cxx +++ b/bbot/agent.cxx @@ -45,8 +45,6 @@ namespace bbot size_t tc_num; string tc_id; - strings controllers; - string hname; uid_t uid; string uname; @@ -257,6 +255,24 @@ try bootstrapped_machine_manifests rm; dir_paths rd; + if (ops.fake_machine_specified ()) + { + auto mh ( + parse_manifest ( + ops.fake_machine (), "machine header")); + + rm.push_back ( + bootstrapped_machine_manifest { + machine_manifest {mh.id, mh.name, mh.summary, machine_type::kvm}, + toolchain_manifest {tc_id}, + bootstrap_manifest {} + }); + + rd.push_back (dir_path (ops.machines ()) /= mh.name); // For diagnostics. + + return make_pair (move (rm), move (rd)); + } + // The first level are machine volumes. // for (const dir_entry& ve: dir_iterator (machines)) @@ -517,6 +533,7 @@ static result_manifest perform_task (const dir_path& md, const bootstrapped_machine_manifest& mm, const task_manifest& tm) +try { tracer trace ("perform_task"); @@ -542,7 +559,36 @@ perform_task (const dir_path& md, // // 5. Clean up (force the machine down and delete the snapshot). // - try + + // TFTP server mapping (server chroot is --tftp): + // + // GET requests to .../build//get/* + // PUT requests to .../build//put/* + // + auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name); + + dir_path gd (dir_path (arm.path ()) /= "get"); + dir_path pd (dir_path (arm.path ()) /= "put"); + + try_mkdir_p (gd); + try_mkdir_p (pd); + + path tf (gd / "manifest"); // Task manifest file. + path rf (pd / "manifest"); // Result manifest file. + + serialize_manifest (tm, tf, "task"); + + if (ops.fake_machine_specified ()) + { + // Simply wait for the file to appear. + // + for (size_t i (0); !file_exists (rf); sleep (1)) + if (i++ % 10 == 0) + l3 ([&]{trace << "waiting for result manifest";}); + + r = parse_manifest (rf, "result"); + } + else { // -- // @@ -553,24 +599,8 @@ perform_task (const dir_path& md, string br ("br1"); // Using private bridge for now. - // Start the TFTP server (server chroot is --tftp). Map: - // - // GET requests to .../build//get/* - // PUT requests to .../build//put/* + // Start the TFTP server. // - auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name); - - dir_path gd (dir_path (arm.path ()) /= "get"); - dir_path pd (dir_path (arm.path ()) /= "put"); - - try_mkdir_p (gd); - try_mkdir_p (pd); - - path tf (gd / "manifest"); // Task manifest file. - path rf (pd / "manifest"); // Result manifest file. - - serialize_manifest (tm, tf, "task"); - tftp_server tftpd ("Gr ^/?(.+)$ /build/" + tc_name + "/get/\\1\n" + "Pr ^/?(.+)$ /build/" + tc_name + "/put/\\1\n"); @@ -640,16 +670,6 @@ perform_task (const dir_path& md, // r = parse_manifest (rf, "result"); - // Update package name/version if the returned value as "unknown". - // - if (r.version == bpkg::version ("0")) - { - assert (r.status == result_status::abnormal); - - r.name = tm.name; - r.version = tm.version; - } - // If the build terminated abnormally, suspent the machine for // investigation (note that here we don't wait or return). // @@ -664,13 +684,23 @@ perform_task (const dir_path& md, run_btrfs (trace, "subvolume", "delete", xp); } - catch (const system_error& e) + + // Update package name/version if the returned value as "unknown". + // + if (r.version == bpkg::version ("0")) { - fail << "build error: " << e; + assert (r.status == result_status::abnormal); + + r.name = tm.name; + r.version = tm.version; } return r; } +catch (const system_error& e) +{ + fail << "build error: " << e << endf; +} extern "C" void handle_signal (int sig) @@ -748,10 +778,15 @@ try tc_num = ops.toolchain_num (); tc_id = ops.toolchain_id (); - if (argc < 2) + + // Controller URLs. + // + if (argc < 2 && !ops.fake_request_specified ()) fail << "controller url expected" << info << "run " << argv[0] << " --help for details"; + strings controllers; + for (int i (1); i != argc; ++i) controllers.push_back (argv[i]); @@ -830,18 +865,15 @@ try if (ops.fake_request_specified ()) { - const path& f (ops.fake_request ()); - task_manifest t (f.string () != "-" - ? parse_manifest (f, "task") - : parse_manifest (cin, "stdin", "task")); - - url = controllers[0]; + auto t (parse_manifest (ops.fake_request (), "task")); tr = task_response_manifest { "fake-session", // Dummy session. string (), // Empty challange. url, // Empty result URL. move (t)}; + + url = "http://example.org"; } else { @@ -893,9 +925,9 @@ try // verify it is one of those we sent). // size_t i (0); - for (const bootstrapped_machine_manifest& m: ms) + for (const machine_header_manifest& m: tq.machines) { - if (m.machine.name == tr.task->machine) + if (m.name == tr.task->machine) break; ++i; diff --git a/bbot/utility b/bbot/utility index 59f6e09..f154a9e 100644 --- a/bbot/utility +++ b/bbot/utility @@ -141,6 +141,8 @@ namespace bbot // Manifest parsing and serialization. // + // For parsing, if path is '-', then read from stdin. + // template T parse_manifest (const path&, const char* what, bool ignore_unknown = true); diff --git a/bbot/utility.txx b/bbot/utility.txx index edd674b..4ee4c6f 100644 --- a/bbot/utility.txx +++ b/bbot/utility.txx @@ -2,6 +2,8 @@ // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd // license : TBC; see accompanying LICENSE file +#include // cin + #include #include @@ -148,6 +150,9 @@ namespace bbot try { + if (f.string () == "-") + return parse_manifest (std::cin, "stdin", what, ignore_unknown); + if (!file_exists (f)) fail << what << " manifest file " << f << " does not exist"; diff --git a/bbot/worker.cli b/bbot/worker.cli index fd88521..61d3f4c 100644 --- a/bbot/worker.cli +++ b/bbot/worker.cli @@ -39,6 +39,11 @@ namespace bbot { "\h|OPTIONS|" + // + // NOTE: when adding new options, consider whether they should be + // propagated from startup to build. + // + bool --help {"Print usage information and exit."} bool --version {"Print version and exit."} diff --git a/bbot/worker.cxx b/bbot/worker.cxx index 0212d26..a2e53b2 100644 --- a/bbot/worker.cxx +++ b/bbot/worker.cxx @@ -182,7 +182,9 @@ build (size_t argc, const char* argv[]) for (const variable& v: tm.config) cfg.push_back (v.unquoted ()); - dir_path dir ("build"); + // Use target (if present) or machine as configuration directory name. + // + dir_path dir (tm.target ? tm.target->string () : tm.machine); r.status |= run_bpkg (trace, r.log, "create", @@ -380,10 +382,27 @@ startup () // Run it. // + strings os; + + if (ops.systemd_daemon ()) + os.push_back ("--systemd-daemon"); + + if (ops.verbose_specified ()) + { + os.push_back ("--verbose"); + os.push_back (to_string (ops.verbose ())); + } + + if (ops.tftp_host_specified ()) + { + os.push_back ("--tftp-host"); + os.push_back (ops.tftp_host ()); + } + // Note that we use the effective (absolute) path instead of recall since // we may have changed the CWD. // - run (trace, pp, tg, argv0.effect_string ()); + run (trace, pp, tg, argv0.effect_string (), os); } catch (const failed&) { diff --git a/doc/manual.cli b/doc/manual.cli index 2b2fc21..b96ddd9 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -494,18 +494,19 @@ the environment executable is an error. Once the environment setup executable is determined, the worker re-executes itself as that executable passing to it as command line arguments the target -name (or empty value if not specified) and the path to the \c{bbot} worker to -be executed once the environment is setup. The environment setup executable is -executed in the build directory as its current working directory. The build +name (or empty value if not specified), the path to the \c{bbot} worker to be +executed once the environment is setup, and any additional options that need +to be propagated to the re-executed worker. The environment setup executable +is executed in the build directory as its current working directory. The build directory contains the build task \c{manifest} file. The environment setup executable sets up the necessary execution environment for example by adjusting \c{PATH} or running a suitable \c{vcvars} batch file. It then re-executes itself as the \c{bbot} worker passing to it as command -line arguments the list of build system modules (\c{}) and the -list of configuration variables (\c{}). The environment setup -executable must execute the \c{bbot} worker in the build directory as the -current working directory. +line arguments (in addition to worker options) the list of build system +modules (\c{}) and the list of configuration variables +(\c{}). The environment setup executable must execute the +\c{bbot} worker in the build directory as the current working directory. The re-executed \c{bbot} worker then proceeds to test the package from the repository by executing the following commands (\c{<>}-values are from the @@ -520,23 +521,30 @@ bpkg -v update bpkg -v test \ -As an example, the following bash script can be used to setup the environment -for building C and C++ packages with GCC 6 on most Linux distributions. +As an example, the following POSIX shell script can be used to setup the +environment for building C and C++ packages with GCC 6 on most Linux +distributions. \ -#! /usr/bin/env bash +#!/bin/sh -# $1 - target -# $2 - bbot executable +# Environment setup script for C/C++ compilation with GCC 6. +# +# $1 - target +# $2 - bbot executable +# $3+ - bbot options -trap \"exit 1\" ERR +set -e # Exit on errors. -if [ -n \"$1\" ]; then - echo \"unknown target $1\" 1>&2 +t=\"$1\" +shift + +if test -n \"$t\"; then + echo \"unknown target: $t\" 1>&2 exit 1 fi -exec \"$2\" cc config.c=gcc-6 +exec \"$@\" cc config.c=gcc-6 config.cxx=g++-6 \ \h#arch-controller|Controller Logic| diff --git a/tests/agent/testscript b/tests/agent/testscript index e864bf2..6e9d319 100644 --- a/tests/agent/testscript +++ b/tests/agent/testscript @@ -137,7 +137,7 @@ rm = $src_base/btrfs-rmdir /build/machines machine: windows-msvc EOI %trace: enumerate_machines:\\.*%* - error: task from $u for unknown machine windows-msvc + error: task from http://example.org for unknown machine windows-msvc EOE : result @@ -159,30 +159,3 @@ rm = $src_base/btrfs-rmdir /build/machines -$rm } - -#\ -: bootstrap -: -{ - test.options += --dump-machines - - m = /build/machines/default/linux-gcc - u = https://example.org/?dummy - - +$cp - - ln -T -s linux-gcc-1.0 $m/linux-gcc-1 - - : bootstrap - : - $* $u >>EOO 2>>EOE #2>>~"%EOE%d" - : 1 - id: linux-gcc-1.0 - name: linux-gcc - summary: Linux with GCC - EOO - EOE - - #-$rm -} -#\ diff --git a/tests/integration/buildfile b/tests/integration/buildfile new file mode 100644 index 0000000..f054694 --- /dev/null +++ b/tests/integration/buildfile @@ -0,0 +1,8 @@ +# file : tests/integration/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +./: ../../bbot/exe{bbot-agent bbot-worker} test{testscript} file{tftp-map} +dir{./}: test = ../../bbot/exe{bbot-agent} + +include ../../bbot/ diff --git a/tests/integration/testscript b/tests/integration/testscript new file mode 100644 index 0000000..f43bf02 --- /dev/null +++ b/tests/integration/testscript @@ -0,0 +1,98 @@ +# file : tests/integration/testscript +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +#\ +# Requirement: +# +# 1. tftpd-hpa installed (assumed in /usr/sbin/in.tftpd) +# +# 2. b, bpkg, curl executables in PATH. +# +# TFTP server (tftpd-hpa) setup: from the test out_base, run (sudo is required +# for --secure/chroot): +# + +sudo /usr/sbin/in.tftpd \ + --foreground \ + --address 127.0.0.1:55123 \ + --user "$(whoami)" \ + --permissive \ + --create \ + --secure \ + --map-file tftp-map \ + "$(pwd)" + +#\ + +machine = linux-gcc +c = gcc +cxx = g++ + +# Where we get the task and what we do with the result can be configured +# independently: +# +# - We can poll a controller for a task by specifying its URL or we can +# read a task manifest from a file (--fake-request). +# +# - We can send the result back to the controller or we can dump the result +# to stdout (--dump-result). +# + +#\ +wait=5s +controller = https://stage.build2.org/?build-task +#\ + +wait=1s +controller = --fake-request ../task --dump-result + +pkg = hello +ver = 1.0.0 +rep = https://build2.org/pkg/1/stage/stable +rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94 ++cat <<"EOI" >=task + : 1 + name: $pkg + version: $ver + repository: $rep + trust: $rfp + machine: $machine + EOI + +# +# +tftp = 127.0.0.1:55123 + +a = $0 ++ sed -e 's/-agent$/-worker/' <"$0" | set w + +: agent +: +{ + cat <<"EOI" >=machine-header; + : 1 + id: $machine-1.0 + name: $machine + summary: The $machine fake machine + EOI + $a --verbose 3 --tftp $~ --fake-machine machine-header $controller \ + &build/*** >| 2>| +} + +: worker +: +{ + cat <<"EOI" >=default; + #!/bin/sh + + t="\$1" + shift + + exec "\$@" cc config.c=$c config.cxx=$cxx + EOI + chmod ugo+x default; + sleep $wait; + $w --verbose 3 --startup --tftp-host $tftp --environment \ + &$machine/*** &manifest $~ 2>| +} diff --git a/tests/integration/tftp-map b/tests/integration/tftp-map new file mode 100644 index 0000000..837794d --- /dev/null +++ b/tests/integration/tftp-map @@ -0,0 +1,11 @@ +# file : tests/integration/tftp-map +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +# Test working directories relative to out_base: +# +# agent: test/agent/ +# worker: test/worker/ +# +Gr ^/?(.+)$ /test/agent/build/default/get/\1 +Pr ^/?(.+)$ /test/agent/build/default/put/\1 diff --git a/tests/worker/build.test b/tests/worker/build.test index adedccc..3be9150 100644 --- a/tests/worker/build.test +++ b/tests/worker/build.test @@ -27,8 +27,8 @@ cat <<"EOI" >=manifest; target: x86_64-linux-gnu config: config.cc.coptions=-O3 EOI -$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; - trace: build: bpkg -v create -d build --wipe config.cc.coptions=-O3 cc +$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE"; + trace: build: bpkg -v create -d x86_64-linux-gnu --wipe config.cc.coptions=-O3 cc trace: build: bpkg -v add $rep trace: build: bpkg -v fetch --trust $rfp --trust-no trace: build: bpkg -v build --configure-only --yes $pkg/$ver @@ -78,8 +78,8 @@ cat <<"EOI" >=manifest; machine: linux-gcc target: x86_64-linux-gnu EOI -$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; - trace: build: bpkg -v create -d build --wipe cc +$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE"; + trace: build: bpkg -v create -d x86_64-linux-gnu --wipe cc trace: build: bpkg -v add $rep trace: build: bpkg -v fetch --trust $rfp --trust-no trace: build: bpkg -v build --configure-only --yes bogus/1.2.3 @@ -109,8 +109,8 @@ cat <<"EOI" >=manifest; target: x86_64-linux-gnu config: config.cc.loptions=-lbogus EOI -$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; - trace: build: bpkg -v create -d build --wipe config.cc.loptions=-lbogus cc +$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE"; + trace: build: bpkg -v create -d x86_64-linux-gnu --wipe config.cc.loptions=-lbogus cc trace: build: bpkg -v add $rep trace: build: bpkg -v fetch --trust $rfp --trust-no trace: build: bpkg -v build --configure-only --yes $pkg/$ver -- cgit v1.1