aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bbot/agent5
-rw-r--r--bbot/agent.cli67
-rw-r--r--bbot/agent.cxx473
-rw-r--r--bbot/bbot-agent@.service14
-rw-r--r--bbot/bootstrap-manifest50
-rw-r--r--bbot/bootstrap-manifest.cxx155
-rw-r--r--bbot/buildfile4
-rw-r--r--bbot/machine-manifest54
-rw-r--r--bbot/machine-manifest.cxx154
-rw-r--r--bbot/machine.cxx2
-rw-r--r--bbot/tftp.cxx2
-rw-r--r--bbot/types10
-rw-r--r--bbot/utility53
-rw-r--r--bbot/utility.cxx2
-rw-r--r--bbot/utility.txx91
-rw-r--r--bbot/worker.cli54
-rw-r--r--bbot/worker.cxx455
-rw-r--r--doc/manual.cli35
-rw-r--r--tests/agent/testscript119
-rw-r--r--tests/worker/bootstrap.test13
-rw-r--r--tests/worker/build.test133
-rw-r--r--tests/worker/buildfile26
-rw-r--r--tests/worker/startup.test101
-rw-r--r--unit-tests/bootstrap-manifest/driver.cxx1
24 files changed, 1706 insertions, 367 deletions
diff --git a/bbot/agent b/bbot/agent
index c53c5c9..14d2995 100644
--- a/bbot/agent
+++ b/bbot/agent
@@ -19,9 +19,12 @@ namespace bbot
extern const string bs_prot; // Bootstrap protocol version.
extern string tc_name; // Toolchain name.
- extern string tc_num; // Toolchain number.
+ extern size_t tc_num; // Toolchain number.
extern string tc_id; // Toolchain id.
+ extern strings controllers; // Controller URLs.
+
+ extern string hname; // Our host name.
extern uid_t uid; // Our effective user id.
extern string uname; // Our effective user name.
diff --git a/bbot/agent.cli b/bbot/agent.cli
index 562860f..3e02807 100644
--- a/bbot/agent.cli
+++ b/bbot/agent.cli
@@ -11,22 +11,19 @@ include <bbot/common.cli>;
namespace bbot
{
{
- "<options> <name> <num> <id> ",
+ "<options> <url>",
"
\h|SYNOPSIS|
\cb{bbot-agent --help}\n
\cb{bbot-agent --version}\n
- \c{\b{bbot-agent} [<options>] <name> <num> <id>}
+ \c{\b{bbot-agent} [<options>] <url>...}
\h|DESCRIPTION|
\cb{bbot-agent} @@ TODO.
- The <name> argument is the toolchain name, <id> \- the toolchain id,
- and <num> \- the toolchain number in this deployment.
-
Note that on termination \cb{bbot-agent} may leave a working machine
snapshot behind. It is expected that the caller (normally Build OS
monitor) cleans them up before restarting the agent.
@@ -45,6 +42,26 @@ namespace bbot
"Start as a simple systemd daemon."
}
+ string --toolchain-name = "default"
+ {
+ "<str>",
+ "Toolchain name, \cb{default} by default."
+ }
+
+ size_t --toolchain-num = 1
+ {
+ "<num>",
+ "Toolchain number, 1 by default."
+ }
+
+ string --toolchain-id
+ {
+ "<str>",
+ "Toolchain id. If unspecified or empty, then no re-bootstrapping on
+ toolchain changes will be performed (which is primarily useful for
+ testing)."
+ }
+
size_t --cpu = 1
{
"<num>",
@@ -60,8 +77,13 @@ namespace bbot
dir_path --machines = "/build/machines/"
{
"<dir>",
- "The location of the build machines with \cb{/build/machines/} being
- the default."
+ "The location of the build machines, \cb{/build/machines/} by default."
+ }
+
+ dir_path --tftp = "/build/tftp/"
+ {
+ "<dir>",
+ "The location of the TFTP server root, \cb{/build/tftp/} by default."
}
size_t --bootstrap-timeout = 600
@@ -78,6 +100,13 @@ namespace bbot
minutes) by default."
}
+ size_t --request-timeout = 300
+ {
+ "<sec>",
+ "Maximum number of seconds to wait for controller request completion,
+ 300 (5 minutes) by default."
+ }
+
uint16_t --verbose = 1
{
"<level>",
@@ -89,15 +118,37 @@ namespace bbot
//
bool --dump-machines
{
- "Dump available machines to \c{stdout}, (re)-bootstrapping any if
+ "Dump the available machines to \cb{stdout}, (re)-bootstrapping any if
necessary, and exit."
}
+ bool --dump-task
+ {
+ "Dump the received build task to \cb{stdout} and exit."
+ }
+
+ bool --dump-result
+ {
+ "Dump the obtained build result to \cb{stdout} and exit."
+ }
+
bool --fake-bootstrap
{
"Fake the machine bootstrap process by creating the expected bootstrapped
machine manifest."
}
+
+ bool --fake-build
+ {
+ "Fake the package building process by creating the aborted build result."
+ }
+
+ path --fake-request
+ {
+ "<file>",
+ "Fake the task request process by reading the task manifest from <file>
+ (or \cb{stdin} if <file> is '\cb{-}')."
+ }
};
"
diff --git a/bbot/agent.cxx b/bbot/agent.cxx
index 634a94d..eed49d6 100644
--- a/bbot/agent.cxx
+++ b/bbot/agent.cxx
@@ -28,6 +28,7 @@
#include <bbot/tftp>
#include <bbot/machine>
+#include <bbot/machine-manifest>
#include <bbot/bootstrap-manifest>
using namespace std;
@@ -41,9 +42,12 @@ namespace bbot
const string bs_prot ("1");
string tc_name;
- string tc_num;
+ size_t tc_num;
string tc_id;
+ strings controllers;
+
+ string hname;
uid_t uid;
string uname;
@@ -84,9 +88,9 @@ namespace bbot
//
template <typename... A>
inline void
-btrfs (tracer& t, A&&... a)
+run_btrfs (tracer& t, A&&... a)
{
- if (verb >= 3)
+ if (verb >= 4)
run_io (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...);
else
run_io (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
@@ -96,7 +100,7 @@ template <typename... A>
inline butl::process_exit::code_type
btrfs_exit (tracer& t, A&&... a)
{
- return verb >= 3
+ return verb >= 4
? run_io_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...)
: run_io_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
}
@@ -114,7 +118,7 @@ bootstrap_machine (const dir_path& md,
bootstrapped_machine_manifest r {
mm,
- toolchain_manifest {tc_id},
+ toolchain_manifest {tc_id.empty () ? "bogus" : tc_id},
bootstrap_manifest {
bootstrap_manifest::versions_type {
{"bbot", BBOT_VERSION},
@@ -134,12 +138,12 @@ bootstrap_machine (const dir_path& md,
{
string br ("br1"); // Using private bridge for now.
- // Start the TFTP server (server chroot is /build/tftp). Map:
+ // Start the TFTP server (server chroot is --tftp). Map:
//
- // GET requests to /build/tftp/toolchain/<name>/*
- // PUT requests to /build/tftp/bootstrap/<name>/*
+ // GET requests to .../toolchain/<name>/*
+ // PUT requests to .../bootstrap/<name>/*
//
- auto_rmdir arm (dir_path ("/build/tftp/bootstrap/" + tc_name));
+ auto_rmdir arm ((dir_path (ops.tftp ()) /= "bootstrap") /= tc_name);
try_mkdir_p (arm.path ());
// Bootstrap result manifest.
@@ -150,7 +154,7 @@ bootstrap_machine (const dir_path& md,
tftp_server tftpd ("Gr ^/?(.+)$ /toolchain/" + tc_name + "/\\1\n" +
"Pr ^/?(.+)$ /bootstrap/" + tc_name + "/\\1\n");
- l2 ([&]{trace << "tftp server on port " << tftpd.port ();});
+ l3 ([&]{trace << "tftp server on port " << tftpd.port ();});
// Start the machine.
//
@@ -205,7 +209,7 @@ bootstrap_machine (const dir_path& md,
if (!tftpd.serve ((to = startup_to)))
return soft_fail ("bootstrap startup timeout");
- l2 ([&]{trace << "completed startup in " << startup_to - to << "s";});
+ l3 ([&]{trace << "completed startup in " << startup_to - to << "s";});
// Next the bootstrap process may download additional toolchain
// archives, build things, and then upload the result manifest. So on
@@ -217,50 +221,45 @@ bootstrap_machine (const dir_path& md,
if (to == 0)
return soft_fail ("bootstrap timeout");
- l2 ([&]{trace << "completed bootstrap in " << bootstrap_to - to << "s";});
+ l3 ([&]{trace << "completed bootstrap in " << bootstrap_to - to << "s";});
// Shut the machine down cleanly.
//
if (!m->shutdown ((to = shutdown_to)))
return soft_fail ("bootstrap shutdown timeout");
- l2 ([&]{trace << "completed shutdown in " << shutdown_to - to << "s";});
+ l3 ([&]{trace << "completed shutdown in " << shutdown_to - to << "s";});
}
// Parse the result manifest.
//
- try
- {
- r.bootstrap = parse_manifest<bootstrap_manifest> (mf, "bootstrap");
- }
- catch (const failed&)
- {
- error << "invalid bootstrap manifest for machine " << md;
- return nullopt;
- }
+ r.bootstrap = parse_manifest<bootstrap_manifest> (mf, "bootstrap");
r.machine.mac = m->mac; // Save the MAC address.
}
catch (const system_error& e)
{
- fail << "tftp server error: " << e;
+ fail << "bootstrap error: " << e;
}
serialize_manifest (r, md / "manifest", "bootstrapped machine");
return r;
}
-static machine_header_manifests
-enumerate_machines (const dir_path& rd)
+// Return available machines and their directories as a parallel array.
+//
+static pair<bootstrapped_machine_manifests, dir_paths>
+enumerate_machines (const dir_path& machines)
try
{
tracer trace ("enumerate_machines");
- machine_header_manifests r;
+ bootstrapped_machine_manifests rm;
+ dir_paths rd;
// The first level are machine volumes.
//
- for (const dir_entry& ve: dir_iterator (rd))
+ for (const dir_entry& ve: dir_iterator (machines))
{
const string vn (ve.path ().string ());
@@ -269,7 +268,7 @@ try
if (ve.type () != entry_type::directory || vn[0] == '.')
continue;
- const dir_path vd (dir_path (rd) /= vn);
+ const dir_path vd (dir_path (machines) /= vn);
// Inside we have machines.
//
@@ -302,14 +301,14 @@ try
// any) and see if we need to re-bootstrap. If so, use the snapshot
// as a starting point. Rename to bootstrapped at the end (atomic).
//
- const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
- const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<too...>
+ dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
+ dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<toolchain>
bool te (dir_exists (tp));
auto delete_t = [&tp, &trace] ()
{
- btrfs (trace, "property", "set", "-ts", tp, "ro", "false");
- btrfs (trace, "subvolume", "delete", tp);
+ run_btrfs (trace, "property", "set", "-ts", tp, "ro", "false");
+ run_btrfs (trace, "subvolume", "delete", tp);
};
for (size_t retry (0);; ++retry)
@@ -355,14 +354,14 @@ try
if (te)
delete_t ();
- l2 ([&]{trace << "skipping " << md << ": no subvolume link";});
+ l3 ([&]{trace << "skipping " << md << ": no subvolume link";});
break;
}
// <name>-<toolchain>-<xxx>
//
- const dir_path xp (dir_path (md) /=
- path::traits::temp_name (mn + '-' + tc_name));
+ const dir_path xp (
+ dir_path (md) /= path::traits::temp_name (mn + '-' + tc_name));
if (btrfs_exit (trace, "subvolume", "snapshot", sp, xp) != 0)
{
@@ -422,13 +421,13 @@ try
if (bmm->machine.id != mm.id)
{
- l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";});
+ l3 ([&]{trace << "re-bootstrapping " << tp << ": new machine";});
te = false;
}
- if (bmm->toolchain.id != tc_id)
+ if (!tc_id.empty () && bmm->toolchain.id != tc_id)
{
- l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";});
+ l3 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";});
te = false;
}
@@ -436,13 +435,13 @@ try
{
if (i < 0)
{
- l2 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";});
+ l3 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";});
te = false;
}
else
{
- l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
- btrfs (trace, "subvolume", "delete", xp);
+ l3 ([&]{trace << "ignoring " << tp << ": old bbot";});
+ run_btrfs (trace, "subvolume", "delete", xp);
break;
}
}
@@ -451,7 +450,7 @@ try
delete_t ();
}
else
- l2 ([&]{trace << "bootstrapping " << tp;});
+ l3 ([&]{trace << "bootstrapping " << tp;});
if (!te)
{
@@ -463,8 +462,8 @@ try
if (!bmm)
{
- l2 ([&]{trace << "ignoring " << tp << ": failed to bootstrap";});
- btrfs (trace, "subvolume", "delete", xp);
+ l3 ([&]{trace << "ignoring " << tp << ": failed to bootstrap";});
+ run_btrfs (trace, "subvolume", "delete", xp);
break;
}
@@ -477,25 +476,25 @@ try
fail << "unable to rename " << xp << " to " << tp;
}
- // Check the boostrapped bbot version as above and ignore this
+ l2 ([&]{trace << "bootstrapped " << bmm->machine.name;});
+
+ // Check the bootstrapped bbot version as above and ignore this
// machine if it's newer than us.
//
if (int i = compare_bbot (bmm->bootstrap))
{
assert (i > 0);
- l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
+ l3 ([&]{trace << "ignoring " << tp << ": old bbot";});
break;
}
}
else
- btrfs (trace, "subvolume", "delete", xp);
+ run_btrfs (trace, "subvolume", "delete", xp);
- // Add the machine to the list.
+ // Add the machine to the lists.
//
- r.push_back (
- machine_header_manifest (move (mm.id),
- move (mm.name),
- move (mm.summary)));
+ rm.push_back (move (*bmm));
+ rd.push_back (move (tp));
break;
}
@@ -507,11 +506,161 @@ try
}
}
- return r;
+ return make_pair (move (rm), move (rd));
}
catch (const system_error& e)
{
- fail << "unable to iterate over " << rd << ": " << e << endf;
+ fail << "unable to iterate over " << machines << ": " << e << endf;
+}
+
+static result_manifest
+perform_task (const dir_path& md,
+ const bootstrapped_machine_manifest& mm,
+ const task_manifest& tm)
+{
+ tracer trace ("perform_task");
+
+ result_manifest r {
+ tm.name,
+ tm.version,
+ result_status::abort,
+ operation_results {}};
+
+ if (ops.fake_build ())
+ return r;
+
+ // The overall plan is as follows:
+ //
+ // 1. Snapshot the (bootstrapped) machine.
+ //
+ // 2. Save the task manifest to the TFTP directory (to be accessed by the
+ // worker).
+ //
+ // 3. Start the TFTP server and the machine.
+ //
+ // 4. Serve TFTP requests while watching out for the result manifest.
+ //
+ // 5. Clean up (force the machine down and delete the snapshot).
+ //
+ try
+ {
+ // <name>-<toolchain>-<xxx>
+ //
+ const dir_path xp (
+ md.directory () /= path::traits::temp_name (md.leaf ().string ()));
+
+ run_btrfs (trace, "subvolume", "snapshot", md, xp);
+
+ string br ("br1"); // Using private bridge for now.
+
+ // Start the TFTP server (server chroot is --tftp). Map:
+ //
+ // GET requests to .../build/<name>/get/*
+ // PUT requests to .../build/<name>/put/*
+ //
+ auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name);
+
+ dir_path gd (dir_path (arm.path ()) /= "get");
+ dir_path pd (dir_path (arm.path ()) /= "put");
+
+ try_mkdir_p (gd);
+ try_mkdir_p (pd);
+
+ path tf (gd / "manifest"); // Task manifest file.
+ path rf (pd / "manifest"); // Result manifest file.
+
+ serialize_manifest (tm, tf, "task");
+
+ tftp_server tftpd ("Gr ^/?(.+)$ /build/" + tc_name + "/get/\\1\n" +
+ "Pr ^/?(.+)$ /build/" + tc_name + "/put/\\1\n");
+
+ l3 ([&]{trace << "tftp server on port " << tftpd.port ();});
+
+ // Start the machine.
+ //
+ unique_ptr<machine> m (
+ start_machine (xp,
+ mm.machine,
+ mm.machine.mac,
+ br,
+ tftpd.port ()));
+
+ // Note: the machine handling logic is similar to bootstrap.
+ //
+ {
+ auto mg (
+ make_exception_guard (
+ [&m, &xp] ()
+ {
+ info << "trying to force machine " << xp << " down";
+ try {m->forcedown ();} catch (const failed&) {}
+ }));
+
+ auto soft_fail = [&xp, &m, &r] (const char* msg)
+ {
+ {
+ diag_record dr (error);
+ dr << msg << " for machine " << xp << ", suspending";
+ m->print_info (dr);
+ }
+ m->suspend ();
+ m->wait ();
+ return r;
+ };
+
+ // The first request should be the task manifest download. Wait for up
+ // to 60 seconds for that to arrive. In a sense we use it as an
+ // indication that the machine has booted and the worker process has
+ // started.
+ //
+ size_t to;
+ const size_t startup_to (60);
+ const size_t build_to (ops.build_timeout ());
+
+ if (!tftpd.serve ((to = startup_to)))
+ return soft_fail ("build startup timeout");
+
+ l3 ([&]{trace << "completed startup in " << startup_to - to << "s";});
+
+ // Next the worker builds things and then uploads the result manifest.
+ // So on our side we serve TFTP requests while checking for the manifest
+ // file.
+ //
+ for (to = build_to; to != 0 && !file_exists (rf); tftpd.serve (to)) ;
+
+ if (to == 0)
+ return soft_fail ("build timeout");
+
+ l3 ([&]{trace << "completed build in " << build_to - to << "s";});
+
+ // Force the machine down (there is no need wasting time on clean
+ // shutdown since the next step is to drop the snapshot).
+ //
+ m->forcedown ();
+ }
+
+ run_btrfs (trace, "subvolume", "delete", xp);
+
+ // Parse the result manifest.
+ //
+ r = parse_manifest<result_manifest> (rf, "result");
+
+ // Update package name/version if the returned value as "unknown".
+ //
+ if (r.version == bpkg::version ("0"))
+ {
+ assert (r.status == result_status::abnormal);
+
+ r.name = tm.name;
+ r.version = tm.version;
+ }
+ }
+ catch (const system_error& e)
+ {
+ fail << "build error: " << e;
+ }
+
+ return r;
}
extern "C" void
@@ -538,9 +687,11 @@ try
verb = ops.verbose ();
- uid = getuid ();
- uname = getpwuid (uid)->pw_name;
-
+ // Note that unlike other projects, here we distinguish between fail
+ // (critical error, agent terminates) and error (non-fatal error, agent
+ // continues to run). This means we should be careful not using fail
+ // to report normal errors and vice-versa.
+ //
if (ops.systemd_daemon ())
{
// Map to systemd severity prefixes (see sd-daemon(3) for details). Note
@@ -554,7 +705,7 @@ try
info.indent_ =
text.indent_ = systemd_indent;
- fail.type_ = "<3>";
+ fail.type_ = "<2>";
error.type_ = "<3>";
warn.type_ = "<4>";
info.type_ = "<6>";
@@ -568,6 +719,19 @@ try
tracer trace ("main");
+ uid = getuid ();
+ uname = getpwuid (uid)->pw_name;
+
+ {
+ char buf[HOST_NAME_MAX + 1];
+
+ if (gethostname (buf, sizeof (buf)) == -1)
+ fail << "unable to obtain hostname: "
+ << system_error (errno, generic_category ()); // Sanitize.
+
+ hname = buf;
+ }
+
// On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if
// the pipe reading end is closed. Note that by default this signal
// terminates a process. Also note that there is no way to disable this
@@ -602,13 +766,16 @@ try
return p.wait () ? 0 : 1;
}
- if (argc != 4)
- fail << "toolchain name/id/num excected" <<
+ tc_name = ops.toolchain_name ();
+ tc_num = ops.toolchain_num ();
+ tc_id = ops.toolchain_id ();
+
+ if (argc < 2)
+ fail << "controller url expected" <<
info << "run " << argv[0] << " --help for details";
- tc_name = argv[1];
- tc_num = argv[2];
- tc_id = argv[3];
+ for (int i (1); i != argc; ++i)
+ controllers.push_back (argv[i]);
// Handle SIGHUP and SIGTERM.
//
@@ -619,26 +786,192 @@ try
// The work loop. The steps we go through are:
//
- // 1. Enumerate the available machines, (re-)bootstrapping any of necessary.
+ // 1. Enumerate the available machines, (re-)bootstrapping any if necessary.
//
// 2. Poll controller(s) for build tasks.
//
- // 3. If no build tasks are available, go to #1 after sleeping a bit.
+ // 3. If no build tasks are available, go to #1 (after sleeping a bit).
//
// 4. If a build task is returned, do it, upload the result, and go to #1
- // immediately.
+ // (immediately).
//
- for (unsigned int s; (s = 60); sleep (s))
+ for (bool sleep (false);; ::sleep (sleep ? 60 : 0), sleep = false)
{
- machine_header_manifests mms (enumerate_machines (ops.machines ()));
+ // Enumerate the machines.
+ //
+ auto mp (enumerate_machines (ops.machines ()));
+ bootstrapped_machine_manifests& ms (mp.first);
+ dir_paths& ds (mp.second);
+
+ // Prepare task request.
+ //
+ // @@ TODO: key fingerprint.
+ //
+ task_request_manifest tq {hname, "", machine_header_manifests {}};
+
+ for (const bootstrapped_machine_manifest& m: ms)
+ tq.machines.emplace_back (m.machine.id,
+ m.machine.name,
+ m.machine.summary);
if (ops.dump_machines ())
{
- for (const machine_header_manifest& mm: mms)
- serialize_manifest (mm, cout, "stdout", "machine manifest");
+ for (const machine_header_manifest& m: tq.machines)
+ serialize_manifest (m, cout, "stdout", "machine");
return 0;
}
+
+ if (tq.machines.empty ())
+ {
+ warn << "no build machines for toolchain " << tc_name;
+ sleep = true;
+ continue;
+ }
+
+ // Send task requests.
+ //
+ //
+ string url;
+ task_response_manifest tr;
+
+ if (ops.fake_request_specified ())
+ {
+ const path& f (ops.fake_request ());
+ task_manifest t (f.string () != "-"
+ ? parse_manifest<task_manifest> (f, "task")
+ : parse_manifest<task_manifest> (cin, "stdin", "task"));
+
+ url = controllers[0];
+
+ tr = task_response_manifest {
+ "fake-session", // Dummy session.
+ string (), // Empty challange.
+ url, // Empty result URL.
+ move (t)};
+ }
+ else
+ {
+ for (const string& u: controllers)
+ {
+ try
+ {
+ http_curl c (trace,
+ path ("-"),
+ path ("-"),
+ curl::post,
+ u,
+ "--header", "Content-Type: text/manifest",
+ "--max-time", ops.request_timeout ());
+
+ serialize_manifest (tq, c.out, u, "task request");
+ c.out.close ();
+
+ tr = parse_manifest<task_response_manifest> (
+ c.in, u, "task response");
+ c.in.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to request task from " << u << ": " << e;
+ continue;
+ }
+
+ if (!tr.session.empty ()) // Got a task.
+ {
+ url = u;
+ break;
+ }
+ }
+ }
+
+ if (tr.session.empty ()) // No task from any of the controllers.
+ {
+ sleep = true;
+ continue;
+ }
+
+ // We have a build task.
+ //
+ // First find the index of the machine we were asked to use (and also
+ // verify it is one of those we sent).
+ //
+ size_t i (0);
+ for (const bootstrapped_machine_manifest& m: ms)
+ {
+ if (m.machine.name == tr.task->machine)
+ break;
+
+ ++i;
+ }
+
+ if (i == ms.size ())
+ {
+ error << "task from " << url << " for unknown machine "
+ << tr.task->machine;
+
+ if (ops.dump_task ())
+ return 0;
+
+ continue;
+ }
+
+ const task_manifest& t (*tr.task);
+
+ if (ops.dump_task ())
+ {
+ serialize_manifest (t, cout, "stdout", "task");
+ return 0;
+ }
+
+ const dir_path& d (ds[i]); // The -<toolchain> directory.
+ const bootstrapped_machine_manifest& m (ms[i]);
+
+ result_manifest r (perform_task (d, m, t));
+
+ if (ops.dump_result ())
+ {
+ serialize_manifest (r, cout, "stdout", "result");
+ return 0;
+ }
+
+ // Upload the result.
+ //
+ // @@ TODO challange
+ //
+ result_request_manifest rq {tr.session, "", move (r)};
+ {
+ const string& u (*tr.result_url);
+
+ try
+ {
+ http_curl c (trace,
+ path ("-"),
+ nullfd, // Not expecting any data in response.
+ curl::post,
+ u,
+ "--header", "Content-Type: text/manifest",
+ "--max-time", ops.request_timeout ());
+
+ serialize_manifest (rq, c.out, u, "task request");
+ c.out.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to upload result to " << u << ": " << e;
+ continue;
+ }
+ }
+
+ l2 ([&]{trace << "built " << t.name << '/' << t.version << " "
+ << "on " << t.machine << " "
+ << "for " << url;});
}
}
catch (const failed&)
diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service
index af760b3..5f160cf 100644
--- a/bbot/bbot-agent@.service
+++ b/bbot/bbot-agent@.service
@@ -12,9 +12,13 @@ Environment=RAM=1048576
Environment=BOOTSTRAP_TIMEOUT=600
Environment=BUILD_TIMEOUT=1800
+Environment=REQUEST_TIMEOUT=300
-Environment=TOOLCHAIN_ID=123abc
+Environment=TOOLCHAIN_NAME=%i
Environment=TOOLCHAIN_NUM=1
+Environment=TOOLCHAIN_ID=
+
+Environment="CONTROLLER_URL=https://example.org/?build-task"
ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \
--verbose ${VERBOSE} \
@@ -22,9 +26,11 @@ ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \
--ram ${RAM} \
--bootstrap-timeout ${BOOTSTRAP_TIMEOUT} \
--build-timeout ${BUILD_TIMEOUT} \
- %i \
- ${TOOLCHAIN_NUM} \
- ${TOOLCHAIN_ID}
+ --request-timeout ${REQUEST_TIMEOUT} \
+ --toolchain-name ${TOOLCHAIN_NAME} \
+ --toolchain-num ${TOOLCHAIN_NUM} \
+ --toolchain-id ${TOOLCHAIN_ID} \
+ $CONTROLLER_URL
User=build
Group=build
diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest
index 35feaba..94a2a24 100644
--- a/bbot/bootstrap-manifest
+++ b/bbot/bootstrap-manifest
@@ -12,34 +12,8 @@
#include <bbot/types>
#include <bbot/utility>
-#include <bbot/machine-manifest>
-
namespace bbot
{
- // Toolchain manifest.
- //
- class toolchain_manifest
- {
- public:
-
- // Toolchain id (SHAXXX).
- //
- string id;
-
- explicit
- toolchain_manifest (string i): id (i) {}
-
- public:
- toolchain_manifest () = default; // VC export.
- toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false);
- toolchain_manifest (butl::manifest_parser&,
- butl::manifest_name_value start,
- bool ignore_unknown = false);
-
- void
- serialize (butl::manifest_serializer&) const;
- };
-
// Bootstrap result manifest. Uploaded by the worker to the agent's TFTP
// server.
//
@@ -71,30 +45,6 @@ namespace bbot
void
serialize (butl::manifest_serializer&) const;
};
-
- // The manifest stored in <name>-<toolchain>/ consists of the machine
- // manifest (original), toolchain manifest, and bootstrap manifest.
- //
- class bootstrapped_machine_manifest
- {
- public:
- machine_manifest machine;
- toolchain_manifest toolchain;
- bootstrap_manifest bootstrap;
-
- bootstrapped_machine_manifest (machine_manifest m,
- toolchain_manifest t,
- bootstrap_manifest b)
- : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {}
-
- public:
- bootstrapped_machine_manifest () = default; // VC export.
- bootstrapped_machine_manifest (butl::manifest_parser&,
- bool ignore_unknown = false);
-
- void
- serialize (butl::manifest_serializer&) const;
- };
}
#endif // BBOT_BOOTSTRAP_MANIFEST
diff --git a/bbot/bootstrap-manifest.cxx b/bbot/bootstrap-manifest.cxx
index d08322a..6a0ff8d 100644
--- a/bbot/bootstrap-manifest.cxx
+++ b/bbot/bootstrap-manifest.cxx
@@ -17,78 +17,6 @@ namespace bbot
using serialization = manifest_serialization;
using name_value = manifest_name_value;
- // toolchain_manifest
- //
- toolchain_manifest::
- toolchain_manifest (parser& p, bool iu)
- : toolchain_manifest (p, p.next (), iu)
- {
- // Make sure this is the end.
- //
- name_value nv (p.next ());
- if (!nv.empty ())
- throw parsing (p.name (), nv.name_line, nv.name_column,
- "single toolchain manifest expected");
- }
-
- toolchain_manifest::
- toolchain_manifest (parser& p, name_value nv, bool iu)
- {
- auto bad_name = [&p, &nv] (const string& d)
- {
- throw parsing (p.name (), nv.name_line, nv.name_column, d);
- };
-
- auto bad_value = [&p, &nv] (const string& d)
- {
- throw parsing (p.name (), nv.value_line, nv.value_column, d);
- };
-
- // Make sure this is the start and we support the version.
- //
- if (!nv.name.empty ())
- bad_name ("start of toolchain manifest expected");
-
- if (nv.value != "1")
- bad_value ("unsupported format version");
-
- // Parse the toolchain manifest.
- //
- for (nv = p.next (); !nv.empty (); nv = p.next ())
- {
- string& n (nv.name);
- string& v (nv.value);
-
- if (n == "id")
- {
- if (!id.empty ())
- bad_name ("toolchain id redefinition");
-
- if (v.empty ())
- bad_value ("empty toolchain id");
-
- id = move (v);
- }
- else if (!iu)
- bad_name ("unknown name '" + n + "' in toolchain manifest");
- }
-
- // Verify all non-optional values were specified.
- //
- if (id.empty ())
- bad_value ("no toolchain id specified");
- }
-
- void toolchain_manifest::
- serialize (serializer& s) const
- {
- // @@ Should we check that all non-optional values are specified?
- //
- s.next ("", "1"); // Start of manifest.
- s.next ("id", id);
- s.next ("", ""); // End of manifest.
- }
-
// bootstrap_manifest
//
bootstrap_manifest::
@@ -181,87 +109,4 @@ namespace bbot
s.next ("", ""); // End of manifest.
}
-
- // bootstrapped_machine_manifest
- //
- bootstrapped_machine_manifest::
- bootstrapped_machine_manifest (parser& p, bool iu)
- {
- name_value nv (p.next ());
-
- auto bad_name = [&p, &nv] (const string& d)
- {
- throw parsing (p.name (), nv.name_line, nv.name_column, d);
- };
-
- auto bad_value = [&p, &nv] (const string& d)
- {
- throw parsing (p.name (), nv.value_line, nv.value_column, d);
- };
-
- // Make sure this is the start and we support the version.
- //
- if (!nv.name.empty ())
- bad_name ("start of bootstrapped machine manifest expected");
-
- if (nv.value != "1")
- bad_value ("unsupported format version");
-
- // Parse the bootstrapped machine manifest. Currently there is no values
- // expected.
- //
- for (nv = p.next (); !nv.empty (); nv = p.next ())
- {
- if (!iu)
- bad_name ("unknown name '" + nv.name +
- "' in bootstrapped machine manifest");
- }
-
- nv = p.next ();
- if (nv.empty ())
- bad_value ("machine manifest expected");
-
- machine = machine_manifest (p, nv, iu);
-
- if (!machine.mac)
- bad_name ("mac address must be present in machine manifest");
-
- nv = p.next ();
- if (nv.empty ())
- bad_value ("toolchain manifest expected");
-
- toolchain = toolchain_manifest (p, nv, iu);
-
- nv = p.next ();
- if (nv.empty ())
- bad_value ("bootstrap manifest expected");
-
- bootstrap = bootstrap_manifest (p, nv, iu);
-
- // Make sure this is the end.
- //
- nv = p.next ();
- if (!nv.empty ())
- throw parsing (p.name (), nv.name_line, nv.name_column,
- "single bootstrapped machine manifest expected");
- }
-
- void bootstrapped_machine_manifest::
- serialize (serializer& s) const
- {
- // @@ Should we check that all non-optional values are specified?
- //
- s.next ("", "1"); // Start of manifest.
- s.next ("", ""); // End of manifest.
-
- if (!machine.mac)
- throw serialization (s.name (),
- "mac address must be present in machine manifest");
-
- machine.serialize (s);
- toolchain.serialize (s);
- bootstrap.serialize (s);
-
- s.next ("", ""); // End of stream.
- }
}
diff --git a/bbot/buildfile b/bbot/buildfile
index 311775d..467b53e 100644
--- a/bbot/buildfile
+++ b/bbot/buildfile
@@ -26,7 +26,8 @@ if ($cxx.target.class == "linux")
exe{bbot-agent}: \
{hxx cxx}{ agent } {hxx ixx cxx}{ agent-options } \
- {hxx cxx}{ bootstrap-manifest } {hxx ixx cxx}{ common-options } \
+ {hxx ixx cxx}{ common-options } \
+ {hxx cxx}{ bootstrap-manifest } \
{hxx cxx}{ diagnostics } \
{hxx cxx}{ machine-manifest } \
{hxx cxx}{ machine } \
@@ -43,6 +44,7 @@ if ($cxx.target.class == "linux")
exe{bbot-worker}: \
{ cxx}{ worker } {hxx ixx cxx}{ worker-options } \
{hxx ixx cxx}{ common-options } \
+{hxx cxx}{ bootstrap-manifest } \
{hxx cxx}{ diagnostics } \
{hxx }{ types } \
{hxx cxx}{ types-parsers } \
diff --git a/bbot/machine-manifest b/bbot/machine-manifest
index 77f1c78..3801ae7 100644
--- a/bbot/machine-manifest
+++ b/bbot/machine-manifest
@@ -9,10 +9,12 @@
#include <butl/manifest-forward>
+#include <bbot/manifest> // machine_header
+
#include <bbot/types>
#include <bbot/utility>
-#include <bbot/manifest> // machine_header
+#include <bbot/bootstrap-manifest>
namespace bbot
{
@@ -53,6 +55,56 @@ namespace bbot
void
serialize (butl::manifest_serializer&) const;
};
+
+ // Toolchain.
+ //
+ class toolchain_manifest
+ {
+ public:
+
+ // Toolchain id (SHAXXX).
+ //
+ string id;
+
+ explicit
+ toolchain_manifest (string i): id (i) {}
+
+ public:
+ toolchain_manifest () = default; // VC export.
+ toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false);
+ toolchain_manifest (butl::manifest_parser&,
+ butl::manifest_name_value start,
+ bool ignore_unknown = false);
+
+ void
+ serialize (butl::manifest_serializer&) const;
+ };
+
+ // The manifest stored in <name>-<toolchain>/ consists of the machine
+ // manifest (original), toolchain manifest, and bootstrap manifest.
+ //
+ class bootstrapped_machine_manifest
+ {
+ public:
+ machine_manifest machine;
+ toolchain_manifest toolchain;
+ bootstrap_manifest bootstrap;
+
+ bootstrapped_machine_manifest (machine_manifest m,
+ toolchain_manifest t,
+ bootstrap_manifest b)
+ : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {}
+
+ public:
+ bootstrapped_machine_manifest () = default; // VC export.
+ bootstrapped_machine_manifest (butl::manifest_parser&,
+ bool ignore_unknown = false);
+
+ void
+ serialize (butl::manifest_serializer&) const;
+ };
+
+ using bootstrapped_machine_manifests = vector<bootstrapped_machine_manifest>;
}
#endif // BBOT_MACHINE_MANIFEST
diff --git a/bbot/machine-manifest.cxx b/bbot/machine-manifest.cxx
index 8e5100c..5cef054 100644
--- a/bbot/machine-manifest.cxx
+++ b/bbot/machine-manifest.cxx
@@ -172,4 +172,158 @@ namespace bbot
s.next ("", ""); // End of manifest.
}
+ // toolchain_manifest
+ //
+ toolchain_manifest::
+ toolchain_manifest (parser& p, bool iu)
+ : toolchain_manifest (p, p.next (), iu)
+ {
+ // Make sure this is the end.
+ //
+ name_value nv (p.next ());
+ if (!nv.empty ())
+ throw parsing (p.name (), nv.name_line, nv.name_column,
+ "single toolchain manifest expected");
+ }
+
+ toolchain_manifest::
+ toolchain_manifest (parser& p, name_value nv, bool iu)
+ {
+ auto bad_name = [&p, &nv] (const string& d)
+ {
+ throw parsing (p.name (), nv.name_line, nv.name_column, d);
+ };
+
+ auto bad_value = [&p, &nv] (const string& d)
+ {
+ throw parsing (p.name (), nv.value_line, nv.value_column, d);
+ };
+
+ // Make sure this is the start and we support the version.
+ //
+ if (!nv.name.empty ())
+ bad_name ("start of toolchain manifest expected");
+
+ if (nv.value != "1")
+ bad_value ("unsupported format version");
+
+ // Parse the toolchain manifest.
+ //
+ for (nv = p.next (); !nv.empty (); nv = p.next ())
+ {
+ string& n (nv.name);
+ string& v (nv.value);
+
+ if (n == "id")
+ {
+ if (!id.empty ())
+ bad_name ("toolchain id redefinition");
+
+ if (v.empty ())
+ bad_value ("empty toolchain id");
+
+ id = move (v);
+ }
+ else if (!iu)
+ bad_name ("unknown name '" + n + "' in toolchain manifest");
+ }
+
+ // Verify all non-optional values were specified.
+ //
+ if (id.empty ())
+ bad_value ("no toolchain id specified");
+ }
+
+ void toolchain_manifest::
+ serialize (serializer& s) const
+ {
+ // @@ Should we check that all non-optional values are specified?
+ //
+ s.next ("", "1"); // Start of manifest.
+ s.next ("id", id);
+ s.next ("", ""); // End of manifest.
+ }
+
+ // bootstrapped_machine_manifest
+ //
+ bootstrapped_machine_manifest::
+ bootstrapped_machine_manifest (parser& p, bool iu)
+ {
+ name_value nv (p.next ());
+
+ auto bad_name = [&p, &nv] (const string& d)
+ {
+ throw parsing (p.name (), nv.name_line, nv.name_column, d);
+ };
+
+ auto bad_value = [&p, &nv] (const string& d)
+ {
+ throw parsing (p.name (), nv.value_line, nv.value_column, d);
+ };
+
+ // Make sure this is the start and we support the version.
+ //
+ if (!nv.name.empty ())
+ bad_name ("start of bootstrapped machine manifest expected");
+
+ if (nv.value != "1")
+ bad_value ("unsupported format version");
+
+ // Parse the bootstrapped machine manifest. Currently there is no values
+ // expected.
+ //
+ for (nv = p.next (); !nv.empty (); nv = p.next ())
+ {
+ if (!iu)
+ bad_name ("unknown name '" + nv.name +
+ "' in bootstrapped machine manifest");
+ }
+
+ nv = p.next ();
+ if (nv.empty ())
+ bad_value ("machine manifest expected");
+
+ machine = machine_manifest (p, nv, iu);
+
+ if (!machine.mac)
+ bad_name ("mac address must be present in machine manifest");
+
+ nv = p.next ();
+ if (nv.empty ())
+ bad_value ("toolchain manifest expected");
+
+ toolchain = toolchain_manifest (p, nv, iu);
+
+ nv = p.next ();
+ if (nv.empty ())
+ bad_value ("bootstrap manifest expected");
+
+ bootstrap = bootstrap_manifest (p, nv, iu);
+
+ // Make sure this is the end.
+ //
+ nv = p.next ();
+ if (!nv.empty ())
+ throw parsing (p.name (), nv.name_line, nv.name_column,
+ "single bootstrapped machine manifest expected");
+ }
+
+ void bootstrapped_machine_manifest::
+ serialize (serializer& s) const
+ {
+ // @@ Should we check that all non-optional values are specified?
+ //
+ s.next ("", "1"); // Start of manifest.
+ s.next ("", ""); // End of manifest.
+
+ if (!machine.mac)
+ throw serialization (s.name (),
+ "mac address must be present in machine manifest");
+
+ machine.serialize (s);
+ toolchain.serialize (s);
+ bootstrap.serialize (s);
+
+ s.next ("", ""); // End of stream.
+ }
}
diff --git a/bbot/machine.cxx b/bbot/machine.cxx
index 460e802..0fc38ee 100644
--- a/bbot/machine.cxx
+++ b/bbot/machine.cxx
@@ -149,7 +149,7 @@ namespace bbot
br (br),
tap (create_tap (br, port)),
port (port),
- vnc ("127.0.0.1:" + to_string (5900 + stoul (tc_num))),
+ vnc ("127.0.0.1:" + to_string (5900 + tc_num)),
monitor ("/tmp/" + tc_name + "-monitor")
{
tracer trace ("kvm_machine");
diff --git a/bbot/tftp.cxx b/bbot/tftp.cxx
index 27d58a4..7249214 100644
--- a/bbot/tftp.cxx
+++ b/bbot/tftp.cxx
@@ -118,7 +118,7 @@ namespace bbot
"--map-file", map_.path (), // Path remapping rules.
"--user", uname, // Run as our effective user.
"--secure", // Chroot to data directory.
- "/build/tftp/");
+ ops.tftp ());
sec = static_cast<size_t> (timeout.tv_sec);
return true;
diff --git a/bbot/types b/bbot/types
index e6ad1fb..e4af01e 100644
--- a/bbot/types
+++ b/bbot/types
@@ -22,6 +22,8 @@
#include <butl/path>
#include <butl/optional>
+#include <butl/vector-view>
+#include <butl/small-vector>
namespace bbot
{
@@ -66,6 +68,14 @@ namespace bbot
using butl::optional;
using butl::nullopt;
+ // <butl/vector-view>
+ //
+ using butl::vector_view;
+
+ // <butl/small-vector>
+ //
+ using butl::small_vector;
+
// <butl/path>
//
using butl::path;
diff --git a/bbot/utility b/bbot/utility
index 91bce64..59f6e09 100644
--- a/bbot/utility
+++ b/bbot/utility
@@ -13,7 +13,9 @@
#include <butl/ft/lang>
+#include <butl/curl>
#include <butl/process>
+#include <butl/process-io>
#include <butl/utility> // casecmp(), reverse_iterate(), etc
#include <butl/fdstream>
#include <butl/filesystem>
@@ -69,11 +71,11 @@ namespace bbot
template <typename I, typename O, typename E, typename P, typename... A>
void
- run_io (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+ run_io (tracer&, I&& in, O&& out, E&& err, const P&, A&&...);
template <typename I, typename O, typename E, typename P, typename... A>
process_exit::code_type
- run_io_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+ run_io_exit (tracer&, I&& in, O&& out, E&& err, const P&, A&&...);
template <typename I, typename O, typename E, typename P, typename... A>
process
@@ -82,35 +84,61 @@ namespace bbot
O&& out,
E&& err,
const dir_path& cwd,
- P&&,
+ const P&,
A&&...);
template <typename P>
void
- run_io_finish (tracer&, process&, P&&);
+ run_io_finish (tracer&, process&, const P&);
template <typename P>
process_exit::code_type
- run_io_finish_exit (tracer&, process&, P&&);
+ run_io_finish_exit (tracer&, process&, const P&);
template <typename P, typename... A>
inline void
- run (tracer& t, P&& p, A&&... a)
+ run (tracer& t, const P& p, A&&... a)
{
- run_io (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+ run_io (t, butl::fdnull (), 2, 2, p, forward<A> (a)...);
}
template <typename P, typename... A>
inline process_exit::code_type
- run_exit (tracer& t, P&& p, A&&... a)
+ run_exit (tracer& t, const P& p, A&&... a)
{
return run_io_exit (
- t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+ t, butl::fdnull (), 2, 2, p, forward<A> (a)...);
}
void
run_trace (tracer&, const char*[], size_t);
+ // The curl process wrapper (command line tracing, etc).
+ //
+ class http_curl: public butl::curl
+ {
+ public:
+ template <typename I, typename O, typename... A>
+ http_curl (tracer&,
+ I&& in,
+ O&& out,
+ method_type,
+ const string& url,
+ A&&... options);
+ };
+
+ class tftp_curl: public butl::curl
+ {
+ public:
+ template <typename I, typename O, typename... A>
+ tftp_curl (tracer&,
+ I&& in,
+ O&& out,
+ method_type,
+ const string& url,
+ A&&... options);
+ };
+
// Manifest parsing and serialization.
//
template <typename T>
@@ -118,6 +146,13 @@ namespace bbot
parse_manifest (const path&, const char* what, bool ignore_unknown = true);
template <typename T>
+ T
+ parse_manifest (istream&,
+ const string& name,
+ const char* what,
+ bool ignore_unknown = true);
+
+ template <typename T>
void
serialize_manifest (const T&, const path&, const char* what);
diff --git a/bbot/utility.cxx b/bbot/utility.cxx
index 6b7a8ca..83249ad 100644
--- a/bbot/utility.cxx
+++ b/bbot/utility.cxx
@@ -29,7 +29,7 @@ namespace bbot
void
run_trace (tracer& t, const char* cmd[], size_t n)
{
- if (verb >= 2)
+ if (verb >= 3)
{
diag_record dr (t);
process::print (dr.os, cmd, n);
diff --git a/bbot/utility.txx b/bbot/utility.txx
index a613b11..edd674b 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -11,6 +11,8 @@
namespace bbot
{
+ // run_*()
+ //
template <typename I, typename O, typename E, typename P, typename... A>
process
run_io_start (tracer& t,
@@ -18,7 +20,7 @@ namespace bbot
O&& out,
E&& err,
const dir_path& cwd,
- P&& p,
+ const P& p,
A&&... args)
{
try
@@ -40,17 +42,19 @@ namespace bbot
template <typename P>
process_exit::code_type
- run_io_finish_exit (tracer&, process& pr, P&& p)
+ run_io_finish_exit (tracer&, process& pr, const P& p)
{
try
{
pr.wait ();
- if (!pr.exit->normal ())
- fail << "process " << p << " terminated abnormally: "
- << pr.exit->description ();
+ const process_exit& e (*pr.exit);
- return pr.exit->code ();
+ if (e.normal ())
+ return e.code ();
+
+ fail << "process " << p << " terminated abnormally: "
+ << e.description () << (e.core () ? " (core dumped)" : "") << endf;
}
catch (const process_error& e)
{
@@ -60,7 +64,7 @@ namespace bbot
template <typename P>
inline void
- run_io_finish (tracer& t, process& pr, P&& p)
+ run_io_finish (tracer& t, process& pr, const P& p)
{
if (run_io_finish_exit (t, pr, p) != 0)
fail << "process " << p << " terminated with non-zero exit code";
@@ -95,6 +99,47 @@ namespace bbot
run_io_finish (t, pr, p);
}
+ // curl
+ //
+ template <typename I, typename O, typename... A>
+ http_curl::
+ http_curl (tracer& t,
+ I&& in,
+ O&& out,
+ method_type m,
+ const string& url,
+ A&&... options)
+ : butl::curl ([&t] (const char* c[], size_t n) {run_trace (t, c, n);},
+ forward<I> (in),
+ forward<O> (out),
+ 2, // Diagnostics to stderr.
+ m,
+ url,
+ "-A", BBOT_USER_AGENT,
+ forward<A> (options)...)
+ {
+ }
+
+ template <typename I, typename O, typename... A>
+ tftp_curl::
+ tftp_curl (tracer& t,
+ I&& in,
+ O&& out,
+ method_type m,
+ const string& url,
+ A&&... options)
+ : butl::curl ([&t] (const char* c[], size_t n) {run_trace (t, c, n);},
+ forward<I> (in),
+ forward<O> (out),
+ 2, // Diagnostics to stderr.
+ m,
+ url,
+ forward<A> (options)...)
+ {
+ }
+
+ // *_manifest()
+ //
template <typename T>
T
parse_manifest (const path& f, const char* what, bool ignore_unknown)
@@ -107,23 +152,38 @@ namespace bbot
fail << what << " manifest file " << f << " does not exist";
ifdstream ifs (f);
- manifest_parser p (ifs, f.string ());
+ return parse_manifest<T> (ifs, f.string (), what, ignore_unknown);
+ }
+ catch (const system_error& e) // EACCES, etc.
+ {
+ fail << "unable to access " << what << " manifest " << f << ": " << e
+ << endf;
+ }
+ }
+
+ template <typename T>
+ T
+ parse_manifest (istream& is,
+ const string& name,
+ const char* what,
+ bool ignore_unknown)
+ {
+ using namespace butl;
+
+ try
+ {
+ manifest_parser p (is, name);
return T (p, ignore_unknown);
}
catch (const manifest_parsing& e)
{
fail << "invalid " << what << " manifest: "
- << f << ':' << e.line << ':' << e.column << ": " << e.description
+ << name << ':' << e.line << ':' << e.column << ": " << e.description
<< endf;
}
catch (const io_error& e)
{
- fail << "unable to read " << what << " manifest " << f << ": " << e
- << endf;
- }
- catch (const system_error& e) // EACCES, etc.
- {
- fail << "unable to access " << what << " manifest " << f << ": " << e
+ fail << "unable to read " << what << " manifest " << name << ": " << e
<< endf;
}
}
@@ -158,7 +218,6 @@ namespace bbot
const string& name,
const char* what)
{
- using namespace std;
using namespace butl;
try
diff --git a/bbot/worker.cli b/bbot/worker.cli
index 3691b2d..561d714 100644
--- a/bbot/worker.cli
+++ b/bbot/worker.cli
@@ -11,18 +11,27 @@ include <bbot/common.cli>;
namespace bbot
{
{
- "<options> <file>",
+ "<options> <module> <conf-var>",
"
\h|SYNOPSIS|
\cb{bbot-worker --help}\n
\cb{bbot-worker --version}\n
- \c{\b{bbot-worker} [<options>] <toolchain>}
+ \c{\b{bbot-worker --bootstrap} [<options>]}\n
+ \c{\b{bbot-worker --startup} [<options>]}\n
+ \c{\b{bbot-worker} [<options>] <module>... <conf-var>...}
\h|DESCRIPTION|
\cb{bbot-worker} @@ TODO.
+
+ If the \cb{--bootstrap} mode option is specified, then the worker performs
+ the initial machine bootstrap and writes the bootstrap result manifest to
+ \c{stdout}. If the \cb{--startup} mode option is specified, then the
+ worker performs the environment setup and then re-executes in the build
+ mode. If neither of the mode options is specified, then the worker
+ proceeds to performing the build task.
"
}
@@ -32,6 +41,47 @@ namespace bbot
bool --help {"Print usage information and exit."}
bool --version {"Print version and exit."}
+
+ bool --bootstrap
+ {
+ "Perform the inital machine bootstrap insteading of building."
+ }
+
+ bool --startup
+ {
+ "Perform the environment setup and then re-execute for building."
+ }
+
+ dir_path --build
+ {
+ "<dir>",
+ "The directory to perform the build in. If not specified, then the
+ current working directory is used."
+ }
+
+ dir_path --environment
+ {
+ "<dir>",
+ "The directory containing the environment setup executables. If not
+ specified, then the user's home directory is used."
+ }
+
+ uint16_t --verbose = 1
+ {
+ "<level>",
+ "Set the diagnostics verbosity to <level> between 0 and 6 with level 1
+ being the default."
+ }
+
+ // Testing options.
+ //
+ string --tftp-host = "196.254.111.222"
+ {
+ "<addr>",
+ "The TFTP host address and, optionally, port to use to download the
+ build task and to upload the build result. By default the link-local
+ address 196.254.111.222 with the default TFTP port (69) is used."
+ }
};
"
diff --git a/bbot/worker.cxx b/bbot/worker.cxx
index e0149fc..fbae8c7 100644
--- a/bbot/worker.cxx
+++ b/bbot/worker.cxx
@@ -11,12 +11,16 @@
#include <iostream>
#include <butl/pager>
+#include <butl/filesystem>
+
+#include <bbot/manifest>
#include <bbot/types>
#include <bbot/utility>
#include <bbot/diagnostics>
#include <bbot/worker-options>
+#include <bbot/bootstrap-manifest>
using namespace std;
using namespace butl;
@@ -24,7 +28,412 @@ using namespace bbot;
namespace bbot
{
+ process_path argv0;
worker_options ops;
+
+ dir_path env_dir;
+
+ const size_t tftp_timeout (10); // 10 seconds.
+}
+
+static dir_path
+change_wd (const dir_path& d, bool create = false)
+try
+{
+ if (create)
+ try_mkdir_p (d);
+
+ dir_path r (dir_path::current_directory ());
+
+ dir_path::current_directory (d);
+
+ return r;
+}
+catch (const system_error& e)
+{
+ fail << "unable to change current directory to " << d << ": " << e << endf;
+}
+
+template <typename... A>
+static result_status
+run_bpkg (tracer& t, string& log, const string& cmd, A&&... a)
+{
+ try
+ {
+ // Trace and log the command line.
+ //
+ auto cmdc = [&t, &log] (const char* c[], size_t n)
+ {
+ run_trace (t, c, n);
+
+ ostringstream os;
+ process::print (os, c, n);
+ log += os.str ();
+ log += '\n';
+ };
+
+ fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.
+
+ process pr (
+ process_start (cmdc,
+ fdnull (), // Never reads from stdout.
+ 2, // 1>&2
+ pipe.out,
+ dir_path (),
+ "bpkg",
+ "-v",
+ cmd,
+ forward<A> (a)...));
+
+ pipe.out.close ();
+
+ // Log the diagnostics.
+ //
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip); // Skip on exception.
+
+ for (string l; is.peek () != ifdstream::traits_type::eof (); )
+ {
+ getline (is, l);
+ log += l;
+ log += '\n';
+ }
+ }
+
+ if (pr.wait ())
+ return result_status::success;
+
+ log += "bpkg " + cmd;
+ const process_exit& e (*pr.exit);
+
+ if (e.normal ())
+ {
+ log += " exited with code " + to_string (e.code ()) + "\n";
+ return result_status::error;
+ }
+ else
+ {
+ log += " terminated abnormally: " + e.description () +
+ (e.core () ? " (core dumped)" : "") + "\n";
+ return result_status::abnormal;
+ }
+ }
+ catch (const process_error& e)
+ {
+ fail << "unable to execute bpkg " << cmd << ": " << e << endf;
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to read bpkg " << cmd << " diagnostics: " << e << endf;
+ }
+}
+
+static void
+build (size_t argc, const char* argv[])
+{
+ tracer trace ("build");
+
+ // Our overall plan is as follows:
+ //
+ // 1. Parse the task manifest (it should be in CWD).
+ //
+ // 2. Run bpkg to create the configuration, add the repository, and
+ // configure, build, and test the package all while saving the logs in
+ // the result manifest.
+ //
+ // 3. Upload the result manifest.
+ //
+ // Note also that we are being "watched" by the startup version of us which
+ // will upload an appropriate result in case we exit with an error. So here
+ // for abnormal situations (like a failure to parse the manifest), we just
+ // fail.
+ //
+ task_manifest tm (parse_manifest<task_manifest> (path ("manifest"), "task"));
+
+ result_manifest rm {
+ tm.name,
+ tm.version,
+ result_status::success,
+ operation_results {}
+ };
+
+ auto add_result = [&rm] (string o) -> operation_result&
+ {
+ rm.results.push_back (
+ operation_result {move (o), result_status::success, ""});
+
+ return rm.results.back ();
+ };
+
+ dir_path owd;
+
+ for (;;) // The "breakout" loop.
+ {
+ // Configure.
+ //
+ {
+ operation_result& r (add_result ("configure"));
+
+ // bpkg create <config-vars> <env-module> <env-config-vars>
+ //
+ const vector_view<const char*> env (argv + 1, argc - 1);
+
+ vector<string> cfg;
+ for (const variable& v: tm.config)
+ cfg.push_back (v.unquoted ());
+
+ dir_path dir ("build");
+
+ r.status |= run_bpkg (trace, r.log,
+ "create",
+ "-d", dir.string (),
+ "--wipe",
+ cfg,
+ env);
+
+ if (!r.status)
+ break;
+
+ owd = change_wd (dir);
+
+ // bpkg add <repository-url>
+ //
+ r.status |= run_bpkg (trace, r.log, "add", tm.repository.string ());
+
+ if (!r.status)
+ break;
+
+ // bpkg fetch
+ //
+ string t ("--trust-no");
+
+ cstrings ts;
+ for (const string& fp: tm.trust)
+ {
+ if (fp == "yes")
+ t = "--trust-yes";
+ else
+ {
+ ts.push_back ("--trust");
+ ts.push_back (fp.c_str ());
+ }
+ }
+
+ r.status |= run_bpkg (trace, r.log, "fetch", ts, t);
+
+ if (!r.status)
+ break;
+
+ // bpkg build --configure-only <package-name>/<package-version>
+ //
+ r.status |= run_bpkg (trace, r.log,
+ "build",
+ "--configure-only",
+ "--yes",
+ tm.name + '/' + tm.version.string ());
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ // Update.
+ //
+ {
+ operation_result& r (add_result ("update"));
+
+ // bpkg update <package-name>
+ //
+ r.status |= run_bpkg (trace, r.log, "update", tm.name);
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ // Test.
+ //
+ {
+ operation_result& r (add_result ("test"));
+
+ // bpkg test <package-name>
+ //
+ r.status |= run_bpkg (trace, r.log, "test", tm.name);
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ break;
+ }
+
+ rm.status |= rm.results.back ().status; // Merge last in case of a break.
+
+ if (!owd.empty ())
+ change_wd (owd);
+
+ // Upload the result.
+ //
+ const string url ("tftp://" + ops.tftp_host () + "/manifest");
+
+ try
+ {
+ tftp_curl c (trace,
+ path ("-"),
+ nullfd,
+ curl::put,
+ url,
+ "--max-time", tftp_timeout);
+
+ serialize_manifest (rm, c.out, url, "result");
+ c.out.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to upload result manifest to " << url << ": " << e;
+ }
+}
+
+static void
+startup ()
+{
+ tracer trace ("startup");
+
+ // Our overall plan is as follows:
+ //
+ // 1. Download the task manifest into the build directory (CWD).
+ //
+ // 2. Parse it and get the target.
+ //
+ // 3. Find the environment setup executable for this target.
+ //
+ // 4. Execute the environment setup executable.
+ //
+ // 5. If the environment setup executable fails, then upload the (failed)
+ // result ourselves.
+ //
+ const string url ("tftp://" + ops.tftp_host () + "/manifest");
+ const path mf ("manifest");
+
+ // If we fail, try to upload the result manifest (abnormal termination). The
+ // idea is that the machine gets suspended and we can investigate what's
+ // going on by logging in and examining the diagnostics (e.g., via
+ // journalctl, etc).
+ //
+ task_manifest tm;
+
+ try
+ {
+ // Download the task.
+ //
+ try
+ {
+ tftp_curl c (trace,
+ nullfd,
+ mf,
+ curl::get,
+ url,
+ "--max-time", tftp_timeout);
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to download task manifest from " << url << ": " << e;
+ }
+
+ // Parse it.
+ //
+ tm = parse_manifest<task_manifest> (mf, "task");
+
+ // Find the environment setup executable.
+ //
+ string tg;
+ process_path pp;
+
+ if (tm.target)
+ {
+ tg = tm.target->string ();
+
+ // While the executable path contains a directory (so the PATH search
+ // does not apply) we still use process::path_search() to automatically
+ // handle appending platform-specific executable extensions (.exe/.bat,
+ // etc).
+ //
+ pp = process::try_path_search (env_dir / tg, false);
+ }
+
+ if (pp.empty ())
+ pp = process::try_path_search (env_dir / "default", false);
+
+ if (pp.empty ())
+ fail << "no environment setup executable in " << env_dir << " "
+ << "for target '" << tg << "'";
+
+ // Run it.
+ //
+ // Note that we use the effective (absolute) path instead of recall since
+ // we may have changed the CWD.
+ //
+ run (trace, pp, tg, argv0.effect_string ());
+ }
+ catch (const failed&)
+ {
+ // If we failed before being able to parse the task manifest, use the
+ // "unknown" values for the package name and version.
+ //
+ result_manifest rm {
+ tm.name.empty () ? "unknown" : tm.name,
+ tm.version.empty () ? bpkg::version ("0") : tm.version,
+ result_status::abnormal,
+ operation_results {}
+ };
+
+ try
+ {
+ tftp_curl c (trace,
+ path ("-"),
+ nullfd,
+ curl::put,
+ url,
+ "--max-time", tftp_timeout);
+
+ serialize_manifest (rm, c.out, url, "result");
+ c.out.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to upload result manifest to " << url << ": " << e;
+ }
+
+ throw;
+ }
+}
+
+static void
+bootstrap ()
+{
+ bootstrap_manifest bm {
+ bootstrap_manifest::versions_type {
+ {"bbot", BBOT_VERSION},
+ {"libbbot", LIBBBOT_VERSION},
+ {"libbpkg", LIBBPKG_VERSION},
+ {"libbutl", LIBBUTL_VERSION}
+ }
+ };
+
+ serialize_manifest (bm, cout, "stdout", "bootstrap");
}
int
@@ -69,6 +478,8 @@ try
cli::argv_scanner scan (argc, argv, true);
ops.parse (scan);
+ verb = ops.verbose ();
+
// Version.
//
if (ops.version ())
@@ -94,8 +505,48 @@ try
return p.wait () ? 0 : 1;
}
- warn << "starting up" <<
- info << "lightly";
+ // Figure out our mode.
+ //
+ if (ops.bootstrap () && ops.startup ())
+ fail << "--bootstrap and --startup are mutually exclusive";
+
+ enum class mode {boot, start, build} m (mode::build);
+
+ if (ops.bootstrap ()) m = mode::boot;
+ if (ops.startup ()) m = mode::start;
+
+ // Figure out our path (used for re-exec).
+ //
+ argv0 = process::path_search (argv[0], true);
+
+ // Sort out the build directory.
+ //
+ if (ops.build_specified ())
+ change_wd (ops.build (), true); // Create if does not exist.
+
+ // Sort out the environment directory.
+ //
+ try
+ {
+ env_dir = ops.environment_specified ()
+ ? ops.environment ()
+ : dir_path::home_directory ();
+
+ if (!dir_exists (env_dir))
+ throw_generic_error (ENOENT);
+ }
+ catch (const system_error& e)
+ {
+ fail << "invalid environment directory: " << e;
+ }
+
+ switch (m)
+ {
+ case mode::boot: bootstrap (); break;
+ case mode::start: startup (); break;
+ case mode::build: build (static_cast<size_t> (argc),
+ const_cast<const char**> (argv)); break;
+ }
}
catch (const failed&)
{
diff --git a/doc/manual.cli b/doc/manual.cli
index 5da684a..2b2fc21 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -243,8 +243,9 @@ SYNOPSIS
name: <package-name>
version: <package-version>
-repository: <repository-url>
#location: <package-url>
+repository: <repository-url>
+trust: <repository-fp>
machine: <machine-name>
target: <target-triplet>
@@ -269,6 +270,19 @@ build configuration to use for building the package.
The \c{bpkg} repository that contains the package and its dependencies.|
+\li|\n\c{trust: <repository-fp>}\n
+
+ The SHA256 repository certificate fingerprint to trust (see the \c{bpkg}
+ \c{--trust} option for details). This value may be specified multiple times
+ to establish the authenticity of multiple certificates. If the special
+ \c{yes} value is specified, then all repositories will be trusted without
+ authentication (see the \c{bpkg} \c{--trust-yes} option).
+
+ Note that while the controller may return a task with \c{trust} values,
+ whether they will be used is up to the agent's configuration. For example,
+ some agents may only trust their internally-specified fingerprints to
+ prevent the \"man in the middle\" type of attacks.|
+
\li|\n\c{machine: <machine-name>}\n
The name of the build machine to use.|
@@ -480,16 +494,18 @@ the environment executable is an error.
Once the environment setup executable is determined, the worker re-executes
itself as that executable passing to it as command line arguments the target
-name (or empty value if not specified), the path to the \c{bbot} worker to be
-executed once the environment is setup, and the path to the build task
-manifest.
+name (or empty value if not specified) and the path to the \c{bbot} worker to
+be executed once the environment is setup. The environment setup executable is
+executed in the build directory as its current working directory. The build
+directory contains the build task \c{manifest} file.
The environment setup executable sets up the necessary execution environment
for example by adjusting \c{PATH} or running a suitable \c{vcvars} batch file.
It then re-executes itself as the \c{bbot} worker passing to it as command
-line arguments the path to the build task manifest followed by the list of
-build system modules (\c{<env-modules>}) and the list of configuration
-variables (\c{<env-config-vars>}).
+line arguments the list of build system modules (\c{<env-modules>}) and the
+list of configuration variables (\c{<env-config-vars>}). The environment setup
+executable must execute the \c{bbot} worker in the build directory as the
+current working directory.
The re-executed \c{bbot} worker then proceeds to test the package from the
repository by executing the following commands (\c{<>}-values are from the
@@ -498,7 +514,7 @@ task manifest and environment):
\
bpkg -v create <env-module> <config-vars> <env-config-vars>
bpkg -v add <repository-url>
-bpkg -v fetch
+bpkg -v fetch --trust <repository-fp>
bpkg -v build --yes --configure-only <package-name>/<package-version>
bpkg -v update <package-name>
bpkg -v test <package-name>
@@ -512,7 +528,6 @@ for building C and C++ packages with GCC 6 on most Linux distributions.
# $1 - target
# $2 - bbot executable
-# $3 - task manifest
trap \"exit 1\" ERR
@@ -521,7 +536,7 @@ if [ -n \"$1\" ]; then
exit 1
fi
-exec \"$2\" --build \"$3\" cc config.c=gcc-6
+exec \"$2\" cc config.c=gcc-6
\
\h#arch-controller|Controller Logic|
diff --git a/tests/agent/testscript b/tests/agent/testscript
index cf92ace..e864bf2 100644
--- a/tests/agent/testscript
+++ b/tests/agent/testscript
@@ -3,7 +3,7 @@
# license : TBC; see accompanying LICENSE file
# The /build/machines directory should be on a btrfs filesystem and have the
-# following layout and contents:
+# following layout/contents:
#
# /build/machines/
# └── default/
@@ -17,25 +17,24 @@
#
# - The test must be run serially (@@ TODO: serial directive)
-test.options = --verbose 2
-test.arguments = stage 1
+test.options = --verbose 3
cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines
rm = $src_base/btrfs-rmdir /build/machines
-#\
: dump-machines
:
{
- m = /build/machines/default/linux-gcc
-
test.options += --dump-machines --fake-bootstrap
+ m = /build/machines/default/linux-gcc
+ u = https://example.org/?dummy
+
+$cp
: no-current-machine-symlink
:
- $* 123 2>>"EOE"
+ $* --toolchain-id 123 $u 2>>"EOE"
trace: enumerate_machines: skipping $m/: no subvolume link
EOE
@@ -43,91 +42,140 @@ rm = $src_base/btrfs-rmdir /build/machines
: bootstrap
:
- $* 123 >>EOO 2>>~"%EOE%d"
+ $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d"
: 1
id: linux-gcc-1.0
name: linux-gcc
summary: Linux with GCC
EOO
- %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+%
- trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-default-\\.+%
+ trace: enumerate_machines: bootstrapping $m/linux-gcc-default/
+ trace: enumerate_machines: bootstrapped linux-gcc
EOE
ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1
: re-bootstrap-machine
:
- $* 123 >>EOO 2>>~"%EOE%d"
+ $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d"
: 1
id: linux-gcc-1.1
name: linux-gcc
summary: Linux with GCC
EOO
- %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
- trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine
- trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
- trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new machine
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+ trace: enumerate_machines: bootstrapped linux-gcc
EOE
: re-bootstrap-toolchain
:
- $* 124 >>EOO 2>>~"%EOE%d"
+ $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d"
: 1
id: linux-gcc-1.1
name: linux-gcc
summary: Linux with GCC
EOO
- %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
- trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain
- trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
- trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new toolchain
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+ trace: enumerate_machines: bootstrapped linux-gcc
EOE
- sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest
+ sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-default/manifest
: re-bootstrap-bbot
:
- $* 124 >>EOO 2>>~"%EOE%d"
+ $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d"
: 1
id: linux-gcc-1.1
name: linux-gcc
summary: Linux with GCC
EOO
- %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
- trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot
- trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
- trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new bbot
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default
+ trace: enumerate_machines: bootstrapped linux-gcc
EOE
- sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest
+ sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-default/manifest
: re-bootstrap-bbot-newer
:
- $* 124 2>>~"%EOE%d"
- %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
- trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot
- %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+%
+ $* --toolchain-id 124 $u 2>>~"%EOE%d"
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+%
+ trace: enumerate_machines: ignoring $m/linux-gcc-default/: old bbot
+ %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default-\\.+%
EOE
-$rm
}
-#\
-
-: bootstrap
+: dump-result
:
{
+ test.options += --fake-bootstrap --fake-build --fake-request -
+
m = /build/machines/default/linux-gcc
+ u = https://example.org/?dummy
+
+ +$cp
+ +ln -T -s linux-gcc-1.0 $m/linux-gcc-1
+ +$* --dump-machines --verbose 1 $u >- # Bootstrap.
+
+ : no-machine
+ :
+ $* --dump-task $u <<EOI 2>>~"%EOE%d"
+ : 1
+ name: foo
+ version: 1.2.3
+ repository: https://example.org/1/
+ machine: windows-msvc
+ EOI
+ %trace: enumerate_machines:\\.*%*
+ error: task from $u for unknown machine windows-msvc
+ EOE
+ : result
+ :
+ $* --dump-result $u <<EOI >>EOO 2>>~"%EOE%d"
+ : 1
+ name: foo
+ version: 1.2.3
+ repository: https://example.org/1/
+ machine: linux-gcc
+ EOI
+ : 1
+ name: foo
+ version: 1.2.3
+ status: abort
+ EOO
+ %trace: enumerate_machines:\\.*%*
+ EOE
+
+ -$rm
+}
+
+#\
+: bootstrap
+:
+{
test.options += --dump-machines
+ m = /build/machines/default/linux-gcc
+ u = https://example.org/?dummy
+
+$cp
ln -T -s linux-gcc-1.0 $m/linux-gcc-1
: bootstrap
:
- $* 123 >>EOO 2>>EOE #2>>~"%EOE%d"
+ $* $u >>EOO 2>>EOE #2>>~"%EOE%d"
: 1
id: linux-gcc-1.0
name: linux-gcc
@@ -137,3 +185,4 @@ rm = $src_base/btrfs-rmdir /build/machines
#-$rm
}
+#\
diff --git a/tests/worker/bootstrap.test b/tests/worker/bootstrap.test
new file mode 100644
index 0000000..91e4417
--- /dev/null
+++ b/tests/worker/bootstrap.test
@@ -0,0 +1,13 @@
+# file : tests/worker/bootstrap.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+test.options = --bootstrap
+
+$* >>~%EOO%
+: 1
+%bbot-version: \d+%
+%libbbot-version: \d+%
+%libbpkg-version: \d+%
+%libbutl-version: \d+%
+EOO
diff --git a/tests/worker/build.test b/tests/worker/build.test
new file mode 100644
index 0000000..adedccc
--- /dev/null
+++ b/tests/worker/build.test
@@ -0,0 +1,133 @@
+# file : tests/worker/build.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+# Note: requires TFTP server (see buildfile).
+
+tftp = 127.0.0.1:55123/test
+
+pkg = hello
+ver = 1.0.0
+
+#rep = /home/boris/work/build2/hello/repository/1/stable
+rep = https://build2.org/pkg/1/stage/stable
+#rep = https://build2.org/pkg/1/hello/stable
+
+rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
+
+: pass
+:
+cat <<"EOI" >=manifest;
+ : 1
+ name: $pkg
+ version: $ver
+ repository: $rep
+ trust: $rfp
+ machine: linux-gcc
+ target: x86_64-linux-gnu
+ config: config.cc.coptions=-O3
+ EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+ trace: build: bpkg -v create -d build --wipe config.cc.coptions=-O3 cc
+ trace: build: bpkg -v add $rep
+ trace: build: bpkg -v fetch --trust $rfp --trust-no
+ trace: build: bpkg -v build --configure-only --yes $pkg/$ver
+ trace: build: bpkg -v update $pkg
+ trace: build: bpkg -v test $pkg
+ trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+ EOE
+cat manifest >>~"%EOO%"
+ : 1
+ name: $pkg
+ version: $ver
+ status: success
+ configure-status: success
+ update-status: success
+ test-status: success
+ configure-log: \\
+ %.*%+
+ \\
+ update-log: \\
+ %.*%+
+ \\
+ test-log: \\
+ %.*%+
+ \\
+ EOO
+
+
+: fail-abnormal
+:
+cat <<"EOI" >=manifest;
+ : 1
+ foo: bar
+ EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc 2>>"EOE" != 0
+ error: invalid task manifest: manifest:3:1: no task package name specified
+ EOE
+
+
+: fail-configure
+:
+cat <<"EOI" >=manifest;
+ : 1
+ name: bogus
+ version: 1.2.3
+ repository: $rep
+ trust: $rfp
+ machine: linux-gcc
+ target: x86_64-linux-gnu
+ EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+ trace: build: bpkg -v create -d build --wipe cc
+ trace: build: bpkg -v add $rep
+ trace: build: bpkg -v fetch --trust $rfp --trust-no
+ trace: build: bpkg -v build --configure-only --yes bogus/1.2.3
+ trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+ EOE
+cat manifest >>~"%EOO%"
+ : 1
+ name: bogus
+ version: 1.2.3
+ status: error
+ configure-status: error
+ configure-log: \\
+ %.*%+
+ \\
+ EOO
+
+
+: fail-update
+:
+cat <<"EOI" >=manifest;
+ : 1
+ name: $pkg
+ version: $ver
+ repository: $rep
+ trust: $rfp
+ machine: linux-gcc
+ target: x86_64-linux-gnu
+ config: config.cc.loptions=-lbogus
+ EOI
+$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
+ trace: build: bpkg -v create -d build --wipe config.cc.loptions=-lbogus cc
+ trace: build: bpkg -v add $rep
+ trace: build: bpkg -v fetch --trust $rfp --trust-no
+ trace: build: bpkg -v build --configure-only --yes $pkg/$ver
+ trace: build: bpkg -v update $pkg
+ trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest
+ EOE
+cat manifest >>~"%EOO%"
+ : 1
+ name: $pkg
+ version: $ver
+ status: error
+ configure-status: success
+ update-status: error
+ configure-log: \\
+ %.*%+
+ \\
+ update-log: \\
+ %.*%+
+ \\
+ EOO
diff --git a/tests/worker/buildfile b/tests/worker/buildfile
new file mode 100644
index 0000000..7d0a206
--- /dev/null
+++ b/tests/worker/buildfile
@@ -0,0 +1,26 @@
+# file : tests/worker/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+#\
+
+Some tests in this directory require a running TFTP server.
+
+TFTP server (tftp-hpa) setup: from the test out_base, run (sudo is required
+for --secure/chroot):
+
+sudo /usr/sbin/in.tftpd \
+ --foreground \
+ --address 127.0.0.1:55123 \
+ --user "$(whoami)" \
+ --permissive \
+ --create \
+ --secure \
+ "$(pwd)"
+
+#\
+
+./: ../../bbot/exe{bbot-worker} test{bootstrap startup build}
+dir{./}: test = ../../bbot/exe{bbot-worker}
+
+include ../../bbot/
diff --git a/tests/worker/startup.test b/tests/worker/startup.test
new file mode 100644
index 0000000..d13e6c5
--- /dev/null
+++ b/tests/worker/startup.test
@@ -0,0 +1,101 @@
+# file : tests/worker/startup.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+# Note: requires TFTP server (see buildfile).
+
+test.options = --startup
+tftp = 127.0.0.1:55123/test
+
+: fail-download
+:
+$* --tftp-host "$tftp/$@" 2>>"EOE" &manifest != 0;
+ curl: \(68\) TFTP: File Not Found
+ error: unable to download task manifest from tftp://$tftp/$@/manifest: input/output error
+ EOE
+diff -u - manifest <<EOO
+ : 1
+ name: unknown
+ version: 0
+ status: abnormal
+ EOO
+
+: fail-manifest
+:
+cat <': 1' >=manifest;
+$* --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+ error: invalid task manifest: manifest:2:1: no task package name specified
+ EOE
+diff -u - manifest <<EOO
+ : 1
+ name: unknown
+ version: 0
+ status: abnormal
+ EOO
+
+: fail-env-search
+:
+cat <<EOI >=manifest;
+ : 1
+ name: libhello
+ version: 1.2.3
+ repository: https://pkg.example.org/1/
+ machine: linux-gcc
+ EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+ error: no environment setup executable in $representation($~) for target ''
+ EOE
+diff -u - manifest <<EOO
+ : 1
+ name: libhello
+ version: 1.2.3
+ status: abnormal
+ EOO
+
+: fail-env-run
+:
+cat <<EOI >=x86_64-linux-gnu;
+ #!/bin/sh
+ echo "environment setup failed"
+ exit 1
+ EOI
+chmod ugo+x x86_64-linux-gnu;
+cat <<EOI >=manifest;
+ : 1
+ name: libhello
+ version: 1.2.3
+ repository: https://pkg.example.org/1/
+ machine: linux-gcc
+ target: x86_64-linux-gnu
+ EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0;
+ environment setup failed
+ error: process $~/x86_64-linux-gnu terminated with non-zero exit code
+ EOE
+diff -u - manifest <<EOO
+ : 1
+ name: libhello
+ version: 1.2.3
+ status: abnormal
+ EOO
+
+: pass
+:
+cat <<EOI >=x86_64-linux-gnu;
+ #!/bin/sh
+ echo "$1"
+ echo "$2"
+ EOI
+chmod ugo+x x86_64-linux-gnu;
+cat <<EOI >=manifest;
+ : 1
+ name: libhello
+ version: 1.2.3
+ repository: https://pkg.example.org/1/
+ machine: linux-gcc
+ target: x86_64-linux-gnu
+ EOI
+$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE"
+ x86_64-linux-gnu
+ $0
+ EOE
diff --git a/unit-tests/bootstrap-manifest/driver.cxx b/unit-tests/bootstrap-manifest/driver.cxx
index 18bf7b6..46e64bc 100644
--- a/unit-tests/bootstrap-manifest/driver.cxx
+++ b/unit-tests/bootstrap-manifest/driver.cxx
@@ -12,6 +12,7 @@
#include <bbot/utility>
#include <bbot/bootstrap-manifest>
+#include <bbot/machine-manifest>
using namespace std;
using namespace butl;