aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bbot/agent.cli36
-rw-r--r--bbot/agent.cxx486
-rw-r--r--bbot/bbot-agent@.service2
-rw-r--r--bbot/bootstrap-manifest15
-rw-r--r--bbot/buildfile6
-rw-r--r--bbot/diagnostics12
-rw-r--r--bbot/diagnostics.cxx2
-rw-r--r--bbot/utility47
-rw-r--r--bbot/utility.txx144
-rwxr-xr-xtests/agent/btrfs-cpdir113
-rwxr-xr-xtests/agent/btrfs-rmdir66
-rw-r--r--tests/agent/buildfile8
-rw-r--r--tests/agent/testscript112
13 files changed, 813 insertions, 236 deletions
diff --git a/bbot/agent.cli b/bbot/agent.cli
index c4ee356..7a6c201 100644
--- a/bbot/agent.cli
+++ b/bbot/agent.cli
@@ -37,6 +37,9 @@ namespace bbot
{
"\h|OPTIONS|"
+ bool --help {"Print usage information and exit."}
+ bool --version {"Print version and exit."}
+
bool --systemd-daemon
{
"Start as a simple systemd daemon."
@@ -46,16 +49,41 @@ namespace bbot
{
"<num>",
"Number of CPUs (threads) to use, 1 by default."
- };
+ }
size_t --ram (1024 * 1024) // 1G
{
"<num>",
"Amount of RAM (in kB) to use, 1G by default."
- };
+ }
- bool --help {"Print usage information and exit."}
- bool --version {"Print version and exit."}
+ dir_path --machines = "/build/machines/"
+ {
+ "<dir>",
+ "The location of the build machines with \cb{/build/machines/} being
+ the default."
+ }
+
+ uint16_t --verbose = 1
+ {
+ "<level>",
+ "Set the diagnostics verbosity to <level> between 0 and 6 with level 1
+ being the default."
+ }
+
+ // Testing options.
+ //
+ bool --dump-machines
+ {
+ "Dump available machines to \c{stdout}, (re)-bootstrapping any if
+ necessary, and exit."
+ }
+
+ bool --fake-bootstrap
+ {
+ "Fake the machine bootstrap process by creating the expected bootstrapped
+ machine manifest."
+ }
};
"
diff --git a/bbot/agent.cxx b/bbot/agent.cxx
index 3e4f8dc..76c3a86 100644
--- a/bbot/agent.cxx
+++ b/bbot/agent.cxx
@@ -9,12 +9,8 @@
#include <iostream>
#include <butl/pager>
-#include <butl/fdstream>
#include <butl/filesystem> // dir_iterator
-#include <butl/manifest-parser>
-#include <butl/manifest-serializer>
-
#include <bbot/manifest>
#include <bbot/types>
@@ -29,50 +25,64 @@ using namespace std;
using namespace butl;
using namespace bbot;
+// The btrfs tool likes to print informational messages, like "Created
+// snapshot such and such". Luckily, it writes them to stdout while proper
+// diagnostics to stderr.
+//
+template <typename... A>
+inline void
+btrfs (tracer& t, A&&... a)
+{
+ if (verb >= 3)
+ run (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...);
+ else
+ run (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+}
+
+template <typename... A>
+inline butl::process_exit::code_type
+btrfs_exit (tracer& t, A&&... a)
+{
+ return verb >= 3
+ ? run_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...)
+ : run_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+}
+
+agent_options ops;
+
const string bs_prot ("1"); // Bootstrap protocol version.
string tc_name; // Toolchain name.
string tc_num; // Toolchain number.
string tc_id; // Toolchain id.
-template <typename T>
-static T
-parse_manifest (const path& f, const char* what, bool ignore_unknown = true)
+static bootstrapped_machine_manifest
+bootstrap_machine (const dir_path& md, const machine_manifest& mm)
{
- try
- {
- if (!file_exists (f))
- fail << what << " manifest file " << f << " does not exist";
+ bootstrapped_machine_manifest r {
+ mm,
+ toolchain_manifest {tc_id},
+ bootstrap_manifest {
+ bootstrap_manifest::versions_type {
+ {"bbot", BBOT_VERSION},
+ {"libbbot", LIBBBOT_VERSION},
+ {"libbpkg", LIBBPKG_VERSION},
+ {"libbutl", LIBBUTL_VERSION}
+ }
+ }
+ };
- ifdstream ifs (f);
- manifest_parser mp (ifs, f.string ());
- return T (mp, ignore_unknown);
- }
- catch (const manifest_parsing& e)
- {
- fail << "invalid " << what << " manifest: "
- << f << ':' << e.line << ':' << e.column << ": " << e.description
- << endf;
- }
- catch (const io_error& e)
- {
- fail << "unable to read " << what << " manifest " << f << ": " << e
- << endf;
- }
- catch (const system_error& e) // EACCES, etc.
+ if (!ops.fake_bootstrap ())
{
- fail << "unable to access " << what << " manifest " << f << ": " << e
- << endf;
}
-}
-
-/*
-static bootstrapped_machine_manifest
-bootstrap_machine (const dir_path&);
+ serialize_manifest (r, md / "manifest", "bootstrapped machine");
+ return r;
+}
static machine_manifests
enumerate_machines (const dir_path& rd)
+try
{
tracer trace ("enumerate_machines");
@@ -93,231 +103,242 @@ enumerate_machines (const dir_path& rd)
// Inside we have machines.
//
- for (const dir_entry& me: dir_iterator (vd))
+ try
{
- const string mn (me.path ().string ());
-
- if (me.type () != entry_type::directory || mn[0] == '.')
- continue;
-
- const dir_path md (dir_path (vd) /= mn);
-
- // Our endgoal here is to obtain a bootstrapped snapshot of this machine
- // while watching out for potential race conditions (machines being
- // added/upgraded/removed; see the manual for details).
- //
- // So here is our overall plan:
- //
- // 1. Resolve current subvolume link for our bootstrap protocol.
- //
- // 2. If there is no link, cleanup and ignore this machine.
- //
- // 3. Try to create a snapshot of current subvolume (this operation is
- // atomic). If failed (e.g., someone changed the link and removed the
- // subvolume in the meantime), retry from #1.
- //
- // 4. Compare the snapshot to the already bootstrapped version (if any)
- // and see if we need to re-bootstrap. If so, use the snapshot as a
- // starting point. Rename to bootstrapped at the end (atomic).
- //
- const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
- const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<toolc...>
- bool te (dir_exists (tp));
-
- auto delete_t = [&tp] ()
+ for (const dir_entry& me: dir_iterator (vd))
{
- // btrfs property set -ts $tp ro false
- // btrfs subvolume delete $tp
- };
+ const string mn (me.path ().string ());
- for (size_t retry (0);; ++retry)
- {
- if (retry != 0)
- sleep (1);
+ if (me.type () != entry_type::directory || mn[0] == '.')
+ continue;
+
+ const dir_path md (dir_path (vd) /= mn);
- // Resolve the link to subvolume path.
+ // Our endgoal here is to obtain a bootstrapped snapshot of this
+ // machine while watching out for potential race conditions (machines
+ // being added/upgraded/removed; see the manual for details).
//
- dir_path sp; // <name>-<P>.<R>
- try
+ // So here is our overall plan:
+ //
+ // 1. Resolve current subvolume link for our bootstrap protocol.
+ //
+ // 2. If there is no link, cleanup and ignore this machine.
+ //
+ // 3. Try to create a snapshot of current subvolume (this operation is
+ // atomic). If failed (e.g., someone changed the link and removed
+ // the subvolume in the meantime), retry from #1.
+ //
+ // 4. Compare the snapshot to the already bootstrapped version (if
+ // any) and see if we need to re-bootstrap. If so, use the snapshot
+ // as a starting point. Rename to bootstrapped at the end (atomic).
+ //
+ const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
+ const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<too...>
+ bool te (dir_exists (tp));
+
+ auto delete_t = [&tp, &trace] ()
+ {
+ btrfs (trace, "property", "set", "-ts", tp, "ro", "false");
+ btrfs (trace, "subvolume", "delete", tp);
+ };
+
+ for (size_t retry (0);; ++retry)
{
- char b [PATH_MAX + 1];
- ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b)));
+ if (retry != 0)
+ sleep (1);
- if (r == -1)
+ // Resolve the link to subvolume path.
+ //
+ dir_path sp; // <name>-<P>.<R>
+ try
{
- if (errno != ENOENT)
- throw_generic_error (errno);
+ char b [PATH_MAX + 1];
+ ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b)));
+
+ if (r == -1)
+ {
+ if (errno != ENOENT)
+ throw_generic_error (errno);
+ }
+ else if (static_cast<size_t> (r) >= sizeof (b))
+ throw_generic_error (EINVAL);
+ else
+ {
+ b[r] = '\0';
+ sp = dir_path (b);
+ if (sp.relative ())
+ sp = md / sp;
+ }
}
- else if (static_cast<size_t> (r) >= sizeof (b))
- throw_generic_error (EINVAL);
- else
+ catch (const system_error& e)
{
- b[r] = '\0';
- sp = dir_path (b);
- if (sp.relative ())
- sp = md / sp;
+ fail << "unable to read subvolume link " << lp << ": " << e;
}
- }
- catch (const system_error& e)
- {
- fail << "unable to read subvolume link " << lp << ": " << e;
- }
- // If the resolution fails, then this means there is no current
- // machine subvolume (for this bootstrap protocol). In this case we
- // clean up our toolchain subvolume (<name>-<toolchain>) and ignore
- // this machine.
- //
- if (sp.empty ())
- {
- if (te)
- delete_t ();
+ // If the resolution fails, then this means there is no current
+ // machine subvolume (for this bootstrap protocol). In this case we
+ // clean up our toolchain subvolume (<name>-<toolchain>) and ignore
+ // this machine.
+ //
+ if (sp.empty ())
+ {
+ if (te)
+ delete_t ();
- break;
- }
+ l2 ([&]{trace << "skipping " << md << ": no subvolume link";});
+ break;
+ }
- // <name>-<toolchain>-<xxx>
- //
- const dir_path xp (dir_path (md) /=
- path::traits::temp_name (mn + '-' + tc_name));
+ // <name>-<toolchain>-<xxx>
+ //
+ const dir_path xp (dir_path (md) /=
+ path::traits::temp_name (mn + '-' + tc_name));
- // btrfs subvolume snapshot $sp $xp
- if (false)
- {
- if (retry >= 10)
- fail << "unable to snapshot subvolume " << sp;
+ if (btrfs_exit (trace, "subvolume", "snapshot", sp, xp) != 0)
+ {
+ if (retry >= 10)
+ fail << "unable to snapshot subvolume " << sp;
- continue;
- }
+ continue;
+ }
- // Load the (original) machine manifest.
- //
- auto mm (
- parse_manifest<machine_manifest> (sp / "manifest", "machine"));
+ // Load the (original) machine manifest.
+ //
+ auto mm (
+ parse_manifest<machine_manifest> (sp / "manifest", "machine"));
- // If we already have <name>-<toolchain>, see if it needs to be re-
- // bootstrapped. Things that render it obsolete:
- //
- // 1. New machine revision (compare machine ids).
- // 2. New toolchain (compare toolchain ids).
- // 3. New bbot/libbbot (compare versions).
- //
- // The last case has a complication: what should we do if we have
- // bootstrapped a newer version of bbot? This would mean that we are
- // about to be stopped and upgraded (and the upgraded version will
- // probably be able to use the result). So we simply ignore this
- // machine for this run.
+ // If we already have <name>-<toolchain>, see if it needs to be re-
+ // bootstrapped. Things that render it obsolete:
+ //
+ // 1. New machine revision (compare machine ids).
+ // 2. New toolchain (compare toolchain ids).
+ // 3. New bbot/libbbot (compare versions).
+ //
+ // The last case has a complication: what should we do if we have
+ // bootstrapped a newer version of bbot? This would mean that we are
+ // about to be stopped and upgraded (and the upgraded version will
+ // probably be able to use the result). So we simply ignore this
+ // machine for this run.
- // Return -1 if older, 0 if the same, and +1 if newer.
- //
- auto compare_bbot = [] (const bootstrap_manifest& m) -> int
- {
- auto cmp = [&m] (const string& n, uint64_t v) -> int
+ // Return -1 if older, 0 if the same, and +1 if newer.
+ //
+ auto compare_bbot = [] (const bootstrap_manifest& m) -> int
{
- auto i = m.versions.find (n);
+ auto cmp = [&m] (const string& n, uint64_t v) -> int
+ {
+ auto i = m.versions.find (n);
+ return
+ i == m.versions.end () || i->second < v
+ ? -1
+ : i->second > v ? 1 : 0;
+ };
+
+ // Start from the top assuming a new dependency cannot be added
+ // without changing the dependent's version.
+ //
+ int r;
return
- i == m.versions.end () || i->second < v
- ? -1
- : i->second > v ? 1 : 0;
+ (r = cmp ("bbot", BBOT_VERSION)) != 0 ? r :
+ (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r :
+ (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r :
+ (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0;
};
- // Start from the top assuming a new dependency cannot be added
- // without changing the dependent's version.
- //
- int r;
- return
- (r = cmp ("bbot", BBOT_VERSION)) != 0 ? r :
- (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r :
- (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r :
- (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0;
- };
-
- if (te)
- {
- auto bmm (
- parse_manifest<bootstrapped_machine_manifest> (
- tp / "manifest",
- "bootstrapped machine"));
-
- if (bmm.machine.id != mm.id)
+ if (te)
{
- trace << "re-bootstrapping " << tp << ": new machine";
- te = false;
- }
+ auto bmm (
+ parse_manifest<bootstrapped_machine_manifest> (
+ tp / "manifest",
+ "bootstrapped machine"));
- if (bmm.toolchain.id != tc_id)
- {
- trace << "re-bootstrapping " << tp << ": new toolchain";
- te = false;
- }
+ if (bmm.machine.id != mm.id)
+ {
+ l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";});
+ te = false;
+ }
- if (int i = compare_bbot (bmm.bootstrap))
- {
- if (i < 0)
+ if (bmm.toolchain.id != tc_id)
{
- trace << "re-bootstrapping " << tp << ": new bbot";
+ l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";});
te = false;
}
- else
+
+ if (int i = compare_bbot (bmm.bootstrap))
{
- trace << "ignoring " << tp << ": newer bbot";
- // btrfs subvolume snapshot $xp
- break;
+ if (i < 0)
+ {
+ l2 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";});
+ te = false;
+ }
+ else
+ {
+ l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
+ btrfs (trace, "subvolume", "delete", xp);
+ break;
+ }
}
+
+ if (!te)
+ delete_t ();
}
+ else
+ l2 ([&]{trace << "bootstrapping " << tp;});
if (!te)
- delete_t ();
- }
-
- if (!te)
- {
- // Use the <name>-<toolchain>-<xxx> snapshot that we have made to
- // bootstrap the new machine. Then atomically rename it to
- // <name>-<toolchain>.
- //
- bootstrapped_machine_manifest bmm (bootstrap_machine (xp));
-
- try
- {
- mvdir (xp, tp);
- }
- catch (const system_error& e)
{
- fail << "unable to rename " << xp << " to " << tp;
- }
+ // Use the <name>-<toolchain>-<xxx> snapshot that we have made to
+ // bootstrap the new machine. Then atomically rename it to
+ // <name>-<toolchain>.
+ //
+ bootstrapped_machine_manifest bmm (bootstrap_machine (xp, mm));
- te = true;
+ try
+ {
+ mvdir (xp, tp);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to rename " << xp << " to " << tp;
+ }
- // Check the boostrapped bbot version as above and ignore this
- // machine if it's newer than us.
- //
- if (int i = compare_bbot (bmm.bootstrap))
- {
- assert (i > 0);
- trace << "ignoring " << tp << ": newer bbot";
- break;
+ te = true;
+
+ // Check the boostrapped bbot version as above and ignore this
+ // machine if it's newer than us.
+ //
+ if (int i = compare_bbot (bmm.bootstrap))
+ {
+ assert (i > 0);
+ l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
+ break;
+ }
}
- }
- else
- ;// btrfs subvolume snapshot $xp
+ else
+ btrfs (trace, "subvolume", "delete", xp);
- // Add the machine to the list.
- //
- // In order not to forget to clear new fields, we are instead going to
- // create a new instance with just the required fields.
- //
- r.push_back (machine_manifest (mm.id, mm.name, mm.summary));
+ // Add the machine to the list.
+ //
+ // In order not to forget to clear new fields, we are instead going
+ // to create a new instance with just the required fields.
+ //
+ r.push_back (machine_manifest (mm.id, mm.name, mm.summary));
- break;
+ break;
+ }
}
}
+ catch (const system_error& e)
+ {
+ fail << "unable to iterate over " << vd << ": " << e << endf;
+ }
}
return r;
}
-
-*/
+catch (const system_error& e)
+{
+ fail << "unable to iterate over " << rd << ": " << e << endf;
+}
extern "C" void
handle_signal (int sig)
@@ -339,7 +360,9 @@ main (int argc, char* argv[])
try
{
cli::argv_scanner scan (argc, argv, true);
- agent_options ops (scan);
+ ops.parse (scan);
+
+ verb = ops.verbose ();
if (ops.systemd_daemon ())
{
@@ -359,6 +382,11 @@ try
warn.type_ = "<4>";
info.type_ = "<6>";
trace_type = "<7>";
+
+ info << "bbot agent for " << tc_name << '/' << tc_num <<
+ info << "toolchain id " << tc_id <<
+ info << "CPU(s) " << ops.cpu () <<
+ info << "RAM(kB) " << ops.ram ();
}
tracer trace ("main");
@@ -412,16 +440,28 @@ try
fail << "unable to set signal handler: "
<< system_error (errno, generic_category ()); // Sanitize.
- info << "bbot agent for " << tc_name << '/' << tc_num <<
- info << "toolchain id " << tc_id <<
- info << "CPU(s) " << ops.cpu () <<
- info << "RAM(kB) " << ops.ram ();
-
- for (;;)
+ // The work loop. The steps we go through are:
+ //
+ // 1. Enumerate the available machines, (re-)bootstrapping any of necessary.
+ //
+ // 2. Poll controller(s) for build tasks.
+ //
+ // 3. If no build tasks are available, go to #1 after sleeping a bit.
+ //
+ // 4. If a build task is returned, do it, upload the result, and go to #1
+ // immediately.
+ //
+ for (unsigned int s; (s = 60); sleep (s))
{
- error << "sleeping" <<
- warn << "lightly";
- sleep (10);
+ machine_manifests mms (enumerate_machines (ops.machines ()));
+
+ if (ops.dump_machines ())
+ {
+ for (const machine_manifest& mm: mms)
+ serialize_manifest (mm, cout, "stdout", "machine manifest");
+
+ return 0;
+ }
}
}
catch (const failed&)
diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service
index 496692f..f8349ff 100644
--- a/bbot/bbot-agent@.service
+++ b/bbot/bbot-agent@.service
@@ -7,12 +7,14 @@ Type=simple
Environment=CPU=1
Environment=RAM=1048576
+Environment=VERBOSE=1
Environment=TOOLCHAIN_ID=123abc
Environment=TOOLCHAIN_NUM=1
ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \
--cpu ${CPU} \
--ram ${RAM} \
+ --verbose ${VERBOSE} \
%i \
${TOOLCHAIN_NUM} \
${TOOLCHAIN_ID}
diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest
index c29406a..6007c6e 100644
--- a/bbot/bootstrap-manifest
+++ b/bbot/bootstrap-manifest
@@ -26,6 +26,9 @@ namespace bbot
//
string id;
+ explicit
+ toolchain_manifest (string i): id (i) {}
+
public:
toolchain_manifest () = default; // VC export.
toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false);
@@ -51,7 +54,12 @@ namespace bbot
// libbbot-version: 1010100 # 1.1.1
// bbot-version: 1010200 # 1.1.2
//
- std::map<string, uint64_t> versions;
+ using versions_type = std::map<string, uint64_t>;
+ versions_type versions;
+
+ explicit
+ bootstrap_manifest (versions_type v)
+ : versions (move (v)) {}
public:
bootstrap_manifest () = default; // VC export.
@@ -74,6 +82,11 @@ namespace bbot
toolchain_manifest toolchain;
bootstrap_manifest bootstrap;
+ bootstrapped_machine_manifest (machine_manifest m,
+ toolchain_manifest t,
+ bootstrap_manifest b)
+ : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {}
+
public:
bootstrapped_machine_manifest () = default; // VC export.
bootstrapped_machine_manifest (butl::manifest_parser&,
diff --git a/bbot/buildfile b/bbot/buildfile
index 4045888..8c1e5b4 100644
--- a/bbot/buildfile
+++ b/bbot/buildfile
@@ -30,7 +30,7 @@ if ($cxx.target.class == "linux")
{hxx cxx}{ diagnostics } \
{hxx }{ types } \
{hxx cxx}{ types-parsers } \
- {hxx cxx}{ utility } \
+ {hxx txx cxx}{ utility } \
{hxx }{ version-impl } \
$libs
}
@@ -43,7 +43,7 @@ exe{bbot-worker}: \
{hxx cxx}{ diagnostics } \
{hxx }{ types } \
{hxx cxx}{ types-parsers } \
-{hxx cxx}{ utility } \
+{hxx txx cxx}{ utility } \
{hxx }{ version-impl } \
$libs
@@ -57,7 +57,7 @@ if $cli.configured
cli.options += -I $src_root --include-with-brackets --include-prefix bbot \
--guard-prefix BBOT --cxx-prologue "#include <bbot/types-parsers>" \
---cli-namespace bbot::cli --generate-specifier
+--cli-namespace bbot::cli --generate-specifier --generate-parse
cli.cxx{common-options}: cli.options = $cli.options # No usage.
diff --git a/bbot/diagnostics b/bbot/diagnostics
index 6699c45..ebf658d 100644
--- a/bbot/diagnostics
+++ b/bbot/diagnostics
@@ -7,8 +7,7 @@
#include <butl/diagnostics>
-#include <bbot/types>
-#include <bbot/utility>
+#include <bbot/types> // Note: not <bbot/utility>.
namespace bbot
{
@@ -106,7 +105,12 @@ namespace bbot
trace_mark_base (const char* name, const void* data = nullptr);
};
using trace_mark = butl::diag_mark<trace_mark_base>;
- using tracer = trace_mark;
+
+ // using tracer = trace_mark;
+ class tracer: public trace_mark
+ {
+ public: using trace_mark::trace_mark;
+ };
// fail
//
@@ -144,7 +148,7 @@ namespace bbot
using fail_end = butl::diag_noreturn_end<fail_end_base>;
extern fail_mark fail;
- extern const fail_end endf;
+ extern const fail_end endf;
}
#endif // BBOT_DIAGNOSTICS
diff --git a/bbot/diagnostics.cxx b/bbot/diagnostics.cxx
index 5c356f5..2779770 100644
--- a/bbot/diagnostics.cxx
+++ b/bbot/diagnostics.cxx
@@ -4,6 +4,8 @@
#include <bbot/diagnostics>
+#include <bbot/utility>
+
using namespace std;
using namespace butl;
diff --git a/bbot/utility b/bbot/utility
index 81c6c87..ebab971 100644
--- a/bbot/utility
+++ b/bbot/utility
@@ -13,8 +13,8 @@
#include <butl/ft/lang>
+#include <butl/process>
#include <butl/utility> // casecmp(), reverse_iterate(), etc
-
#include <butl/filesystem>
#include <bbot/types>
@@ -39,6 +39,51 @@ namespace bbot
using butl::exception_guard;
using butl::make_exception_guard;
+
+ // Process execution.
+ //
+ class tracer;
+
+ template <typename I, typename O, typename E, typename P, typename... A>
+ void
+ run (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+
+ template <typename I, typename O, typename E, typename P, typename... A>
+ butl::process_exit::code_type
+ run_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+
+ template <typename P, typename... A>
+ inline void
+ run (tracer& t, P&& p, A&&... a)
+ {
+ run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+ }
+
+ template <typename P, typename... A>
+ inline butl::process_exit::code_type
+ run_exit (tracer& t, P&& p, A&&... a)
+ {
+ return run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+ }
+
+ // Manifest parsing and serialization.
+ //
+ template <typename T>
+ T
+ parse_manifest (const path&, const char* what, bool ignore_unknown = true);
+
+ template <typename T>
+ void
+ serialize_manifest (const T&, const path&, const char* what);
+
+ template <typename T>
+ void
+ serialize_manifest (const T&,
+ ostream&,
+ const string& name,
+ const char* what);
}
+#include <bbot/utility.txx>
+
#endif // BBOT_UTILITY
diff --git a/bbot/utility.txx b/bbot/utility.txx
new file mode 100644
index 0000000..d641612
--- /dev/null
+++ b/bbot/utility.txx
@@ -0,0 +1,144 @@
+// file : bbot/utility.txx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <butl/fdstream>
+
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/diagnostics>
+
+namespace bbot
+{
+ template <typename I, typename O, typename E, typename P, typename... A>
+ butl::process_exit::code_type
+ run_exit (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+ {
+ using namespace butl;
+
+ try
+ {
+ auto cmdc = [&t] (const char* cmd[], size_t n)
+ {
+ if (verb >= 2)
+ {
+ diag_record dr (t);
+ process::print (dr.os, cmd, n);
+ }
+ };
+
+ process_exit r (process_run (cmdc,
+ forward<I> (in),
+ forward<O> (out),
+ forward<E> (err),
+ dir_path (),
+ p,
+ forward<A> (args)...));
+
+ if (!r.normal ())
+ fail << "process " << p << " terminated abnormally: "
+ << r.description ();
+
+ return r.code ();
+ }
+ catch (const process_error& e)
+ {
+ fail << "unable to execute " << p << ": " << e << endf;
+ }
+ }
+
+ template <typename I, typename O, typename E, typename P, typename... A>
+ void
+ run (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+ {
+ if (run_exit (t,
+ forward<I> (in),
+ forward<O> (out),
+ forward<E> (err),
+ p,
+ forward<A> (args)...) != 0)
+ fail << "process " << p << " terminated with non-zero exit code";
+ }
+
+ template <typename T>
+ T
+ parse_manifest (const path& f, const char* what, bool ignore_unknown)
+ {
+ using namespace butl;
+
+ try
+ {
+ if (!file_exists (f))
+ fail << what << " manifest file " << f << " does not exist";
+
+ ifdstream ifs (f);
+ manifest_parser p (ifs, f.string ());
+ return T (p, ignore_unknown);
+ }
+ catch (const manifest_parsing& e)
+ {
+ fail << "invalid " << what << " manifest: "
+ << f << ':' << e.line << ':' << e.column << ": " << e.description
+ << endf;
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to read " << what << " manifest " << f << ": " << e
+ << endf;
+ }
+ catch (const system_error& e) // EACCES, etc.
+ {
+ fail << "unable to access " << what << " manifest " << f << ": " << e
+ << endf;
+ }
+ }
+
+ template <typename T>
+ void
+ serialize_manifest (const T& m, const path& f, const char* what)
+ {
+ using namespace std;
+ using namespace butl;
+
+ try
+ {
+ ofdstream ofs (f, ios::binary);
+ auto_rmfile arm (f); // Try to remove on failure ignoring errors.
+
+ serialize_manifest (m, ofs, f.string (), what);
+
+ ofs.close ();
+ arm.cancel ();
+ }
+ catch (const system_error& e) // EACCES, etc.
+ {
+ fail << "unable to access " << what << " manifest " << f << ": " << e;
+ }
+ }
+
+ template <typename T>
+ void
+ serialize_manifest (const T& m,
+ ostream& os,
+ const string& name,
+ const char* what)
+ {
+ using namespace std;
+ using namespace butl;
+
+ try
+ {
+ manifest_serializer s (os, name);
+ m.serialize (s);
+ }
+ catch (const manifest_serialization& e)
+ {
+ fail << "invalid " << what << " manifest: " << e.description;
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to write " << what << " manifest " << name << ": " << e;
+ }
+ }
+}
diff --git a/tests/agent/btrfs-cpdir b/tests/agent/btrfs-cpdir
new file mode 100755
index 0000000..d2dbd43
--- /dev/null
+++ b/tests/agent/btrfs-cpdir
@@ -0,0 +1,113 @@
+#! /usr/bin/env bash
+
+# Copy a directory on (the same) btrfs filesystem. Inside, subvolumes
+# are copied with btrfs subvolume snapshot and everything else with
+# cp --reflink=always.
+#
+# If the -f option is specified and <dst> exists, it is first removed by
+# calling btrfs-rmdir (which is expected to be found next to btrfs-cpdir).
+#
+# Notes:
+#
+# 1. <src> should not be a subvolume (use snapshot directly in this case).
+#
+# 2. Copying of symlinks is not supported (symlinks in submodules are ok).
+#
+# 3. Read-only subvolumes are snapshotted as read-only.
+#
+# Note also that <src> is clones as <dst>, not into <dst> (i.e., like cp -T).
+#
+usage="usage: $0 [-f] <src>/ <dst>/"
+
+owd="$(pwd)"
+trap "{ cd '$owd'; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+force=
+
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -f)
+ shift
+ force="true"
+ ;;
+ --)
+ break
+ ;;
+ -*)
+ error "unknown option: $1"
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+src="${1%/}"
+dst="${2%/}"
+
+if [ -z "$src" -o -z "$dst" ]; then
+ error "$usage"
+fi
+
+shopt -s nullglob dotglob
+
+function cp_dir () # <src> <dst>
+{
+ local src="$1"
+ local dst="$2"
+
+ mkdir "$dst"
+
+ local s d
+ for s in "$src"/*; do
+
+ d="$dst/${s#$src/}"
+
+ if [ -f "$s" ]; then
+ cp -p --reflink=always "$s" "$d"
+ continue
+ fi
+
+ if [ -d "$s" ]; then
+
+ # See if this is a subvolume: btrfs subvolume list requires root
+ # priviliges so we use the inode number which for subvolumes is always
+ # 256.
+ #
+ if [ "$(stat --format=%i "$s")" -eq 256 ]; then
+ cp_subvol "$s" "$d"
+ else
+ cp_dir "$s" "$d"
+ fi
+ continue
+ fi
+
+ error "$s is not a file/directory, not supported"
+ done
+
+ chmod --reference="$src" "$dst"
+ chown --reference="$src" "$dst"
+}
+
+function cp_subvol () # <src> <dst>
+{
+ local src="$1"
+ local dst="$2"
+
+ local o=()
+ if [ "$(btrfs property get -ts "$src" ro)" = "ro=true" ]; then
+ o+=(-r)
+ fi
+
+ btrfs subvolume snapshot "${o[@]}" "$src" "$dst" >/dev/null
+}
+
+if [ -d "$dst" -a -n "$force" ]; then
+ "$(dirname "$(realpath ${BASH_SOURCE[0]})")/btrfs-rmdir" "$dst"
+fi
+
+cp_dir "$src" "$dst"
diff --git a/tests/agent/btrfs-rmdir b/tests/agent/btrfs-rmdir
new file mode 100755
index 0000000..cc51dc2
--- /dev/null
+++ b/tests/agent/btrfs-rmdir
@@ -0,0 +1,66 @@
+#! /usr/bin/env bash
+
+# Remove a directory on a btrfs filesystem. Inside, subvolumes are removed
+# with btrfs subvolume delete and everything else with rm.
+#
+# Notes:
+#
+# 1. <dir> should not be a subvolume (use delete directly in this case).
+#
+# 2. Read-only subvolumes are changed to read-write before deleting.
+#
+usage="usage: $0 <dir>/"
+
+owd="$(pwd)"
+trap "{ cd '$owd'; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+dir="${1%/}"
+
+if [ -z "$dir" ]; then
+ error "$usage"
+fi
+
+shopt -s nullglob dotglob
+
+function rm_dir () # <dir>
+{
+ local dir="$1"
+
+ local p
+ for p in "$dir"/*; do
+
+ if [ -d "$p" -a ! -L "$p" ]; then
+
+ # See if this is a subvolume: btrfs subvolume list requires root
+ # priviliges so we use the inode number which for subvolumes is always
+ # 256.
+ #
+ if [ "$(stat --format=%i "$p")" -eq 256 ]; then
+ rm_subvol "$p"
+ else
+ rm_dir "$p"
+ fi
+ else
+ rm "$p"
+ fi
+ done
+
+ rmdir "$dir"
+}
+
+function rm_subvol () # <dir>
+{
+ local dir="$1"
+
+ if [ "$(btrfs property get -ts "$dir" ro)" = "ro=true" ]; then
+ btrfs property set -ts "$dir" ro false >/dev/null
+ fi
+
+ btrfs subvolume delete "$dir" >/dev/null
+}
+
+rm_dir "$dir"
diff --git a/tests/agent/buildfile b/tests/agent/buildfile
new file mode 100644
index 0000000..1c6d59b
--- /dev/null
+++ b/tests/agent/buildfile
@@ -0,0 +1,8 @@
+# file : tests/agent/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+./: ../../bbot/exe{bbot-agent} test{testscript} file{btrfs-cpdir btrfs-rmdir}
+dir{./}: test = ../../bbot/exe{bbot-agent}
+
+include ../../bbot/
diff --git a/tests/agent/testscript b/tests/agent/testscript
new file mode 100644
index 0000000..fa465a4
--- /dev/null
+++ b/tests/agent/testscript
@@ -0,0 +1,112 @@
+# file : tests/agent/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# The /build/machines directory should be on a btrfs filesystem and have the
+# following layout and contents:
+#
+# /build/machines/
+# └── default/
+# └── linux-gcc/
+# ├── linux-gcc-1.0/
+# │ └── manifest
+# └── linux-gcc-1.1/
+# └── manifest
+#
+# Notes:
+#
+# - The test must be run serially (@@ TODO: serial directive)
+
+test.options = --verbose 2
+test.arguments = stage 1
+
+cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines
+rm = $src_base/btrfs-rmdir /build/machines
+
+: dump-machines
+:
+{
+ m = /build/machines/default/linux-gcc
+
+ test.options += --dump-machines --fake-bootstrap
+
+ +$cp
+
+ : no-current-machine-symlink
+ :
+ $* 123 2>>"EOE"
+ trace: enumerate_machines: skipping $m/: no subvolume link
+ EOE
+
+ ln -T -s linux-gcc-1.0 $m/linux-gcc-1
+
+ : bootstrap
+ :
+ $* 123 >>EOO 2>>~"%EOE%d"
+ : 1
+ id: linux-gcc-1.0
+ name: linux-gcc
+ summary: Linux with GCC
+ EOO
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+%
+ trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/
+ EOE
+
+ ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1
+
+ : re-bootstrap-machine
+ :
+ $* 123 >>EOO 2>>~"%EOE%d"
+ : 1
+ id: linux-gcc-1.1
+ name: linux-gcc
+ summary: Linux with GCC
+ EOO
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ EOE
+
+ : re-bootstrap-toolchain
+ :
+ $* 124 >>EOO 2>>~"%EOE%d"
+ : 1
+ id: linux-gcc-1.1
+ name: linux-gcc
+ summary: Linux with GCC
+ EOO
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ EOE
+
+ sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest
+
+ : re-bootstrap-bbot
+ :
+ $* 124 >>EOO 2>>~"%EOE%d"
+ : 1
+ id: linux-gcc-1.1
+ name: linux-gcc
+ summary: Linux with GCC
+ EOO
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+ trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot
+ trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+ trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+ EOE
+
+ sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest
+
+ : re-bootstrap-bbot-newer
+ :
+ $* 124 2>>~"%EOE%d"
+ %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+ trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot
+ %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+%
+ EOE
+
+ -$rm
+}