diff options
| -rw-r--r-- | bbot/agent.cli | 36 | ||||
| -rw-r--r-- | bbot/agent.cxx | 486 | ||||
| -rw-r--r-- | bbot/bbot-agent@.service | 2 | ||||
| -rw-r--r-- | bbot/bootstrap-manifest | 15 | ||||
| -rw-r--r-- | bbot/buildfile | 6 | ||||
| -rw-r--r-- | bbot/diagnostics | 12 | ||||
| -rw-r--r-- | bbot/diagnostics.cxx | 2 | ||||
| -rw-r--r-- | bbot/utility | 47 | ||||
| -rw-r--r-- | bbot/utility.txx | 144 | ||||
| -rwxr-xr-x | tests/agent/btrfs-cpdir | 113 | ||||
| -rwxr-xr-x | tests/agent/btrfs-rmdir | 66 | ||||
| -rw-r--r-- | tests/agent/buildfile | 8 | ||||
| -rw-r--r-- | tests/agent/testscript | 112 |
13 files changed, 813 insertions, 236 deletions
diff --git a/bbot/agent.cli b/bbot/agent.cli index c4ee356..7a6c201 100644 --- a/bbot/agent.cli +++ b/bbot/agent.cli @@ -37,6 +37,9 @@ namespace bbot { "\h|OPTIONS|" + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} + bool --systemd-daemon { "Start as a simple systemd daemon." @@ -46,16 +49,41 @@ namespace bbot { "<num>", "Number of CPUs (threads) to use, 1 by default." - }; + } size_t --ram (1024 * 1024) // 1G { "<num>", "Amount of RAM (in kB) to use, 1G by default." - }; + } - bool --help {"Print usage information and exit."} - bool --version {"Print version and exit."} + dir_path --machines = "/build/machines/" + { + "<dir>", + "The location of the build machines with \cb{/build/machines/} being + the default." + } + + uint16_t --verbose = 1 + { + "<level>", + "Set the diagnostics verbosity to <level> between 0 and 6 with level 1 + being the default." + } + + // Testing options. + // + bool --dump-machines + { + "Dump available machines to \c{stdout}, (re)-bootstrapping any if + necessary, and exit." + } + + bool --fake-bootstrap + { + "Fake the machine bootstrap process by creating the expected bootstrapped + machine manifest." + } }; " diff --git a/bbot/agent.cxx b/bbot/agent.cxx index 3e4f8dc..76c3a86 100644 --- a/bbot/agent.cxx +++ b/bbot/agent.cxx @@ -9,12 +9,8 @@ #include <iostream> #include <butl/pager> -#include <butl/fdstream> #include <butl/filesystem> // dir_iterator -#include <butl/manifest-parser> -#include <butl/manifest-serializer> - #include <bbot/manifest> #include <bbot/types> @@ -29,50 +25,64 @@ using namespace std; using namespace butl; using namespace bbot; +// The btrfs tool likes to print informational messages, like "Created +// snapshot such and such". Luckily, it writes them to stdout while proper +// diagnostics to stderr. +// +template <typename... A> +inline void +btrfs (tracer& t, A&&... a) +{ + if (verb >= 3) + run (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...); + else + run (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...); +} + +template <typename... A> +inline butl::process_exit::code_type +btrfs_exit (tracer& t, A&&... a) +{ + return verb >= 3 + ? run_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...) + : run_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...); +} + +agent_options ops; + const string bs_prot ("1"); // Bootstrap protocol version. string tc_name; // Toolchain name. string tc_num; // Toolchain number. string tc_id; // Toolchain id. -template <typename T> -static T -parse_manifest (const path& f, const char* what, bool ignore_unknown = true) +static bootstrapped_machine_manifest +bootstrap_machine (const dir_path& md, const machine_manifest& mm) { - try - { - if (!file_exists (f)) - fail << what << " manifest file " << f << " does not exist"; + bootstrapped_machine_manifest r { + mm, + toolchain_manifest {tc_id}, + bootstrap_manifest { + bootstrap_manifest::versions_type { + {"bbot", BBOT_VERSION}, + {"libbbot", LIBBBOT_VERSION}, + {"libbpkg", LIBBPKG_VERSION}, + {"libbutl", LIBBUTL_VERSION} + } + } + }; - ifdstream ifs (f); - manifest_parser mp (ifs, f.string ()); - return T (mp, ignore_unknown); - } - catch (const manifest_parsing& e) - { - fail << "invalid " << what << " manifest: " - << f << ':' << e.line << ':' << e.column << ": " << e.description - << endf; - } - catch (const io_error& e) - { - fail << "unable to read " << what << " manifest " << f << ": " << e - << endf; - } - catch (const system_error& e) // EACCES, etc. + if (!ops.fake_bootstrap ()) { - fail << "unable to access " << what << " manifest " << f << ": " << e - << endf; } -} - -/* -static bootstrapped_machine_manifest -bootstrap_machine (const dir_path&); + serialize_manifest (r, md / "manifest", "bootstrapped machine"); + return r; +} static machine_manifests enumerate_machines (const dir_path& rd) +try { tracer trace ("enumerate_machines"); @@ -93,231 +103,242 @@ enumerate_machines (const dir_path& rd) // Inside we have machines. // - for (const dir_entry& me: dir_iterator (vd)) + try { - const string mn (me.path ().string ()); - - if (me.type () != entry_type::directory || mn[0] == '.') - continue; - - const dir_path md (dir_path (vd) /= mn); - - // Our endgoal here is to obtain a bootstrapped snapshot of this machine - // while watching out for potential race conditions (machines being - // added/upgraded/removed; see the manual for details). - // - // So here is our overall plan: - // - // 1. Resolve current subvolume link for our bootstrap protocol. - // - // 2. If there is no link, cleanup and ignore this machine. - // - // 3. Try to create a snapshot of current subvolume (this operation is - // atomic). If failed (e.g., someone changed the link and removed the - // subvolume in the meantime), retry from #1. - // - // 4. Compare the snapshot to the already bootstrapped version (if any) - // and see if we need to re-bootstrap. If so, use the snapshot as a - // starting point. Rename to bootstrapped at the end (atomic). - // - const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P> - const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<toolc...> - bool te (dir_exists (tp)); - - auto delete_t = [&tp] () + for (const dir_entry& me: dir_iterator (vd)) { - // btrfs property set -ts $tp ro false - // btrfs subvolume delete $tp - }; + const string mn (me.path ().string ()); - for (size_t retry (0);; ++retry) - { - if (retry != 0) - sleep (1); + if (me.type () != entry_type::directory || mn[0] == '.') + continue; + + const dir_path md (dir_path (vd) /= mn); - // Resolve the link to subvolume path. + // Our endgoal here is to obtain a bootstrapped snapshot of this + // machine while watching out for potential race conditions (machines + // being added/upgraded/removed; see the manual for details). // - dir_path sp; // <name>-<P>.<R> - try + // So here is our overall plan: + // + // 1. Resolve current subvolume link for our bootstrap protocol. + // + // 2. If there is no link, cleanup and ignore this machine. + // + // 3. Try to create a snapshot of current subvolume (this operation is + // atomic). If failed (e.g., someone changed the link and removed + // the subvolume in the meantime), retry from #1. + // + // 4. Compare the snapshot to the already bootstrapped version (if + // any) and see if we need to re-bootstrap. If so, use the snapshot + // as a starting point. Rename to bootstrapped at the end (atomic). + // + const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P> + const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<too...> + bool te (dir_exists (tp)); + + auto delete_t = [&tp, &trace] () + { + btrfs (trace, "property", "set", "-ts", tp, "ro", "false"); + btrfs (trace, "subvolume", "delete", tp); + }; + + for (size_t retry (0);; ++retry) { - char b [PATH_MAX + 1]; - ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b))); + if (retry != 0) + sleep (1); - if (r == -1) + // Resolve the link to subvolume path. + // + dir_path sp; // <name>-<P>.<R> + try { - if (errno != ENOENT) - throw_generic_error (errno); + char b [PATH_MAX + 1]; + ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b))); + + if (r == -1) + { + if (errno != ENOENT) + throw_generic_error (errno); + } + else if (static_cast<size_t> (r) >= sizeof (b)) + throw_generic_error (EINVAL); + else + { + b[r] = '\0'; + sp = dir_path (b); + if (sp.relative ()) + sp = md / sp; + } } - else if (static_cast<size_t> (r) >= sizeof (b)) - throw_generic_error (EINVAL); - else + catch (const system_error& e) { - b[r] = '\0'; - sp = dir_path (b); - if (sp.relative ()) - sp = md / sp; + fail << "unable to read subvolume link " << lp << ": " << e; } - } - catch (const system_error& e) - { - fail << "unable to read subvolume link " << lp << ": " << e; - } - // If the resolution fails, then this means there is no current - // machine subvolume (for this bootstrap protocol). In this case we - // clean up our toolchain subvolume (<name>-<toolchain>) and ignore - // this machine. - // - if (sp.empty ()) - { - if (te) - delete_t (); + // If the resolution fails, then this means there is no current + // machine subvolume (for this bootstrap protocol). In this case we + // clean up our toolchain subvolume (<name>-<toolchain>) and ignore + // this machine. + // + if (sp.empty ()) + { + if (te) + delete_t (); - break; - } + l2 ([&]{trace << "skipping " << md << ": no subvolume link";}); + break; + } - // <name>-<toolchain>-<xxx> - // - const dir_path xp (dir_path (md) /= - path::traits::temp_name (mn + '-' + tc_name)); + // <name>-<toolchain>-<xxx> + // + const dir_path xp (dir_path (md) /= + path::traits::temp_name (mn + '-' + tc_name)); - // btrfs subvolume snapshot $sp $xp - if (false) - { - if (retry >= 10) - fail << "unable to snapshot subvolume " << sp; + if (btrfs_exit (trace, "subvolume", "snapshot", sp, xp) != 0) + { + if (retry >= 10) + fail << "unable to snapshot subvolume " << sp; - continue; - } + continue; + } - // Load the (original) machine manifest. - // - auto mm ( - parse_manifest<machine_manifest> (sp / "manifest", "machine")); + // Load the (original) machine manifest. + // + auto mm ( + parse_manifest<machine_manifest> (sp / "manifest", "machine")); - // If we already have <name>-<toolchain>, see if it needs to be re- - // bootstrapped. Things that render it obsolete: - // - // 1. New machine revision (compare machine ids). - // 2. New toolchain (compare toolchain ids). - // 3. New bbot/libbbot (compare versions). - // - // The last case has a complication: what should we do if we have - // bootstrapped a newer version of bbot? This would mean that we are - // about to be stopped and upgraded (and the upgraded version will - // probably be able to use the result). So we simply ignore this - // machine for this run. + // If we already have <name>-<toolchain>, see if it needs to be re- + // bootstrapped. Things that render it obsolete: + // + // 1. New machine revision (compare machine ids). + // 2. New toolchain (compare toolchain ids). + // 3. New bbot/libbbot (compare versions). + // + // The last case has a complication: what should we do if we have + // bootstrapped a newer version of bbot? This would mean that we are + // about to be stopped and upgraded (and the upgraded version will + // probably be able to use the result). So we simply ignore this + // machine for this run. - // Return -1 if older, 0 if the same, and +1 if newer. - // - auto compare_bbot = [] (const bootstrap_manifest& m) -> int - { - auto cmp = [&m] (const string& n, uint64_t v) -> int + // Return -1 if older, 0 if the same, and +1 if newer. + // + auto compare_bbot = [] (const bootstrap_manifest& m) -> int { - auto i = m.versions.find (n); + auto cmp = [&m] (const string& n, uint64_t v) -> int + { + auto i = m.versions.find (n); + return + i == m.versions.end () || i->second < v + ? -1 + : i->second > v ? 1 : 0; + }; + + // Start from the top assuming a new dependency cannot be added + // without changing the dependent's version. + // + int r; return - i == m.versions.end () || i->second < v - ? -1 - : i->second > v ? 1 : 0; + (r = cmp ("bbot", BBOT_VERSION)) != 0 ? r : + (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r : + (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r : + (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0; }; - // Start from the top assuming a new dependency cannot be added - // without changing the dependent's version. - // - int r; - return - (r = cmp ("bbot", BBOT_VERSION)) != 0 ? r : - (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r : - (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r : - (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0; - }; - - if (te) - { - auto bmm ( - parse_manifest<bootstrapped_machine_manifest> ( - tp / "manifest", - "bootstrapped machine")); - - if (bmm.machine.id != mm.id) + if (te) { - trace << "re-bootstrapping " << tp << ": new machine"; - te = false; - } + auto bmm ( + parse_manifest<bootstrapped_machine_manifest> ( + tp / "manifest", + "bootstrapped machine")); - if (bmm.toolchain.id != tc_id) - { - trace << "re-bootstrapping " << tp << ": new toolchain"; - te = false; - } + if (bmm.machine.id != mm.id) + { + l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";}); + te = false; + } - if (int i = compare_bbot (bmm.bootstrap)) - { - if (i < 0) + if (bmm.toolchain.id != tc_id) { - trace << "re-bootstrapping " << tp << ": new bbot"; + l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";}); te = false; } - else + + if (int i = compare_bbot (bmm.bootstrap)) { - trace << "ignoring " << tp << ": newer bbot"; - // btrfs subvolume snapshot $xp - break; + if (i < 0) + { + l2 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";}); + te = false; + } + else + { + l2 ([&]{trace << "ignoring " << tp << ": old bbot";}); + btrfs (trace, "subvolume", "delete", xp); + break; + } } + + if (!te) + delete_t (); } + else + l2 ([&]{trace << "bootstrapping " << tp;}); if (!te) - delete_t (); - } - - if (!te) - { - // Use the <name>-<toolchain>-<xxx> snapshot that we have made to - // bootstrap the new machine. Then atomically rename it to - // <name>-<toolchain>. - // - bootstrapped_machine_manifest bmm (bootstrap_machine (xp)); - - try - { - mvdir (xp, tp); - } - catch (const system_error& e) { - fail << "unable to rename " << xp << " to " << tp; - } + // Use the <name>-<toolchain>-<xxx> snapshot that we have made to + // bootstrap the new machine. Then atomically rename it to + // <name>-<toolchain>. + // + bootstrapped_machine_manifest bmm (bootstrap_machine (xp, mm)); - te = true; + try + { + mvdir (xp, tp); + } + catch (const system_error& e) + { + fail << "unable to rename " << xp << " to " << tp; + } - // Check the boostrapped bbot version as above and ignore this - // machine if it's newer than us. - // - if (int i = compare_bbot (bmm.bootstrap)) - { - assert (i > 0); - trace << "ignoring " << tp << ": newer bbot"; - break; + te = true; + + // Check the boostrapped bbot version as above and ignore this + // machine if it's newer than us. + // + if (int i = compare_bbot (bmm.bootstrap)) + { + assert (i > 0); + l2 ([&]{trace << "ignoring " << tp << ": old bbot";}); + break; + } } - } - else - ;// btrfs subvolume snapshot $xp + else + btrfs (trace, "subvolume", "delete", xp); - // Add the machine to the list. - // - // In order not to forget to clear new fields, we are instead going to - // create a new instance with just the required fields. - // - r.push_back (machine_manifest (mm.id, mm.name, mm.summary)); + // Add the machine to the list. + // + // In order not to forget to clear new fields, we are instead going + // to create a new instance with just the required fields. + // + r.push_back (machine_manifest (mm.id, mm.name, mm.summary)); - break; + break; + } } } + catch (const system_error& e) + { + fail << "unable to iterate over " << vd << ": " << e << endf; + } } return r; } - -*/ +catch (const system_error& e) +{ + fail << "unable to iterate over " << rd << ": " << e << endf; +} extern "C" void handle_signal (int sig) @@ -339,7 +360,9 @@ main (int argc, char* argv[]) try { cli::argv_scanner scan (argc, argv, true); - agent_options ops (scan); + ops.parse (scan); + + verb = ops.verbose (); if (ops.systemd_daemon ()) { @@ -359,6 +382,11 @@ try warn.type_ = "<4>"; info.type_ = "<6>"; trace_type = "<7>"; + + info << "bbot agent for " << tc_name << '/' << tc_num << + info << "toolchain id " << tc_id << + info << "CPU(s) " << ops.cpu () << + info << "RAM(kB) " << ops.ram (); } tracer trace ("main"); @@ -412,16 +440,28 @@ try fail << "unable to set signal handler: " << system_error (errno, generic_category ()); // Sanitize. - info << "bbot agent for " << tc_name << '/' << tc_num << - info << "toolchain id " << tc_id << - info << "CPU(s) " << ops.cpu () << - info << "RAM(kB) " << ops.ram (); - - for (;;) + // The work loop. The steps we go through are: + // + // 1. Enumerate the available machines, (re-)bootstrapping any of necessary. + // + // 2. Poll controller(s) for build tasks. + // + // 3. If no build tasks are available, go to #1 after sleeping a bit. + // + // 4. If a build task is returned, do it, upload the result, and go to #1 + // immediately. + // + for (unsigned int s; (s = 60); sleep (s)) { - error << "sleeping" << - warn << "lightly"; - sleep (10); + machine_manifests mms (enumerate_machines (ops.machines ())); + + if (ops.dump_machines ()) + { + for (const machine_manifest& mm: mms) + serialize_manifest (mm, cout, "stdout", "machine manifest"); + + return 0; + } } } catch (const failed&) diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service index 496692f..f8349ff 100644 --- a/bbot/bbot-agent@.service +++ b/bbot/bbot-agent@.service @@ -7,12 +7,14 @@ Type=simple Environment=CPU=1 Environment=RAM=1048576 +Environment=VERBOSE=1 Environment=TOOLCHAIN_ID=123abc Environment=TOOLCHAIN_NUM=1 ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \ --cpu ${CPU} \ --ram ${RAM} \ + --verbose ${VERBOSE} \ %i \ ${TOOLCHAIN_NUM} \ ${TOOLCHAIN_ID} diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest index c29406a..6007c6e 100644 --- a/bbot/bootstrap-manifest +++ b/bbot/bootstrap-manifest @@ -26,6 +26,9 @@ namespace bbot // string id; + explicit + toolchain_manifest (string i): id (i) {} + public: toolchain_manifest () = default; // VC export. toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false); @@ -51,7 +54,12 @@ namespace bbot // libbbot-version: 1010100 # 1.1.1 // bbot-version: 1010200 # 1.1.2 // - std::map<string, uint64_t> versions; + using versions_type = std::map<string, uint64_t>; + versions_type versions; + + explicit + bootstrap_manifest (versions_type v) + : versions (move (v)) {} public: bootstrap_manifest () = default; // VC export. @@ -74,6 +82,11 @@ namespace bbot toolchain_manifest toolchain; bootstrap_manifest bootstrap; + bootstrapped_machine_manifest (machine_manifest m, + toolchain_manifest t, + bootstrap_manifest b) + : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {} + public: bootstrapped_machine_manifest () = default; // VC export. bootstrapped_machine_manifest (butl::manifest_parser&, diff --git a/bbot/buildfile b/bbot/buildfile index 4045888..8c1e5b4 100644 --- a/bbot/buildfile +++ b/bbot/buildfile @@ -30,7 +30,7 @@ if ($cxx.target.class == "linux") {hxx cxx}{ diagnostics } \ {hxx }{ types } \ {hxx cxx}{ types-parsers } \ - {hxx cxx}{ utility } \ + {hxx txx cxx}{ utility } \ {hxx }{ version-impl } \ $libs } @@ -43,7 +43,7 @@ exe{bbot-worker}: \ {hxx cxx}{ diagnostics } \ {hxx }{ types } \ {hxx cxx}{ types-parsers } \ -{hxx cxx}{ utility } \ +{hxx txx cxx}{ utility } \ {hxx }{ version-impl } \ $libs @@ -57,7 +57,7 @@ if $cli.configured cli.options += -I $src_root --include-with-brackets --include-prefix bbot \ --guard-prefix BBOT --cxx-prologue "#include <bbot/types-parsers>" \ ---cli-namespace bbot::cli --generate-specifier +--cli-namespace bbot::cli --generate-specifier --generate-parse cli.cxx{common-options}: cli.options = $cli.options # No usage. diff --git a/bbot/diagnostics b/bbot/diagnostics index 6699c45..ebf658d 100644 --- a/bbot/diagnostics +++ b/bbot/diagnostics @@ -7,8 +7,7 @@ #include <butl/diagnostics> -#include <bbot/types> -#include <bbot/utility> +#include <bbot/types> // Note: not <bbot/utility>. namespace bbot { @@ -106,7 +105,12 @@ namespace bbot trace_mark_base (const char* name, const void* data = nullptr); }; using trace_mark = butl::diag_mark<trace_mark_base>; - using tracer = trace_mark; + + // using tracer = trace_mark; + class tracer: public trace_mark + { + public: using trace_mark::trace_mark; + }; // fail // @@ -144,7 +148,7 @@ namespace bbot using fail_end = butl::diag_noreturn_end<fail_end_base>; extern fail_mark fail; - extern const fail_end endf; + extern const fail_end endf; } #endif // BBOT_DIAGNOSTICS diff --git a/bbot/diagnostics.cxx b/bbot/diagnostics.cxx index 5c356f5..2779770 100644 --- a/bbot/diagnostics.cxx +++ b/bbot/diagnostics.cxx @@ -4,6 +4,8 @@ #include <bbot/diagnostics> +#include <bbot/utility> + using namespace std; using namespace butl; diff --git a/bbot/utility b/bbot/utility index 81c6c87..ebab971 100644 --- a/bbot/utility +++ b/bbot/utility @@ -13,8 +13,8 @@ #include <butl/ft/lang> +#include <butl/process> #include <butl/utility> // casecmp(), reverse_iterate(), etc - #include <butl/filesystem> #include <bbot/types> @@ -39,6 +39,51 @@ namespace bbot using butl::exception_guard; using butl::make_exception_guard; + + // Process execution. + // + class tracer; + + template <typename I, typename O, typename E, typename P, typename... A> + void + run (tracer&, I&& in, O&& out, E&& err, P&&, A&&...); + + template <typename I, typename O, typename E, typename P, typename... A> + butl::process_exit::code_type + run_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...); + + template <typename P, typename... A> + inline void + run (tracer& t, P&& p, A&&... a) + { + run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...); + } + + template <typename P, typename... A> + inline butl::process_exit::code_type + run_exit (tracer& t, P&& p, A&&... a) + { + return run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...); + } + + // Manifest parsing and serialization. + // + template <typename T> + T + parse_manifest (const path&, const char* what, bool ignore_unknown = true); + + template <typename T> + void + serialize_manifest (const T&, const path&, const char* what); + + template <typename T> + void + serialize_manifest (const T&, + ostream&, + const string& name, + const char* what); } +#include <bbot/utility.txx> + #endif // BBOT_UTILITY diff --git a/bbot/utility.txx b/bbot/utility.txx new file mode 100644 index 0000000..d641612 --- /dev/null +++ b/bbot/utility.txx @@ -0,0 +1,144 @@ +// file : bbot/utility.txx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <butl/fdstream> + +#include <butl/manifest-parser> +#include <butl/manifest-serializer> + +#include <bbot/diagnostics> + +namespace bbot +{ + template <typename I, typename O, typename E, typename P, typename... A> + butl::process_exit::code_type + run_exit (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args) + { + using namespace butl; + + try + { + auto cmdc = [&t] (const char* cmd[], size_t n) + { + if (verb >= 2) + { + diag_record dr (t); + process::print (dr.os, cmd, n); + } + }; + + process_exit r (process_run (cmdc, + forward<I> (in), + forward<O> (out), + forward<E> (err), + dir_path (), + p, + forward<A> (args)...)); + + if (!r.normal ()) + fail << "process " << p << " terminated abnormally: " + << r.description (); + + return r.code (); + } + catch (const process_error& e) + { + fail << "unable to execute " << p << ": " << e << endf; + } + } + + template <typename I, typename O, typename E, typename P, typename... A> + void + run (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args) + { + if (run_exit (t, + forward<I> (in), + forward<O> (out), + forward<E> (err), + p, + forward<A> (args)...) != 0) + fail << "process " << p << " terminated with non-zero exit code"; + } + + template <typename T> + T + parse_manifest (const path& f, const char* what, bool ignore_unknown) + { + using namespace butl; + + try + { + if (!file_exists (f)) + fail << what << " manifest file " << f << " does not exist"; + + ifdstream ifs (f); + manifest_parser p (ifs, f.string ()); + return T (p, ignore_unknown); + } + catch (const manifest_parsing& e) + { + fail << "invalid " << what << " manifest: " + << f << ':' << e.line << ':' << e.column << ": " << e.description + << endf; + } + catch (const io_error& e) + { + fail << "unable to read " << what << " manifest " << f << ": " << e + << endf; + } + catch (const system_error& e) // EACCES, etc. + { + fail << "unable to access " << what << " manifest " << f << ": " << e + << endf; + } + } + + template <typename T> + void + serialize_manifest (const T& m, const path& f, const char* what) + { + using namespace std; + using namespace butl; + + try + { + ofdstream ofs (f, ios::binary); + auto_rmfile arm (f); // Try to remove on failure ignoring errors. + + serialize_manifest (m, ofs, f.string (), what); + + ofs.close (); + arm.cancel (); + } + catch (const system_error& e) // EACCES, etc. + { + fail << "unable to access " << what << " manifest " << f << ": " << e; + } + } + + template <typename T> + void + serialize_manifest (const T& m, + ostream& os, + const string& name, + const char* what) + { + using namespace std; + using namespace butl; + + try + { + manifest_serializer s (os, name); + m.serialize (s); + } + catch (const manifest_serialization& e) + { + fail << "invalid " << what << " manifest: " << e.description; + } + catch (const io_error& e) + { + fail << "unable to write " << what << " manifest " << name << ": " << e; + } + } +} diff --git a/tests/agent/btrfs-cpdir b/tests/agent/btrfs-cpdir new file mode 100755 index 0000000..d2dbd43 --- /dev/null +++ b/tests/agent/btrfs-cpdir @@ -0,0 +1,113 @@ +#! /usr/bin/env bash + +# Copy a directory on (the same) btrfs filesystem. Inside, subvolumes +# are copied with btrfs subvolume snapshot and everything else with +# cp --reflink=always. +# +# If the -f option is specified and <dst> exists, it is first removed by +# calling btrfs-rmdir (which is expected to be found next to btrfs-cpdir). +# +# Notes: +# +# 1. <src> should not be a subvolume (use snapshot directly in this case). +# +# 2. Copying of symlinks is not supported (symlinks in submodules are ok). +# +# 3. Read-only subvolumes are snapshotted as read-only. +# +# Note also that <src> is clones as <dst>, not into <dst> (i.e., like cp -T). +# +usage="usage: $0 [-f] <src>/ <dst>/" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +force= + +while [ "$#" -gt 0 ]; do + case "$1" in + -f) + shift + force="true" + ;; + --) + break + ;; + -*) + error "unknown option: $1" + ;; + *) + break + ;; + esac +done + +src="${1%/}" +dst="${2%/}" + +if [ -z "$src" -o -z "$dst" ]; then + error "$usage" +fi + +shopt -s nullglob dotglob + +function cp_dir () # <src> <dst> +{ + local src="$1" + local dst="$2" + + mkdir "$dst" + + local s d + for s in "$src"/*; do + + d="$dst/${s#$src/}" + + if [ -f "$s" ]; then + cp -p --reflink=always "$s" "$d" + continue + fi + + if [ -d "$s" ]; then + + # See if this is a subvolume: btrfs subvolume list requires root + # priviliges so we use the inode number which for subvolumes is always + # 256. + # + if [ "$(stat --format=%i "$s")" -eq 256 ]; then + cp_subvol "$s" "$d" + else + cp_dir "$s" "$d" + fi + continue + fi + + error "$s is not a file/directory, not supported" + done + + chmod --reference="$src" "$dst" + chown --reference="$src" "$dst" +} + +function cp_subvol () # <src> <dst> +{ + local src="$1" + local dst="$2" + + local o=() + if [ "$(btrfs property get -ts "$src" ro)" = "ro=true" ]; then + o+=(-r) + fi + + btrfs subvolume snapshot "${o[@]}" "$src" "$dst" >/dev/null +} + +if [ -d "$dst" -a -n "$force" ]; then + "$(dirname "$(realpath ${BASH_SOURCE[0]})")/btrfs-rmdir" "$dst" +fi + +cp_dir "$src" "$dst" diff --git a/tests/agent/btrfs-rmdir b/tests/agent/btrfs-rmdir new file mode 100755 index 0000000..cc51dc2 --- /dev/null +++ b/tests/agent/btrfs-rmdir @@ -0,0 +1,66 @@ +#! /usr/bin/env bash + +# Remove a directory on a btrfs filesystem. Inside, subvolumes are removed +# with btrfs subvolume delete and everything else with rm. +# +# Notes: +# +# 1. <dir> should not be a subvolume (use delete directly in this case). +# +# 2. Read-only subvolumes are changed to read-write before deleting. +# +usage="usage: $0 <dir>/" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +dir="${1%/}" + +if [ -z "$dir" ]; then + error "$usage" +fi + +shopt -s nullglob dotglob + +function rm_dir () # <dir> +{ + local dir="$1" + + local p + for p in "$dir"/*; do + + if [ -d "$p" -a ! -L "$p" ]; then + + # See if this is a subvolume: btrfs subvolume list requires root + # priviliges so we use the inode number which for subvolumes is always + # 256. + # + if [ "$(stat --format=%i "$p")" -eq 256 ]; then + rm_subvol "$p" + else + rm_dir "$p" + fi + else + rm "$p" + fi + done + + rmdir "$dir" +} + +function rm_subvol () # <dir> +{ + local dir="$1" + + if [ "$(btrfs property get -ts "$dir" ro)" = "ro=true" ]; then + btrfs property set -ts "$dir" ro false >/dev/null + fi + + btrfs subvolume delete "$dir" >/dev/null +} + +rm_dir "$dir" diff --git a/tests/agent/buildfile b/tests/agent/buildfile new file mode 100644 index 0000000..1c6d59b --- /dev/null +++ b/tests/agent/buildfile @@ -0,0 +1,8 @@ +# file : tests/agent/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: ../../bbot/exe{bbot-agent} test{testscript} file{btrfs-cpdir btrfs-rmdir} +dir{./}: test = ../../bbot/exe{bbot-agent} + +include ../../bbot/ diff --git a/tests/agent/testscript b/tests/agent/testscript new file mode 100644 index 0000000..fa465a4 --- /dev/null +++ b/tests/agent/testscript @@ -0,0 +1,112 @@ +# file : tests/agent/testscript +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# The /build/machines directory should be on a btrfs filesystem and have the +# following layout and contents: +# +# /build/machines/ +# └── default/ +# └── linux-gcc/ +# ├── linux-gcc-1.0/ +# │ └── manifest +# └── linux-gcc-1.1/ +# └── manifest +# +# Notes: +# +# - The test must be run serially (@@ TODO: serial directive) + +test.options = --verbose 2 +test.arguments = stage 1 + +cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines +rm = $src_base/btrfs-rmdir /build/machines + +: dump-machines +: +{ + m = /build/machines/default/linux-gcc + + test.options += --dump-machines --fake-bootstrap + + +$cp + + : no-current-machine-symlink + : + $* 123 2>>"EOE" + trace: enumerate_machines: skipping $m/: no subvolume link + EOE + + ln -T -s linux-gcc-1.0 $m/linux-gcc-1 + + : bootstrap + : + $* 123 >>EOO 2>>~"%EOE%d" + : 1 + id: linux-gcc-1.0 + name: linux-gcc + summary: Linux with GCC + EOO + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+% + trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/ + EOE + + ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1 + + : re-bootstrap-machine + : + $* 123 >>EOO 2>>~"%EOE%d" + : 1 + id: linux-gcc-1.1 + name: linux-gcc + summary: Linux with GCC + EOO + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + EOE + + : re-bootstrap-toolchain + : + $* 124 >>EOO 2>>~"%EOE%d" + : 1 + id: linux-gcc-1.1 + name: linux-gcc + summary: Linux with GCC + EOO + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + EOE + + sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest + + : re-bootstrap-bbot + : + $* 124 >>EOO 2>>~"%EOE%d" + : 1 + id: linux-gcc-1.1 + name: linux-gcc + summary: Linux with GCC + EOO + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + EOE + + sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest + + : re-bootstrap-bbot-newer + : + $* 124 2>>~"%EOE%d" + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% + trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot + %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+% + EOE + + -$rm +} |