aboutsummaryrefslogtreecommitdiff
path: root/bbot/worker/worker.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'bbot/worker/worker.cxx')
-rw-r--r--bbot/worker/worker.cxx656
1 files changed, 656 insertions, 0 deletions
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
new file mode 100644
index 0000000..2aad8aa
--- /dev/null
+++ b/bbot/worker/worker.cxx
@@ -0,0 +1,656 @@
+// file : bbot/worker.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : TBC; see accompanying LICENSE file
+
+#ifndef _WIN32
+# include <signal.h> // signal()
+#else
+# include <stdlib.h> // getenv(), _putenv()
+#endif
+
+#include <regex>
+#include <iostream>
+
+#include <libbutl/pager.hxx>
+#include <libbutl/filesystem.hxx>
+
+#include <libbbot/manifest.hxx>
+
+#include <bbot/types.hxx>
+#include <bbot/utility.hxx>
+
+#include <bbot/diagnostics.hxx>
+#include <bbot/bootstrap-manifest.hxx>
+
+#include <bbot/worker/worker-options.hxx>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+
+namespace bbot
+{
+ process_path argv0;
+ worker_options ops;
+
+ dir_path env_dir;
+
+ const size_t tftp_timeout (10); // 10 seconds.
+ const size_t tftp_retries (3); // Task request retries (see startup()).
+}
+
+static dir_path
+change_wd (const dir_path& d, bool create = false)
+try
+{
+ if (create)
+ try_mkdir_p (d);
+
+ dir_path r (dir_path::current_directory ());
+
+ dir_path::current_directory (d);
+
+ return r;
+}
+catch (const system_error& e)
+{
+ fail << "unable to change current directory to " << d << ": " << e << endf;
+}
+
+using regexes = vector<regex>;
+
+// Match lines read from the command's stderr against the regular expressions
+// and return the warning result status (instead of success) in case of a
+// match.
+//
+template <typename... A>
+static result_status
+run_bpkg (tracer& t,
+ string& log, const regexes& warn_detect,
+ const string& cmd, A&&... a)
+{
+ try
+ {
+ // Trace and log the command line.
+ //
+ auto cmdc = [&t, &log] (const char* c[], size_t n)
+ {
+ t (c, n);
+
+ ostringstream os;
+ process::print (os, c, n);
+ log += os.str ();
+ log += '\n';
+ };
+
+ fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.
+
+ process pr (
+ process_start (cmdc,
+ fdnull (), // Never reads from stdout.
+ 2, // 1>&2
+ pipe.out,
+ "bpkg",
+ "-v",
+ cmd,
+ forward<A> (a)...));
+
+ pipe.out.close ();
+
+ result_status r (result_status::success);
+
+ // Log the diagnostics.
+ //
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip); // Skip on exception.
+
+ for (string l; is.peek () != ifdstream::traits_type::eof (); )
+ {
+ getline (is, l);
+ log += l;
+ log += '\n';
+
+ // Match the log line with the warning-detecting regular expressions
+ // until the first match.
+ //
+ if (r != result_status::warning)
+ {
+ for (const auto& re: warn_detect)
+ {
+ // Only examine the first 512 bytes. Long lines (e.g., linker
+ // command lines) could trigger implementation-specific limitations
+ // (like stack overflow). Plus, it is a performance concern.
+ //
+ if (regex_search (l.begin (),
+ l.size () < 512 ? l.end () : l.begin () + 512,
+ re))
+ {
+ r = result_status::warning;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (pr.wait ())
+ return r;
+
+ log += "bpkg " + cmd;
+ const process_exit& e (*pr.exit);
+
+ if (e.normal ())
+ {
+ log += " exited with code " + to_string (e.code ()) + "\n";
+ return result_status::error;
+ }
+ else
+ {
+ log += " terminated abnormally: " + e.description () +
+ (e.core () ? " (core dumped)" : "") + "\n";
+ return result_status::abnormal;
+ }
+ }
+ catch (const process_error& e)
+ {
+ fail << "unable to execute bpkg " << cmd << ": " << e << endf;
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to read bpkg " << cmd << " diagnostics: " << e << endf;
+ }
+}
+
+static void
+build (size_t argc, const char* argv[])
+{
+ tracer trace ("build");
+
+ // Our overall plan is as follows:
+ //
+ // 1. Parse the task manifest (it should be in CWD).
+ //
+ // 2. Run bpkg to create the configuration, add the repository, and
+ // configure, build, and test the package all while saving the logs in
+ // the result manifest.
+ //
+ // 3. Upload the result manifest.
+ //
+ // Note also that we are being "watched" by the startup version of us which
+ // will upload an appropriate result in case we exit with an error. So here
+ // for abnormal situations (like a failure to parse the manifest), we just
+ // fail.
+ //
+ task_manifest tm (parse_manifest<task_manifest> (path ("manifest"), "task"));
+
+ result_manifest rm {
+ tm.name,
+ tm.version,
+ result_status::success,
+ operation_results {}
+ };
+
+ auto add_result = [&rm] (string o) -> operation_result&
+ {
+ rm.results.push_back (
+ operation_result {move (o), result_status::success, ""});
+
+ return rm.results.back ();
+ };
+
+ dir_path owd;
+
+ for (;;) // The "breakout" loop.
+ {
+ // Regular expressions that detect different forms of build2 toolchain
+ // warnings. Accidently (or not), they also cover GCC and Clang warnings
+ // (for the English locale).
+ //
+ // The expressions will be matched multiple times, so let's make the
+ // matching faster, with the potential cost of making regular expressions
+ // creation slower.
+ //
+ regex::flag_type f (regex_constants::optimize); // ECMAScript is implied.
+
+ regexes wre ({
+ regex ("^warning: ", f),
+ regex ("^.+: warning: ", f)});
+
+ for (const auto& re: tm.unquoted_warning_regex ())
+ wre.emplace_back (re, f);
+
+ // Configure.
+ //
+ {
+ operation_result& r (add_result ("configure"));
+
+ // bpkg create <config-vars> <env-module> <env-config-vars>
+ //
+ const vector_view<const char*> env (argv + 1, argc - 1);
+
+ // Use target (if present) or machine as configuration directory name.
+ //
+ dir_path dir (tm.target ? tm.target->string () : tm.machine);
+
+ r.status |= run_bpkg (trace, r.log, wre,
+ "create",
+ "-d", dir.string (),
+ "--wipe",
+ tm.unquoted_config (),
+ env);
+
+ if (!r.status)
+ break;
+
+ owd = change_wd (dir);
+
+ // bpkg add <repository-url>
+ //
+ r.status |= run_bpkg (trace, r.log, wre, "add", tm.repository.string ());
+
+ if (!r.status)
+ break;
+
+ // bpkg fetch
+ //
+ string t ("--trust-no");
+
+ cstrings ts;
+ for (const string& fp: tm.trust)
+ {
+ if (fp == "yes")
+ t = "--trust-yes";
+ else
+ {
+ ts.push_back ("--trust");
+ ts.push_back (fp.c_str ());
+ }
+ }
+
+ r.status |= run_bpkg (trace, r.log, wre, "fetch", ts, t);
+
+ if (!r.status)
+ break;
+
+ // bpkg build --configure-only <package-name>/<package-version>
+ //
+ r.status |= run_bpkg (trace, r.log, wre,
+ "build",
+ "--configure-only",
+ "--yes",
+ tm.name + '/' + tm.version.string ());
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ // Update.
+ //
+ {
+ operation_result& r (add_result ("update"));
+
+ // bpkg update <package-name>
+ //
+ r.status |= run_bpkg (trace, r.log, wre, "update", tm.name);
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ // Test.
+ //
+ {
+ operation_result& r (add_result ("test"));
+
+ // bpkg test <package-name>
+ //
+ r.status |= run_bpkg (trace, r.log, wre, "test", tm.name);
+
+ if (!r.status)
+ break;
+
+ rm.status |= r.status;
+ }
+
+ break;
+ }
+
+ rm.status |= rm.results.back ().status; // Merge last in case of a break.
+
+ if (!owd.empty ())
+ change_wd (owd);
+
+ // Upload the result.
+ //
+ const string url ("tftp://" + ops.tftp_host () + "/manifest");
+
+ try
+ {
+ tftp_curl c (trace,
+ path ("-"),
+ nullfd,
+ curl::put,
+ url,
+ "--max-time", tftp_timeout);
+
+ serialize_manifest (rm, c.out, url, "result");
+ c.out.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to upload result manifest to " << url << ": " << e;
+ }
+}
+
+static void
+startup ()
+{
+ tracer trace ("startup");
+
+ // Our overall plan is as follows:
+ //
+ // 1. Download the task manifest into the build directory (CWD).
+ //
+ // 2. Parse it and get the target.
+ //
+ // 3. Find the environment setup executable for this target.
+ //
+ // 4. Execute the environment setup executable.
+ //
+ // 5. If the environment setup executable fails, then upload the (failed)
+ // result ourselves.
+ //
+ const string url ("tftp://" + ops.tftp_host () + "/manifest");
+ const path mf ("manifest");
+
+ // If we fail, try to upload the result manifest (abnormal termination). The
+ // idea is that the machine gets suspended and we can investigate what's
+ // going on by logging in and examining the diagnostics (e.g., via
+ // journalctl, etc).
+ //
+ task_manifest tm;
+
+ try
+ {
+ // Download the task.
+ //
+ // We are downloading from our host so there shouldn't normally be any
+ // connectivity issues. Unless, of course, we are on Windows where all
+ // kinds of flakiness is business as usual. Note that having a long enough
+ // timeout is not enough: if we try to connect before the network is up,
+ // we will keep waiting forever, even after it is up. So we have to
+ // timeout and try again. This is also pretty bad (unlike, say during
+ // bootstrap which doesn't happen very often) since we are wasting the
+ // machine time. So we are going to log it as a warning and not merely a
+ // trace since if this is a common occurrence, then something has to be
+ // done about it.
+ //
+ for (size_t retry (1);; ++retry)
+ {
+ try
+ {
+ tftp_curl c (trace,
+ nullfd,
+ mf,
+ curl::get,
+ url,
+ "--max-time", tftp_timeout);
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+
+ break;
+ }
+ catch (const system_error& e)
+ {
+ bool bail (retry > tftp_retries);
+ diag_record dr (bail ? error : warn);
+
+ dr << "unable to download task manifest from " << url << " on "
+ << retry << " try: " << e;
+
+ if (bail)
+ throw failed ();
+ }
+ }
+
+ // Parse it.
+ //
+ tm = parse_manifest<task_manifest> (mf, "task");
+
+ // Find the environment setup executable.
+ //
+ string tg;
+ process_path pp;
+
+ if (tm.target)
+ {
+ tg = tm.target->string ();
+
+ // While the executable path contains a directory (so the PATH search
+ // does not apply) we still use process::path_search() to automatically
+ // handle appending platform-specific executable extensions (.exe/.bat,
+ // etc).
+ //
+ pp = process::try_path_search (env_dir / tg, false);
+ }
+
+ if (pp.empty ())
+ pp = process::try_path_search (env_dir / "default", false);
+
+ if (pp.empty ())
+ fail << "no environment setup executable in " << env_dir << " "
+ << "for target '" << tg << "'";
+
+ // Run it.
+ //
+ strings os;
+
+ if (ops.systemd_daemon ())
+ os.push_back ("--systemd-daemon");
+
+ if (ops.verbose_specified ())
+ {
+ os.push_back ("--verbose");
+ os.push_back (to_string (ops.verbose ()));
+ }
+
+ if (ops.tftp_host_specified ())
+ {
+ os.push_back ("--tftp-host");
+ os.push_back (ops.tftp_host ());
+ }
+
+ // Note that we use the effective (absolute) path instead of recall since
+ // we may have changed the CWD.
+ //
+ run (trace, pp, tg, argv0.effect_string (), os);
+ }
+ catch (const failed&)
+ {
+ // If we failed before being able to parse the task manifest, use the
+ // "unknown" values for the package name and version.
+ //
+ result_manifest rm {
+ tm.name.empty () ? "unknown" : tm.name,
+ tm.version.empty () ? bpkg::version ("0") : tm.version,
+ result_status::abnormal,
+ operation_results {}
+ };
+
+ try
+ {
+ tftp_curl c (trace,
+ path ("-"),
+ nullfd,
+ curl::put,
+ url,
+ "--max-time", tftp_timeout);
+
+ serialize_manifest (rm, c.out, url, "result");
+ c.out.close ();
+
+ if (!c.wait ())
+ throw_generic_error (EIO);
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to upload result manifest to " << url << ": " << e;
+ }
+
+ throw;
+ }
+}
+
+static void
+bootstrap ()
+{
+ bootstrap_manifest bm {
+ bootstrap_manifest::versions_type {
+ {"bbot", standard_version (BBOT_VERSION_STR)},
+ {"libbbot", standard_version (LIBBBOT_VERSION_STR)},
+ {"libbpkg", standard_version (LIBBPKG_VERSION_STR)},
+ {"libbutl", standard_version (LIBBUTL_VERSION_STR)}
+ }
+ };
+
+ serialize_manifest (bm, cout, "stdout", "bootstrap");
+}
+
+int
+main (int argc, char* argv[])
+try
+{
+ // This is a little hack to make out baseutils for Windows work when called
+ // with absolute path. In a nutshell, MSYS2's exec*p() doesn't search in the
+ // parent's executable directory, only in PATH. And since we are running
+ // without a shell (that would read /etc/profile which sets PATH to some
+ // sensible values), we are only getting Win32 PATH values. And MSYS2 /bin
+ // is not one of them. So what we are going to do is add /bin at the end of
+ // PATH (which will be passed as is by the MSYS2 machinery). This will make
+ // MSYS2 search in /bin (where our baseutils live). And for everyone else
+ // this should be harmless since it is not a valid Win32 path.
+ //
+#ifdef _WIN32
+ {
+ string mp ("PATH=");
+ if (const char* p = getenv ("PATH"))
+ {
+ mp += p;
+ mp += ';';
+ }
+ mp += "/bin";
+
+ _putenv (mp.c_str ());
+ }
+#endif
+
+ // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if
+ // the pipe reading end is closed. Note that by default this signal
+ // terminates a process. Also note that there is no way to disable this
+ // behavior on a file descriptor basis or for the write() function call.
+ //
+#ifndef _WIN32
+ if (signal (SIGPIPE, SIG_IGN) == SIG_ERR)
+ fail << "unable to ignore broken pipe (SIGPIPE) signal: "
+ << system_error (errno, generic_category ()); // Sanitize.
+#endif
+
+ cli::argv_scanner scan (argc, argv, true);
+ ops.parse (scan);
+
+ verb = ops.verbose ();
+
+ if (ops.systemd_daemon ())
+ systemd_diagnostics (false);
+
+ // Version.
+ //
+ if (ops.version ())
+ {
+ cout << "bbot-worker " << BBOT_VERSION_ID << endl
+ << "libbbot " << LIBBBOT_VERSION_ID << endl
+ << "libbpkg " << LIBBBOT_VERSION_ID << endl
+ << "libbutl " << LIBBUTL_VERSION_ID << endl
+ << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl
+ << "TBC; All rights reserved" << endl;
+
+ return 0;
+ }
+
+ // Help.
+ //
+ if (ops.help ())
+ {
+ pager p ("bbot-worker help", false);
+ print_bbot_worker_usage (p.stream ());
+
+ // If the pager failed, assume it has issued some diagnostics.
+ //
+ return p.wait () ? 0 : 1;
+ }
+
+ // Figure out our mode.
+ //
+ if (ops.bootstrap () && ops.startup ())
+ fail << "--bootstrap and --startup are mutually exclusive";
+
+ enum class mode {boot, start, build} m (mode::build);
+
+ if (ops.bootstrap ()) m = mode::boot;
+ if (ops.startup ()) m = mode::start;
+
+ if (ops.systemd_daemon ())
+ {
+ info << "bbot worker " << BBOT_VERSION_ID;
+ }
+
+ // Figure out our path (used for re-exec).
+ //
+ argv0 = process::path_search (argv[0], true);
+
+ // Sort out the build directory.
+ //
+ if (ops.build_specified ())
+ change_wd (ops.build (), true); // Create if does not exist.
+
+ // Sort out the environment directory.
+ //
+ try
+ {
+ env_dir = ops.environments_specified ()
+ ? ops.environments ()
+ : dir_path::home_directory ();
+
+ if (!dir_exists (env_dir))
+ throw_generic_error (ENOENT);
+ }
+ catch (const system_error& e)
+ {
+ fail << "invalid environment directory: " << e;
+ }
+
+ switch (m)
+ {
+ case mode::boot: bootstrap (); break;
+ case mode::start: startup (); break;
+ case mode::build: build (static_cast<size_t> (argc),
+ const_cast<const char**> (argv)); break;
+ }
+}
+catch (const failed&)
+{
+ return 1; // Diagnostics has already been issued.
+}
+catch (const cli::exception& e)
+{
+ error << e;
+ return 1;
+}
generated by cgit v1.1 at 2024-05-15 20:04:58 +0000