path: root/bbot/worker/worker.cxx

// file      : bbot/worker.cxx -*- C++ -*-
// license   : TBC; see accompanying LICENSE file

#ifndef _WIN32
#  include <signal.h> // signal()
#else
#  include <libbutl/win32-utility.hxx>
#endif

#include <map>
#include <regex>
#include <cstring>   // strchr()
#include <sstream>
#include <iostream>
#include <algorithm> // find(), find_if(), remove_if()

#include <libbutl/b.mxx>
#include <libbutl/pager.mxx>
#include <libbutl/utility.mxx>       // to_utf8()
#include <libbutl/filesystem.mxx>
#include <libbutl/string-parser.mxx>

#include <libbbot/manifest.hxx>

#include <bbot/types.hxx>
#include <bbot/utility.hxx>

#include <bbot/diagnostics.hxx>
#include <bbot/bootstrap-manifest.hxx>

#include <bbot/worker/worker-options.hxx>

using namespace butl;
using namespace bbot;

using std::cout;
using std::endl;

namespace bbot
{
  int
  main (int argc, char* argv[]);

  static int
  build (size_t argc, const char* argv[]);

  process_path argv0;
  worker_options ops;

  dir_path env_dir;

  // Note that upload can be quite large and take a while to upload under high
  // load.
  //
  const size_t tftp_blksize     (1468); // Between 512 (default) and 65464.
  const size_t tftp_put_timeout (3600); // 1 hour (also the default).
  const size_t tftp_get_timeout (10);   // 10 seconds.
  const size_t tftp_get_retries (3);    // Task request retries (see startup()).
}

static dir_path
current_directory ()
try
{
  return dir_path::current_directory ();
}
catch (const system_error& e)
{
  fail << "unable to obtain current directory: " << e << endf;
}

static dir_path
change_wd (tracer& t, string* log, const dir_path& d, bool create = false)
try
{
  if (create)
  {
    if (verb >= 3)
      t << "mkdir -p " << d;

    if (log != nullptr)
      *log += "mkdir -p " + d.representation () + '\n';

    try_mkdir_p (d);
  }

  dir_path r (current_directory ());

  if (verb >= 3)
    t << "cd " << d;

  if (log != nullptr)
    *log += "cd " + d.representation () + '\n';

  dir_path::current_directory (d);
  return r;
}
catch (const system_error& e)
{
  fail << "unable to change current directory to " << d << ": " << e << endf;
}

using std::regex;
namespace regex_constants = std::regex_constants;
using regexes = vector<regex>;

// Run a named command. Name is used for logging and diagnostics only. Match
// lines read from the command's stderr against the regular expressions and
// return the warning result status (instead of success) in case of a match.
//
template <typename... A>
static result_status
run_cmd (tracer& t,
         string& log, const regexes& warn_detect,
         const string& name,
         const process_env& pe,
         A&&... a)
{
  try
  {
    // Trace and log the command line.
    //
    auto cmdc = [&t, &log] (const char* c[], size_t n)
    {
      t (c, n);

      std::ostringstream os;
      process::print (os, c, n);
      log += os.str ();
      log += '\n';
    };

    fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.

    process pr (
      process_start_callback (cmdc,
                              fdopen_null (), // Never reads from stdin.
                              2,              // 1>&2
                              pipe,
                              pe,
                              forward<A> (a)...));

    pipe.out.close ();

    result_status r (result_status::success);

    // UTF-8-sanitize and log the diagnostics. Also print the raw diagnostics
    // to stderr at verbosity level 3 or higher.
    //
    auto add = [&log, &t] (string&& s, bool trace = true)
    {
      if (verb >= 3)
      {
        if (trace)
          t << s;
        else
          text << s;
      }

      to_utf8 (s, '?', codepoint_types::graphic, U"\n\r\t");

      log += s;
      log += '\n';
    };

    {
      ifdstream is (move (pipe.in), fdstream_mode::skip); // Skip on exception.

      for (string l; is.peek () != ifdstream::traits_type::eof (); )
      {
        getline (is, l);

        // Match the log line with the warning-detecting regular expressions
        // until the first match.
        //
        if (r != result_status::warning)
        {
          for (const regex& re: warn_detect)
          {
            // Only examine the first 512 bytes. Long lines (e.g., linker
            // command lines) could trigger implementation-specific limitations
            // (like stack overflow). Plus, it is a performance concern.
            //
            if (regex_search (l.begin (),
                              l.size () < 512 ? l.end () : l.begin () + 512,
                              re))
            {
              r = result_status::warning;
              break;
            }
          }
        }

        add (move (l), false /* trace */);
      }
    }

    if (pr.wait ())
      return r;

    const process_exit& e (*pr.exit);

    add (name + " " + to_string (e));

    return e.normal () ? result_status::error : result_status::abnormal;
  }
  catch (const process_error& e)
  {
    fail << "unable to execute " << name << ": " << e << endf;
  }
  catch (const io_error& e)
  {
    fail << "unable to read " << name << " diagnostics: " << e << endf;
  }
}

template <typename V, typename... A>
static result_status
run_bpkg (const V& envvars,
          tracer& t,
          string& log, const regexes& warn_detect,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  return run_cmd (t,
                  log, warn_detect,
                  "bpkg " + cmd,
                  process_env ("bpkg", envvars),
                  verbosity, cmd, forward<A> (a)...);
}

template <typename... A>
static result_status
run_bpkg (tracer& t,
          string& log, const regexes& warn_detect,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  const char* const* envvars (nullptr);

  return run_bpkg (envvars,
                   t,
                   log, warn_detect,
                   verbosity, cmd, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_b (const V& envvars,
       tracer& t,
       string& log, const regexes& warn_detect,
       const char* verbosity,
       const strings& buildspecs, A&&... a)
{
  string name ("b");
  for (const string& s: buildspecs)
  {
    if (!name.empty ())
      name += ' ';

    name += s;
  }

  return run_cmd (t,
                  log, warn_detect,
                  name,
                  process_env ("b", envvars),
                  verbosity, buildspecs, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_b (const V& envvars,
       tracer& t,
       string& log, const regexes& warn_detect,
       const char* verbosity,
       const string& buildspec, A&&... a)
{
  return run_cmd (t,
                  log, warn_detect,
                  "b " + buildspec,
                  process_env ("b", envvars),
                  verbosity, buildspec, forward<A> (a)...);
}

template <typename... A>
static result_status
run_b (tracer& t,
       string& log, const regexes& warn_detect,
       const char* verbosity,
       const string& buildspec, A&&... a)
{
  const char* const* envvars (nullptr);
  return run_b (envvars,
                t,
                log, warn_detect,
                verbosity, buildspec, forward<A> (a)...);
}

static int bbot::
build (size_t argc, const char* argv[])
{
  using namespace bpkg;

  using string_parser::unquote;

  tracer trace ("build");

  // Our overall plan is as follows:
  //
  // 1. Parse the task manifest (it should be in CWD).
  //
  // 2. Run bpkg to create the configuration, add the repository, and
  //    configure, build, test, optionally install, test installed and
  //    uninstall the package all while saving the logs in the result manifest.
  //
  // 3. Upload the result manifest.
  //
  // Note also that we are being "watched" by the startup version of us which
  // will upload an appropriate result in case we exit with an error. So here
  // for abnormal situations (like a failure to parse the manifest), we just
  // fail.
  //
  task_manifest tm (
    parse_manifest<task_manifest> (path ("task.manifest"), "task"));

  result_manifest rm {
    tm.name,
    tm.version,
    result_status::success,
    operation_results {}
  };

  // Reserve storage large enough to hold all the potential operation results
  // without reallocations. Note that this is not an optimization but is
  // required to make sure that element references are not invalidated when
  // new results are added.
  //
  size_t max_results (6);
  rm.results.reserve (max_results);

  auto add_result = [&rm, max_results] (string o) -> operation_result&
  {
    assert (rm.results.size () < max_results);

    rm.results.push_back (
      operation_result {move (o), result_status::success, ""});

    return rm.results.back ();
  };

  // Note that we don't consider the build system module configuring and
  // testing during the "pre-step" as separate operations and share the
  // operation logs with the "main" configure and test steps (see below).
  // Thus, we save pointers to the added result objects for the subsequent
  // use.
  //
  operation_result* configure_result (nullptr);
  operation_result* test_result (nullptr);

  dir_path rwd; // Root working directory.

  for (;;) // The "breakout" loop.
  {
    // Regular expressions that detect different forms of build2 toolchain
    // warnings. Accidently (or not), they also cover GCC and Clang warnings
    // (for the English locale).
    //
    // The expressions will be matched multiple times, so let's make the
    // matching faster, with the potential cost of making regular expressions
    // creation slower.
    //
    regex::flag_type f (regex_constants::optimize); // ECMAScript is implied.

    regexes wre {
      regex ("^warning: ", f),
      regex ("^.+: warning: ", f)};

    for (const string& re: tm.unquoted_warning_regex ())
      wre.emplace_back (re, f);

    // Step IDs.
    //
    enum class step_id
    {
      bpkg_configure_create,
      bpkg_configure_add,
      bpkg_configure_fetch,
      bpkg_configure_build,
      bpkg_update_update,
      bpkg_test_test,
      bpkg_install_install,
      b_test_installed_create,
      b_test_installed_configure,
      b_test_installed_test,
      bpkg_test_installed_create,
      bpkg_uninstall_uninstall
    };

    const strings step_id_str {
      "bpkg.configure.create",
      "bpkg.configure.add",
      "bpkg.configure.fetch",
      "bpkg.configure.build",
      "bpkg.update.update",
      "bpkg.test.test",
      "bpkg.install.install",
      "b.test-installed.create",
      "b.test-installed.configure",
      "b.test-installed.test",
      "bpkg.test-installed.create",
      "bpkg.uninstall.uninstall"};

    // Split the argument into prefix (empty if not present) and unquoted
    // value. Return nullopt if the prefix is invalid.
    //
    auto parse_arg =
      [&step_id_str] (const string& a) -> optional<pair<string, string>>
    {
      size_t p (a.find_first_of (":=\"'"));

      if (p == string::npos || a[p] != ':') // No prefix.
        return make_pair (string (), unquote (a));

      for (const string& id: step_id_str)
      {
        if (a.compare (0, p, id, 0, p) == 0 &&
            (id.size () == p || (id.size () > p && id[p] == '.')))
          return make_pair (a.substr (0, p), unquote (a.substr (p + 1)));
      }

      return nullopt; // Prefix is invalid.
    };

    // Enter split arguments into a map. Those without a prefix are
    // entered for the *.create steps.
    //
    auto add_arg = [] (std::multimap<string, string>& args,
                       pair<string, string>&& a)
    {
      if (!a.first.empty ())
        args.emplace (move (a));
      else
      {
        args.emplace ("bpkg.configure.create", a.second);
        args.emplace ("b.test-installed.create", a.second);
        args.emplace ("bpkg.test-installed.create", move (a.second));
      }
    };

    // Parse configuration arguments. Report failures to the bbot controller.
    //
    std::multimap<string, string> config_args;

    for (const string& c: tm.config)
    {
      optional<pair<string, string>> v (parse_arg (c));

      if (!v)
      {
        rm.status |= result_status::abort;
        l3 ([&]{trace << "invalid configuration argument prefix in "
                      << "'" << c << "'";});
        break;
      }

      if (v->second[0] != '-' && v->second.find ('=') == string::npos)
      {
        rm.status |= result_status::abort;
        l3 ([&]{trace << "invalid configuration argument '" << c << "'";});
        break;
      }

      add_arg (config_args, move (*v));
    }

    if (!rm.status)
      break;

    // Parse environment arguments.
    //
    std::multimap<string, string> modules;
    std::multimap<string, string> env_args;

    for (size_t i (1); i != argc; ++i)
    {
      const char* a (argv[i]);
      optional<pair<string, string>> v (parse_arg (a));

      if (!v)
        fail << "invalid environment argument prefix in '" << a << "'";

      bool mod (v->second[0] != '-' && v->second.find ('=') == string::npos);

      if (mod && !v->first.empty () &&
          v->first != "bpkg.configure.create" &&
          v->first != "b.test-installed.create")
        fail << "invalid module prefix in '" << a << "'";

      add_arg (mod ? modules : env_args, move (*v));
    }

    // Return command arguments for the specified step id. Arguments with more
    // specific prefixes come last.
    //
    auto step_args = [&step_id_str] (const std::multimap<string, string>& args,
                                     step_id step) -> strings
    {
      strings r;
      const string& s (step_id_str[static_cast<size_t> (step)]);

      for (size_t n (0);; ++n)
      {
        n = s.find ('.', n);

        auto range (
          args.equal_range (n == string::npos ? s : string (s, 0, n)));

        for (auto i (range.first); i != range.second; ++i)
          r.emplace_back (i->second);

        if (n == string::npos)
          break;
      }

      return r;
    };

    // Search for config.install.root variable. If it is present and has a
    // non-empty value, then test the package installation and uninstall. Note
    // that passing [null] value would be meaningless, so we don't recognize it
    // as a special one. While at it, cache the bpkg.configure.create args for
    // later use.
    //
    dir_path install_root;
    strings cargs (step_args (config_args, step_id::bpkg_configure_create));
    {
      size_t n (19);
      auto space = [] (char c) {return c == ' ' || c == '\t';};

      for (const string& s: reverse_iterate (cargs))
      {
        if (s.compare (0, n, "config.install.root") == 0 &&
            (s[n] == '=' || space (s[n])))
        {
          while (space (s[n])) ++n; // Skip spaces.
          if (s[n] == '=')     ++n; // Skip the equal sign.
          while (space (s[n])) ++n; // Skip spaces.

          // Note that the config.install.root variable value may
          // potentially be quoted.
          //
          install_root = dir_path (unquote (string (s, n, s.size () - n)));
          break;
        }
      }
    }

    // bpkg-rep-fetch trust options.
    //
    cstrings trust_ops;
    {
      const char* t ("--trust-no");

      for (const string& fp: tm.trust)
      {
        if (fp == "yes")
          t = "--trust-yes";
        else
        {
          trust_ops.push_back ("--trust");
          trust_ops.push_back (fp.c_str ());
        }
      }

      trust_ops.push_back (t);
    }

    const string&  pkg     (tm.name.string ());
    const version& ver     (tm.version);
    const string   repo    (tm.repository.string ());
    const dir_path pkg_dir (pkg + '-' + ver.string ());

    // Specify the revision explicitly for the bpkg-build command not to end
    // up with a race condition building the latest revision rather than the
    // zero revision.
    //
    const string pkg_rev (pkg +
                          '/' +
                          version (ver.epoch,
                                   ver.upstream,
                                   ver.release,
                                   ver.effective_revision (),
                                   ver.iteration).string ());

    // Query the project's build system information with `b info`.
    //
    auto prj_info = [&trace] (const dir_path& d,
                              bool ext_mods,
                              const char* what)
    {
      // Note that the `b info` diagnostics won't be copied into any of the
      // build logs. This is fine as this is likely to be an infrastructure
      // problem, given that the project distribution has been successfully
      // created. It's actually not quite clear which log this diagnostics
      // could go into.
      //
      try
      {
        return b_info (d, ext_mods, verb, trace);
      }
      catch (const b_error& e)
      {
        if (e.normal ())
          throw failed (); // Assume the build2 process issued diagnostics.

        fail << "unable to query " << what << ' ' << d << " info: " << e
             << endf;
      }
    };

    b_project_info prj; // Package project information.

    // If this is a build system module, perform a "pre-step" by building it
    // in a separate configuration reproducing the one used to build build2
    // itself. Note that the configuration and the environment options and
    // variables are not passed to commands that may affect this
    // configuration.
    //
    bool module (pkg.compare (0, 10, "libbuild2-") == 0);
    dir_path module_dir ("build-module");

    rwd = current_directory ();

    // If this is a build system module that requires bootstrap, then its
    // importation into the dependent (test) projects cannot be configured and
    // the corresponding config.import.* variable needs to be specified on the
    // bpkg/build2 command line as a global override, whenever required.
    //
    // Note that such a module must be explicitly marked with `requires:
    // bootstrap` in its manifest. This can only be detected after the module
    // is configured and it's manifest available.
    //
    bool bootstrap (false);

    // Note that we will parse the package manifest right after the package is
    // configured.
    //
    package_manifest pm;
    path mf (pkg_dir / "manifest"); // Relative to the configuration directory.

    if (module)
    {
      // Configure.
      //
      {
        operation_result& r (add_result ("configure"));
        configure_result = &r;

        // Noop, just for the log record.
        //
        change_wd (trace, &r.log, rwd);

        // b create(<dir>) config.config.load=~build2
        //
        // Note also that we suppress warnings about unused config.* values,
        // such CLI configuration.
        //
        // What if a module wants to use CLI? The current thinking is that we
        // will be "whitelisting" base (i.e., those that can plausibly be used
        // by multiple modules) libraries and tools for use by build system
        // modules. So if and when we whitelist CLI, we will add it here, next
        // to cc.
        //
        r.status |= run_b (
          trace, r.log, wre,
          "-V",
          "create(" + module_dir.representation () + ",cc)",
          "config.config.load=~build2",
          "config.config.persist+='config.*'@unused=drop");

        if (!r.status)
          break;

        change_wd (trace, &r.log, module_dir);

        // bpkg create --existing
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "create",
          "--existing");

        if (!r.status)
          break;

        // bpkg add <env-config-args> <config-args> <repository-url>
        //
        // bpkg.configure.add
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "add",
          step_args (env_args,    step_id::bpkg_configure_add),
          step_args (config_args, step_id::bpkg_configure_add),
          repo);

        if (!r.status)
          break;

        // bpkg fetch <env-config-args> <config-args> <trust-options>
        //
        // bpkg.configure.fetch
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "fetch",
          step_args (env_args,    step_id::bpkg_configure_fetch),
          step_args (config_args, step_id::bpkg_configure_fetch),
          trust_ops);

        if (!r.status)
          break;

        // bpkg build --configure-only <package-name>/<package-version>
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "build",
          "--configure-only",
          "--yes",
          pkg_rev);

        if (!r.status)
          break;

        rm.status |= r.status;

        // Note that being unable to parse the package manifest is likely to
        // be an infrastructure problem, given that the package has been
        // successfully configured.
        //
        pm = parse_manifest<package_manifest> (mf, "package");

        bootstrap = find_if (pm.requirements.begin (),
                             pm.requirements.end (),
                             [] (const requirement_alternatives& r)
                             {
                               return r.size () == 1 && r[0] == "bootstrap";
                             }) != pm.requirements.end ();
      }

      // Update.
      //
      {
        operation_result& r (add_result ("update"));

        // Noop, just for the log record to reduce the potential confusion for
        // the combined log reader due to the configure operation log sharing
        // (see above for details).
        //
        change_wd (trace, &r.log, current_directory ());

        // bpkg update <package-name>
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "update",
          pkg);

        if (!r.status)
          break;

        rm.status |= r.status;
      }

      // Run the package internal tests if the test operation is supported by
      // the project.
      //
      prj = prj_info (pkg_dir, true /* ext_mods */, "project");

      if (find (prj.operations.begin (), prj.operations.end (), "test") !=
          prj.operations.end ())
      {
        operation_result& r (add_result ("test"));
        test_result = &r;

        // Use --package-cwd to help ported to build2 third-party packages a
        // bit (see bpkg-pkg-test(1) for details).
        //
        // Note that internal tests that load the module itself don't make
        // much sense, thus we don't pass the config.import.*  variable on
        // the command line for modules that require bootstrap.
        //
        // bpkg test <package-name>
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "test",
          "--package-cwd",
          pkg);

        if (!r.status)
          break;

        rm.status |= r.status;
      }
    }

    // The "main" step.
    //

    // Use the global override for modules that require bootstrap.
    //
    string module_import (
      module
      ? ((bootstrap ? "!config.import." : "config.import.") +
         tm.name.variable () + "=" + (rwd / module_dir).string ())
      : "");

    // Configure.
    //
    dir_path build_dir ("build"); // Configuration directory name.
    dir_path pkg_config (rwd / (module ? module_dir : build_dir));
    {
      operation_result& r (configure_result != nullptr
                           ? *configure_result
                           : add_result ("configure"));

      change_wd (trace, &r.log, rwd);

      // bpkg create <env-modules> <env-config-args> <config-args>
      //
      // bpkg.configure.create
      //
      {
        // If the package is a build system module, then make sure it is
        // importable in this configuration (see above about bootstrap).
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-V",
          "create",
          "-d", build_dir.string (),
          "--wipe",
          step_args (modules,  step_id::bpkg_configure_create),
          step_args (env_args, step_id::bpkg_configure_create),
          cargs,
          module && !bootstrap ? module_import.c_str () : nullptr);

        if (!r.status)
          break;
      }

      change_wd (trace, &r.log, build_dir);

      // bpkg add <env-config-args> <config-args> <repository-url>
      //
      // bpkg.configure.add
      //
      r.status |= run_bpkg (
        trace, r.log, wre,
        "-v",
        "add",
        step_args (env_args,    step_id::bpkg_configure_add),
        step_args (config_args, step_id::bpkg_configure_add),
        repo);

      if (!r.status)
        break;

      // bpkg fetch <env-config-args> <config-args> <trust-options>
      //
      // bpkg.configure.fetch
      //
      r.status |= run_bpkg (
        trace, r.log, wre,
        "-v",
        "fetch",
        step_args (env_args,    step_id::bpkg_configure_fetch),
        step_args (config_args, step_id::bpkg_configure_fetch),
        trust_ops);

      if (!r.status)
        break;

      // bpkg build --configure-only <env-config-args> <config-args>
      //            <package-name>/<package-version>
      //
      // bpkg.configure.build
      //
      if (!module) // Note: the module is already built in the pre-step.
      {
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "build",
          "--configure-only",
          "--yes",
          step_args (env_args,    step_id::bpkg_configure_build),
          step_args (config_args, step_id::bpkg_configure_build),
          "--",
          pkg_rev);

        if (!r.status)
          break;

        pm = parse_manifest<package_manifest> (mf, "package");
      }

      rm.status |= r.status;
    }

    // Update.
    //
    if (!module) // Note: the module is already built in the pre-step.
    {
      operation_result& r (add_result ("update"));

      // bpkg update <env-config-args> <config-args> <package-name>
      //
      // bpkg.update.update
      //
      r.status |= run_bpkg (
        trace, r.log, wre,
        "-v",
        "update",
        step_args (env_args,    step_id::bpkg_update_update),
        step_args (config_args, step_id::bpkg_update_update),
        pkg);

      if (!r.status)
        break;

      rm.status |= r.status;
    }

    // Run the package internal tests if the test operation is supported by
    // the project, except for the build system module which is taken care of
    // in the pre-step.
    //
    bool internal_tests;

    if (module)
      internal_tests = false;
    else
    {
      prj = prj_info (pkg_dir, true /* ext_mods */, "project");

      internal_tests = find (prj.operations.begin (),
                             prj.operations.end (),
                             "test") != prj.operations.end ();
    }

    // Run the package external tests, if specified. But first filter them
    // against the test-exclude task manifest values using the package names.
    //
    // Note that a proper implementation should also make sure that the
    // excluded test package version matches the version that will supposedly
    // be configured by bpkg and probably abort the build if that's not the
    // case. Such a mismatch can happen due to some valid reasons (the
    // repository was updated since the task was issued, etc) and should
    // probably be followed with automatic rebuild (the flake monitor idea).
    // Anyway, this all requires additional thinking, so let's keep it simple
    // for now.
    //
    // Filter the external test dependencies in place.
    //
    pm.tests.erase (
      remove_if (pm.tests.begin (), pm.tests.end (),
                 [&tm] (const test_dependency& td)
                 {
                   return find_if (tm.test_exclusions.begin (),
                                   tm.test_exclusions.end (),
                                   [&td] (const package& te)
                                   {
                                     return te.name == td.name;
                                   }) != tm.test_exclusions.end ();
                 }),
      pm.tests.end ());

    bool external_tests (!pm.tests.empty ());

    // Configure, update, and test packages in the bpkg configuration in the
    // current working directory. Optionally pass the config.import.* variable
    // override and/or set the environment variables for bpkg processes.
    // Return true if all operations for all packages succeed.
    //
    // Pass true as the sys_dep argument to configure the dependent package as
    // a system dependency, which is normally required for testing modules and
    // installed dependents. Note that bpkg configures the dependent package
    // as a special dependency for the test package.
    //
    auto test = [&pm, &trace, &wre, &step_args, &config_args, &env_args]
                (operation_result& r,
                 bool sys_dep,
                 const char* import = nullptr,
                 const small_vector<string, 1>& envvars = {})
    {
      for (const test_dependency& td: pm.tests)
      {
        const string& pkg (td.name.string ());

        // Configure.
        //
        // bpkg build --configure-only <env-config-args> <config-args>
        //            '<package-name>[ <version-constraint>]'
        //
        // bpkg.configure.build
        //
        r.status |= run_bpkg (
          envvars,
          trace, r.log, wre,
          "-v",
          "build",
          "--configure-only",
          "--yes",
          step_args (env_args,    step_id::bpkg_configure_build),
          step_args (config_args, step_id::bpkg_configure_build),
          import,
          "--",
          td.string (),
          sys_dep ? ("?sys:" + pm.name.string ()).c_str () : nullptr);

        if (!r.status)
          return false;

        // Update.
        //
        // bpkg update <env-config-args> <config-args> <package-name>
        //
        // bpkg.update.update
        //
        r.status |= run_bpkg (
          envvars,
          trace, r.log, wre,
          "-v",
          "update",
          step_args (env_args,    step_id::bpkg_update_update),
          step_args (config_args, step_id::bpkg_update_update),
          import,
          pkg);

        if (!r.status)
          return false;

        // Test.
        //
        // Note that we assume that the package supports the test operation
        // since this is its main purpose.
        //
        // bpkg test <env-config-args> <config-args> <package-name>
        //
        // bpkg.test.test
        //
        r.status |= run_bpkg (
          envvars,
          trace, r.log, wre,
          "-v",
          "test",
          "--package-cwd", // See above for details.
          step_args (env_args,    step_id::bpkg_test_test),
          step_args (config_args, step_id::bpkg_test_test),
          import,
          pkg);

        if (!r.status)
          return false;
      }

      return true;
    };

    if (internal_tests || external_tests)
    {
      operation_result& r (test_result != nullptr
                           ? *test_result
                           : add_result ("test"));

      // Run internal tests.
      //
      if (internal_tests)
      {
        // bpkg test <env-config-args> <config-args> <package-name>
        //
        // bpkg.test.test
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "test",
          "--package-cwd", // See above for details.
          step_args (env_args,    step_id::bpkg_test_test),
          step_args (config_args, step_id::bpkg_test_test),
          pkg);

        if (!r.status)
          break;
      }

      // Run external tests.
      //
      // Note that we assume that these packages belong to the dependent
      // package's repository or its complement repositories, recursively.
      // Thus, we test them in the configuration used to build the dependent
      // package (except for the build system module).
      //
      if (external_tests &&
          !test (r, module, bootstrap ? module_import.c_str () : nullptr))
        break;

      rm.status |= r.status;
    }

    // Install the package, optionally test the installation and uninstall
    // afterwards.
    //
    // These operations are triggered by presence of config.install.root
    // configuration variable having a non-empty value for
    // bpkg.configure.create step.
    //
    if (install_root.empty ())
      break;

    // Now the overall plan is as follows:
    //
    // 1. Install the package.
    //
    // 2. If the package has subprojects that support the test operation, then
    //    configure, build, and test them out of the source tree against the
    //    installed package.
    //
    // 3. If any of the test packages are specified, then configure, build,
    //    and test them in a separate bpkg configuration against the installed
    //    package.
    //
    // 4. Uninstall the package.
    //
    // Install.
    //
    {
      operation_result& r (add_result ("install"));

      change_wd (trace, &r.log, pkg_config);

      // bpkg install <env-config-args> <config-args> <package-name>
      //
      // bpkg.install.install
      //
      r.status |= run_bpkg (
        trace, r.log, wre,
        "-v",
        "install",
        step_args (env_args,    step_id::bpkg_install_install),
        step_args (config_args, step_id::bpkg_install_install),
        pkg);

      if (!r.status)
        break;

      rm.status |= r.status;
    }

    // Test installed.
    //
    // Make sure that the installed package executables are properly imported
    // when configuring and running tests, unless we are testing the build
    // system module (that supposedly doesn't install any executables).
    //
    small_vector<string, 1> envvars;

    dir_paths subprj_dirs; // "Testable" package subprojects.

    // We expect the build system modules to not have any testable subprojects
    // but to have external tests package instead.
    //
    if (module)
      internal_tests = false;
    else
    {
      // Note that we add the $config.install.root/bin directory at the
      // beginning of the PATH environment variable value, so the installed
      // executables are found first.
      //
      string paths ("PATH=" + (install_root / "bin").string ());

      if (optional<string> s = getenv ("PATH"))
      {
        paths += path::traits_type::path_separator;
        paths += *s;
      }

      envvars.push_back (move (paths));

      // Collect the "testable" subprojects.
      //
      for (const b_project_info::subproject& sp: prj.subprojects)
      {
        // Retrieve the subproject information similar to how we've done it
        // for the package.
        //
        b_project_info si (prj_info (pkg_dir / sp.path,
                                     true /* ext_mods */,
                                     "subproject"));

        const strings& ops (si.operations);
        if (find (ops.begin (), ops.end (), "test") != ops.end ())
          subprj_dirs.push_back (sp.path);
      }

      // If there are any "testable" subprojects, then configure them
      // (sequentially) and test/build in parallel afterwards.
      //
      internal_tests = !subprj_dirs.empty ();
    }

    if (internal_tests || external_tests)
    {
      operation_result& r (add_result ("test-installed"));

      change_wd (trace, &r.log, rwd);

      // Run internal tests.
      //
      if (internal_tests)
      {
        string mods; // build2 create meta-operation parameters.

        for (const string& m: step_args (modules,
                                         step_id::b_test_installed_create))
        {
          mods += mods.empty () ? ", " : " ";
          mods += m;
        }

        // b create(<dir>, <env-modules>) <env-config-args> <config-args>
        //
        // b.test-installed.create
        //
        // Amalgamation directory that will contain configuration subdirectory
        // for package tests out of source tree build.
        //
        dir_path out_dir ("build-installed");

        r.status |= run_b (
          trace, r.log, wre,
          "-V",
          "create('" + out_dir.representation () + "'" + mods + ")",
          step_args (env_args,    step_id::b_test_installed_create),
          step_args (config_args, step_id::b_test_installed_create));

        if (!r.status)
          break;

        // Configure subprojects and create buildspecs for their testing.
        //
        strings test_specs;
        for (const dir_path& d: subprj_dirs)
        {
          // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args>
          //                                                <config-args>
          //
          // b.test-installed.configure
          //
          dir_path subprj_src_dir (build_dir / pkg_dir / d);
          dir_path subprj_out_dir (out_dir / d);

          r.status |= run_b (
            envvars,
            trace, r.log, wre,
            "-v",
            "configure('" +
            subprj_src_dir.representation () + "'@'" +
            subprj_out_dir.representation () + "')",
            step_args (env_args,    step_id::b_test_installed_configure),
            step_args (config_args, step_id::b_test_installed_configure));

          if (!r.status)
            break;

          test_specs.push_back (
            "test('" + subprj_out_dir.representation () + "')");
        }

        if (!r.status)
          break;

        // Build/test subprojects.
        //
        // b test(<subprj-out-dir>)... <env-config-args> <config-args>
        //
        // b.test-installed.test
        //
        r.status |= run_b (
          envvars,
          trace, r.log, wre,
          "-v",
          test_specs,
          step_args (env_args,    step_id::b_test_installed_test),
          step_args (config_args, step_id::b_test_installed_test));

        if (!r.status)
          break;
      }

      // Run external tests.
      //
      if (external_tests)
      {
        // Configure.
        //
        // bpkg create <env-modules> <env-config-args> <config-args>
        //
        // bpkg.test-installed.create
        //
        dir_path config_dir ("build-installed-bpkg");

        r.status |= run_bpkg (
          trace, r.log, wre,
          "-V",
          "create",
          "-d", config_dir.string (),
          "--wipe",
          step_args (modules,     step_id::bpkg_test_installed_create),
          step_args (env_args,    step_id::bpkg_test_installed_create),
          step_args (config_args, step_id::bpkg_test_installed_create));

        if (!r.status)
          break;

        change_wd (trace, &r.log, config_dir);

        // bpkg add <env-config-args> <config-args> <repository-url>
        //
        // bpkg.configure.add
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "add",
          step_args (env_args,    step_id::bpkg_configure_add),
          step_args (config_args, step_id::bpkg_configure_add),
          repo);

        if (!r.status)
          break;

        // bpkg fetch <env-config-args> <config-args> <trust-options>
        //
        // bpkg.configure.fetch
        //
        r.status |= run_bpkg (
          trace, r.log, wre,
          "-v",
          "fetch",
          step_args (env_args,    step_id::bpkg_configure_fetch),
          step_args (config_args, step_id::bpkg_configure_fetch),
          trust_ops);

        if (!r.status)
          break;

        // Build/test.
        //
        if (!test (r, true /* sys_dep */, nullptr /* import */, envvars))
          break;
      }

      rm.status |= r.status;
    }

    // Uninstall.
    //
    {
      operation_result& r (add_result ("uninstall"));

      change_wd (trace, &r.log, pkg_config);

      // bpkg uninstall <env-config-args> <config-args> <package-name>
      //
      // bpkg.uninstall.uninstall
      //
      r.status |= run_bpkg (
        trace, r.log, wre,
        "-v",
        "uninstall",
        step_args (env_args,    step_id::bpkg_uninstall_uninstall),
        step_args (config_args, step_id::bpkg_uninstall_uninstall),
        pkg);

      if (!r.status)
        break;

      rm.status |= r.status;
    }

    break;
  }

  if (!rm.results.empty ())
  {
    rm.status |= rm.results.back ().status; // Merge last in case of a break.

    // Also merge statuses of the configure and test operations, which logs
    // can potentially be shared across multiple steps and which results may
    // not be the last in the list.
    //
    if (configure_result != nullptr)
      rm.status |= configure_result->status;

    if (test_result != nullptr)
      rm.status |= test_result->status;
  }
  else
    assert (rm.status == result_status::abort);

  if (!rwd.empty ())
    change_wd (trace, nullptr /* log */, rwd);

  // Upload the result.
  //
  const string url ("tftp://" + ops.tftp_host () + "/result.manifest");

  try
  {
    tftp_curl c (trace,
                 path ("-"),
                 nullfd,
                 curl::put,
                 url,
                 "--tftp-blksize", tftp_blksize,
                 "--max-time", tftp_put_timeout);

    manifest_serializer s (c.out, url);
    rm.serialize (s);

    c.out.close ();
    if (!c.wait ())
      throw_generic_error (EIO);

    // We use exit code 2 to signal abnormal termination but where we managed
    // to upload the result manifest. See startup() for details.
    //
    return rm.status != result_status::abnormal ? 0 : 2;
  }
  catch (const manifest_serialization& e)
  {
    fail << "invalid result manifest: " << e.description;
  }
  catch (const io_error& e) // In case not derived from system_error.
  {
    error << "unable to upload result manifest to " << url << ": " << e;
  }
  catch (const system_error& e)
  {
    error << "unable to upload result manifest to " << url << ": " << e;
  }

  // We use exit code 3 to signal an unsuccessful attempt to upload the result
  // manifest. See startup() for details.
  //
  return 3;
}

static int
startup ()
{
  tracer trace ("startup");

  // Our overall plan is as follows:
  //
  // 1. Download the task manifest into the build directory (CWD).
  //
  // 2. Parse it and get the target.
  //
  // 3. Find the environment setup executable for this target.
  //
  // 4. Execute the environment setup executable.
  //
  // 5. If the environment setup executable fails, then upload the (failed)
  //    result ourselves.
  //
  const string url ("tftp://" + ops.tftp_host () + "/task.manifest");
  const path mf ("task.manifest");

  // If we fail, try to upload the result manifest (abnormal termination). The
  // idea is that the machine gets suspended and we can investigate what's
  // going on by logging in and examining the diagnostics (e.g., via
  // journalctl, etc).
  //
  task_manifest tm;

  try
  {
    // Download the task.
    //
    // We are downloading from our host so there shouldn't normally be any
    // connectivity issues. Unless, of course, we are on Windows where all
    // kinds of flakiness is business as usual. Note that having a long enough
    // timeout is not enough: if we try to connect before the network is up,
    // we will keep waiting forever, even after it is up. So we have to
    // timeout and try again. This is also pretty bad (unlike, say during
    // bootstrap which doesn't happen very often) since we are wasting the
    // machine time. So we are going to log it as a warning and not merely a
    // trace since if this is a common occurrence, then something has to be
    // done about it.
    //
    for (size_t retry (1);; ++retry)
    {
      try
      {
        tftp_curl c (trace,
                     nullfd,
                     mf,
                     curl::get,
                     url,
                     "--tftp-blksize", tftp_blksize,
                     "--max-time", tftp_get_timeout);

        if (!c.wait ())
          throw_generic_error (EIO);

        break;
      }
      catch (const system_error& e)
      {
        bool bail (retry > tftp_get_retries);
        diag_record dr (bail ? error : warn);

        dr << "unable to download task manifest from " << url << " on "
           << retry << " try: " << e;

        if (bail)
          throw failed ();
      }
    }

    // Parse it.
    //
    tm = parse_manifest<task_manifest> (mf, "task");

    // Find the environment setup executable.
    //
    // While the executable path contains a directory (so the PATH search does
    // not apply) we still use process::path_search() to automatically handle
    // appending platform-specific executable extensions (.exe/.bat, etc).
    //
    process_path pp;

    if (tm.environment)
    {
      try
      {
        pp = process::try_path_search (env_dir / *tm.environment,
                                       false /* init */);
      }
      catch (const invalid_path& e)
      {
        fail << "invalid environment name '" << e.path << "': " << e;
      }

      if (pp.empty ())
        fail << "no environment setup executable in " << env_dir << " "
             << "for environment name '" << *tm.environment << "'";
    }
    else
    {
      pp = process::try_path_search (env_dir / "default", false /* init */);

      if (pp.empty ())
        fail << "no default environment setup executable in " << env_dir;
    }

    // Run it.
    //
    strings os;

    if (ops.systemd_daemon ())
      os.push_back ("--systemd-daemon");

    if (ops.verbose_specified ())
    {
      os.push_back ("--verbose");
      os.push_back (to_string (ops.verbose ()));
    }

    if (ops.tftp_host_specified ())
    {
      os.push_back ("--tftp-host");
      os.push_back (ops.tftp_host ());
    }

    // Note that we use the effective (absolute) path instead of recall since
    // we may have changed the CWD.
    //
    // Exit code 2 signals abnormal termination but where the worker uploaded
    // the result itself.
    //
    // Exit code 3 signals an unsuccessful attempt by the worker to upload the
    // result manifest. There is no reason to retry (most likely there is
    // nobody listening on the other end anymore).
    //
    string tg (tm.target.string ());
    switch (run_exit (trace, pp, tg, argv0.effect_string (), os))
    {
    case 3:
    case 2:  return 1;
    case 0:  return 0;
    default: fail << "process " << pp << " exited with non-zero code" << endf;
    }
  }
  catch (const failed&)
  {
    const string url ("tftp://" + ops.tftp_host () + "/result.manifest");

    // If we failed before being able to parse the task manifest, use the
    // "unknown" values for the package name and version.
    //
    result_manifest rm {
      tm.name.empty ()    ? bpkg::package_name ("unknown") : tm.name,
      tm.version.empty () ? bpkg::version ("0")            : tm.version,
      result_status::abnormal,
      operation_results {}
    };

    try
    {
      tftp_curl c (trace,
                   path ("-"),
                   nullfd,
                   curl::put,
                   url,
                   "--tftp-blksize", tftp_blksize,
                   "--max-time", tftp_put_timeout);

      serialize_manifest (rm, c.out, url, "result");
      c.out.close ();

      if (!c.wait ())
        throw_generic_error (EIO);
    }
    catch (const system_error& e)
    {
      fail << "unable to upload result manifest to " << url << ": " << e;
    }

    return 1;
  }
}

static int
bootstrap ()
{
  bootstrap_manifest bm {
    bootstrap_manifest::versions_type {
      {"bbot",    standard_version (BBOT_VERSION_STR)},
      {"libbbot", standard_version (LIBBBOT_VERSION_STR)},
      {"libbpkg", standard_version (LIBBPKG_VERSION_STR)},
      {"libbutl", standard_version (LIBBUTL_VERSION_STR)}
    }
  };

  serialize_manifest (bm, cout, "stdout", "bootstrap");

  return 0;
}

int bbot::
main (int argc, char* argv[])
try
{
  tracer trace ("main");

  // This is a little hack to make our baseutils for Windows work when called
  // with absolute path. In a nutshell, MSYS2's exec*p() doesn't search in the
  // parent's executable directory, only in PATH. And since we are running
  // without a shell (that would read /etc/profile which sets PATH to some
  // sensible values), we are only getting Win32 PATH values. And MSYS2 /bin
  // is not one of them. So what we are going to do is add /bin at the end of
  // PATH (which will be passed as is by the MSYS2 machinery). This will make
  // MSYS2 search in /bin (where our baseutils live). And for everyone else
  // this should be harmless since it is not a valid Win32 path.
  //
#ifdef _WIN32
  {
    string mp;
    if (optional<string> p = getenv ("PATH"))
    {
      mp = move (*p);
      mp += ';';
    }
    mp += "/bin";

    setenv ("PATH", mp);
  }
#endif

  // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if
  // the pipe reading end is closed. Note that by default this signal
  // terminates a process. Also note that there is no way to disable this
  // behavior on a file descriptor basis or for the write() function call.
  //
  // On Windows disable displaying error reporting dialog box. Note that the
  // error mode is inherited by child processes.
  //
#ifndef _WIN32
  if (signal (SIGPIPE, SIG_IGN) == SIG_ERR)
    fail << "unable to ignore broken pipe (SIGPIPE) signal: "
         << system_error (errno, std::generic_category ()); // Sanitize.
#else
  SetErrorMode (SetErrorMode (0) | // Returns the current mode.
                SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
#endif

  cli::argv_scanner scan (argc, argv, true);
  ops.parse (scan);

  verb = ops.verbose ();

  // @@ systemd 231 added JOURNAL_STREAM environment variable which allows
  //    detecting if stderr is connected to the journal.
  //
  if (ops.systemd_daemon ())
    systemd_diagnostics (false);

  // Version.
  //
  if (ops.version ())
  {
    cout << "bbot-worker " << BBOT_VERSION_ID << endl
         << "libbbot " << LIBBBOT_VERSION_ID << endl
         << "libbpkg " << LIBBPKG_VERSION_ID << endl
         << "libbutl " << LIBBUTL_VERSION_ID << endl
         << "Copyright (c) " << BBOT_COPYRIGHT << "." << endl
         << "TBC; All rights reserved" << endl;

    return 0;
  }

  // Help.
  //
  if (ops.help ())
  {
    pager p ("bbot-worker help", false);
    print_bbot_worker_usage (p.stream ());

    // If the pager failed, assume it has issued some diagnostics.
    //
    return p.wait () ? 0 : 1;
  }

  // Figure out our mode.
  //
  if (ops.bootstrap () && ops.startup ())
    fail << "--bootstrap and --startup are mutually exclusive";

  enum class mode {boot, start, build} m (mode::build);

  if (ops.bootstrap ()) m = mode::boot;
  if (ops.startup ())   m = mode::start;

  if (ops.systemd_daemon ())
  {
    info << "bbot worker " << BBOT_VERSION_ID;
  }

  // Figure out our path (used for re-exec).
  //
  argv0 = process::path_search (argv[0], true);

  // Sort out the build directory.
  //
  if (ops.build_specified ())
    change_wd (trace,
               nullptr /* log */,
               ops.build (),
               true /* create */);

  // Sort out the environment directory.
  //
  try
  {
    env_dir = ops.environments_specified ()
      ? ops.environments ()
      : dir_path::home_directory ();

    if (!dir_exists (env_dir))
      throw_generic_error (ENOENT);
  }
  catch (const system_error& e)
  {
    fail << "invalid environment directory: " << e;
  }

  int r (1);
  switch (m)
  {
  case mode::boot:  r = bootstrap ();                            break;
  case mode::start: r = startup ();                              break;
  case mode::build: r = build (static_cast<size_t> (argc),
                               const_cast<const char**> (argv)); break;
  }

  return r;
}
catch (const failed&)
{
  return 1; // Diagnostics has already been issued.
}
catch (const cli::exception& e)
{
  error << e;
  return 1;
}

int
main (int argc, char* argv[])
{
  return bbot::main (argc, argv);
}