From 18baf3784407f28f61d9e8d90daf1ce99c7e86d3 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 6 Sep 2021 16:38:36 +0300 Subject: Add support for target, host, and module package types --- bbot/worker/worker.cxx | 2054 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 1356 insertions(+), 698 deletions(-) (limited to 'bbot') diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index c1530f8..10509ff 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -153,62 +153,143 @@ catch (const system_error& e) // enum class step_id { - bpkg_module_create, - bpkg_module_configure_add, - bpkg_module_configure_fetch, - bpkg_module_configure_build, - bpkg_module_update, - bpkg_module_test, - bpkg_create, + // Note that bpkg_module_* options are only used if the main package is a + // build system module (using just ~build2 otherwise). They also have no + // fallback (build system modules are just too different to try to handle + // them together with target and host; e.g., install root). However, + // bpkg_module_create is complemented with arguments from un-prefixed step + // ids, the same way as other *.create[_for_*] steps (note that un-prefixed + // steps are not fallbacks, they are always added first). + // + bpkg_create, // Breakpoint and base. + bpkg_target_create, //: bpkg_create + bpkg_host_create, //: bpkg_create + bpkg_module_create, //: no fallback + + bpkg_link, + bpkg_configure_add, bpkg_configure_fetch, - bpkg_configure_build, + + // Global (as opposed to package-specific) bpkg-pkg-build options (applies + // to all *_configure_build* steps). Note: not a breakpoint. + // + bpkg_global_configure_build, + + // Note that bpkg_configure_build serves as a breakpoint for the + // bpkg-pkg-build call that configures (at once) the main package and all + // its external tests. + // + bpkg_configure_build, // Breakpoint and base. + bpkg_target_configure_build, //: bpkg_configure_build + bpkg_host_configure_build, //: bpkg_configure_build + bpkg_module_configure_build, //: bpkg_configure_build + bpkg_update, bpkg_test, - bpkg_test_separate_configure_build, - bpkg_test_separate_update, - bpkg_test_separate_test, + + // Note that separate test packages are configures as part of the + // bpkg_configure_build step above with options taken from + // bpkg_{target,host}_configure_build, depending on tests package type. + // + bpkg_test_separate_update, //: bpkg_update + bpkg_test_separate_test, //: bpkg_test + + // Note that we only perform the installation tests if this is a target + // package or a self-hosted configuration. + // bpkg_install, + + // Note: skipped for modules. + // b_test_installed_create, b_test_installed_configure, b_test_installed_test, - bpkg_test_installed_create, - bpkg_test_installed_configure_add, - bpkg_test_installed_configure_fetch, - bpkg_test_separate_installed_configure_build, - bpkg_test_separate_installed_update, - bpkg_test_separate_installed_test, + + // Note that for a host package this can involve both run-time and build- + // time tests (which means we may also need a shared configuration for + // modules). + // + // The *_for_{target,host,module} denote main package type, not + // configuration being created, which will always be target (more precisely, + // target or host, but host only in a self-hosted case, which means it's + // the same as target). + // + // Note that if this is a non-self-hosted configuration, we can only end up + // here if building target package and so can just use *_create and *_build + // values in buildtabs. + // + bpkg_test_separate_installed_create, // Breakpoint and base. + bpkg_test_separate_installed_create_for_target, //: bpkg_test_separate_installed_create + bpkg_test_separate_installed_create_for_host, //: bpkg_test_separate_installed_create + bpkg_test_separate_installed_create_for_module, //: no fallback + + bpkg_test_separate_installed_link, // breakpoint only + bpkg_test_separate_installed_configure_add, //: bpkg_configure_add + bpkg_test_separate_installed_configure_fetch, //: bpkg_configure_fetch + + bpkg_test_separate_installed_configure_build, // Breakpoint and base. + bpkg_test_separate_installed_configure_build_for_target, //: bpkg_test_separate_installed_configure_build + bpkg_test_separate_installed_configure_build_for_host, //: bpkg_test_separate_installed_configure_build + bpkg_test_separate_installed_configure_build_for_module, //: bpkg_test_separate_installed_configure_build + + bpkg_test_separate_installed_update, //: bpkg_update + bpkg_test_separate_installed_test, //: bpkg_test + bpkg_uninstall, + end }; static const strings step_id_str { - "bpkg.module.create", - "bpkg.module.configure.add", - "bpkg.module.configure.fetch", - "bpkg.module.configure.build", - "bpkg.module.update", - "bpkg.module.test", "bpkg.create", + "bpkg.target.create", + "bpkg.host.create", + "bpkg.module.create", + + "bpkg.link", + "bpkg.configure.add", "bpkg.configure.fetch", + + "bpkg.global.configure.build", + "bpkg.configure.build", + "bpkg.target.configure.build", + "bpkg.host.configure.build", + "bpkg.module.configure.build", + "bpkg.update", "bpkg.test", - "bpkg.test-separate.configure.build", + "bpkg.test-separate.update", "bpkg.test-separate.test", + "bpkg.install", + "b.test-installed.create", "b.test-installed.configure", "b.test-installed.test", - "bpkg.test-installed.create", - "bpkg.test-installed.configure.add", - "bpkg.test-installed.configure.fetch", + + "bpkg.test-separate-installed.create", + "bpkg.test-separate-installed.create_for_target", + "bpkg.test-separate-installed.create_for_host", + "bpkg.test-separate-installed.create_for_module", + + "bpkg.test-separate-installed.link", + "bpkg.test-separate-installed.configure.add", + "bpkg.test-separate-installed.configure.fetch", + "bpkg.test-separate-installed.configure.build", + "bpkg.test-separate-installed.configure.build_for_target", + "bpkg.test-separate-installed.configure.build_for_host", + "bpkg.test-separate-installed.configure.build_for_module", + "bpkg.test-separate-installed.update", "bpkg.test-separate-installed.test", + "bpkg.uninstall", + "end"}; using std::regex; @@ -645,9 +726,10 @@ build (size_t argc, const char* argv[]) // // 1. Parse the task manifest (it should be in CWD). // - // 2. Run bpkg to create the configuration, add the repository, and - // configure, build, test, optionally install, test installed and - // uninstall the package all while saving the logs in the result manifest. + // 2. Run bpkg to create the package/tests configurations, add the + // repository to them, and configure, build, test, optionally install, + // test installed and uninstall the package all while saving the logs in + // the result manifest. // // 3. Upload the result manifest. // @@ -666,33 +748,14 @@ build (size_t argc, const char* argv[]) operation_results {} }; - // Reserve storage large enough to hold all the potential operation results - // without reallocations. Note that this is not an optimization but is - // required to make sure that element references are not invalidated when - // new results are added. - // - size_t max_results (6); - rm.results.reserve (max_results); - - auto add_result = [&rm, max_results] (string o) -> operation_result& + auto add_result = [&rm] (string o) -> operation_result& { - assert (rm.results.size () < max_results); - rm.results.push_back ( operation_result {move (o), result_status::success, ""}); return rm.results.back (); }; - // Note that we don't consider the build system module configuring and - // testing during the "pre-step" as separate operations and share the - // operation logs with the "main" configure and test steps (see below). - // Thus, we save pointers to the added result objects for the subsequent - // use. - // - operation_result* configure_result (nullptr); - operation_result* test_result (nullptr); - dir_path rwd; // Root working directory. // Resolve the breakpoint specified by the interactive manifest value into @@ -706,6 +769,14 @@ build (size_t argc, const char* argv[]) for (;;) // The "breakout" loop. { + auto abort_operation = [&trace] (operation_result& r, const string& e) + { + l3 ([&]{trace << e;}); + + r.log += "error: " + e + '\n'; + r.status = result_status::abort; + }; + // Regular expressions that detect different forms of build2 toolchain // warnings. Accidently (or not), they also cover GCC and Clang warnings // (for the English locale). @@ -745,14 +816,8 @@ build (size_t argc, const char* argv[]) if (!bkp_step && !bkp_status) { - string e ("invalid interactive build breakpoint '" + b + "'"); - - l3 ([&]{trace << e;}); - - operation_result& r (add_result ("configure")); - - r.log = "error: " + e + '\n'; - r.status = result_status::abort; + abort_operation (add_result ("configure"), + "invalid interactive build breakpoint '" + b + "'"); break; } @@ -778,22 +843,6 @@ build (size_t argc, const char* argv[]) return nullopt; // Prefix is invalid. }; - // Enter split arguments into a map. Those without a prefix are - // entered for the *.create steps. - // - auto add_arg = [] (std::multimap& args, - pair&& a) - { - if (!a.first.empty ()) - args.emplace (move (a)); - else - { - args.emplace ("bpkg.create", a.second); - args.emplace ("b.test-installed.create", a.second); - args.emplace ("bpkg.test-installed.create", move (a.second)); - } - }; - // Parse configuration arguments. Report failures to the bbot controller. // std::multimap config_args; @@ -817,7 +866,7 @@ build (size_t argc, const char* argv[]) break; } - add_arg (config_args, move (*v)); + config_args.emplace (move (*v)); } if (!rm.status) @@ -838,41 +887,79 @@ build (size_t argc, const char* argv[]) bool mod (v->second[0] != '-' && v->second.find ('=') == string::npos); - if (mod && !v->first.empty () && - v->first != "bpkg.create" && - v->first != "b.test-installed.create" && - v->first != "bpkg.test-installed.create") + if (mod && !v->first.empty () && + v->first != "bpkg.create" && + v->first != "bpkg.target.create" && + v->first != "bpkg.host.create" && + v->first != "bpkg.module.create" && + v->first != "b.test-installed.create" && + v->first != "bpkg.test-separate-installed.create" && + v->first != "bpkg.test-separate-installed.create_for_target" && + v->first != "bpkg.test-separate-installed.create_for_host" && + v->first != "bpkg.test-separate-installed.create_for_module") fail << "invalid module prefix in '" << a << "'"; - add_arg (mod ? modules : env_args, move (*v)); + (mod ? modules : env_args).emplace (move (*v)); } - // Return command arguments for the specified step id. Arguments with more + // Return command arguments for the specified step id, complementing + // *.create[_for_*] steps with un-prefixed arguments. Arguments with more // specific prefixes come last. // auto step_args = [] (const std::multimap& args, step_id step, - optional fallback = nullopt) -> strings + optional fallback = nullopt) -> cstrings { - strings r; - const string& sid (step_id_str[static_cast (step)]); + cstrings r; // If no arguments found for the step id, then use the fallback step id, // if specified. // - const string& s (args.find (sid) == args.end () && fallback - ? step_id_str[static_cast (*fallback)] - : sid); + { + const string& s (step_id_str[static_cast (step)]); + + if (args.find (s) == args.end () && fallback) + step = *fallback; + } + + // Add arguments for a specified, potentially empty, prefix. + // + auto add_args = [&args, &r] (const string& prefix) + { + auto range (args.equal_range (prefix)); + + for (auto i (range.first); i != range.second; ++i) + r.emplace_back (i->second.c_str ()); + }; + + // Add un-prefixed arguments if this is one of the *.create[_for_*] + // steps. + // + switch (step) + { + case step_id::bpkg_create: + case step_id::bpkg_target_create: + case step_id::bpkg_host_create: + case step_id::bpkg_module_create: + case step_id::b_test_installed_create: + case step_id::bpkg_test_separate_installed_create: + case step_id::bpkg_test_separate_installed_create_for_target: + case step_id::bpkg_test_separate_installed_create_for_host: + case step_id::bpkg_test_separate_installed_create_for_module: + { + add_args (""); + break; + } + default: break; + } + + const string& s (step_id_str[static_cast (step)]); for (size_t n (0);; ++n) { n = s.find ('.', n); - auto range ( - args.equal_range (n == string::npos ? s : string (s, 0, n))); - - for (auto i (range.first); i != range.second; ++i) - r.emplace_back (i->second); + add_args (n == string::npos ? s : string (s, 0, n)); if (n == string::npos) break; @@ -884,16 +971,15 @@ build (size_t argc, const char* argv[]) // Search for config.install.root variable. If it is present and has a // non-empty value, then test the package installation and uninstall. Note // that passing [null] value would be meaningless, so we don't recognize - // it as a special one. While at it, cache the bpkg.create args for later - // use. + // it as a special one. // dir_path install_root; - strings cargs (step_args (config_args, step_id::bpkg_create)); { size_t n (19); auto space = [] (char c) {return c == ' ' || c == '\t';}; - for (const string& s: reverse_iterate (cargs)) + for (const string& s: + reverse_iterate (step_args (config_args, step_id::bpkg_create))) { if (s.compare (0, n, "config.install.root") == 0 && (s[n] == '=' || space (s[n]))) @@ -974,8 +1060,6 @@ build (size_t argc, const char* argv[]) } }; - b_project_info prj; // Package project information. - rwd = current_directory (); // If the package comes from a version control-based repository, then we @@ -1001,7 +1085,7 @@ build (size_t argc, const char* argv[]) operation_result& r, const dir_path& dist_root, const dir_path& pkg_dir, // - - const char* import = nullptr, + const optional& import = nullopt, const small_vector& envvars = {}) { // Temporarily change the current directory to the distribution root @@ -1045,53 +1129,244 @@ build (size_t argc, const char* argv[]) return true; }; - // The module phase. + // Note that if this is not a self-hosted configuration, then we do not + // build external runtime tests nor run internal for host or module + // packages because the assumption is that they have been built/run (and + // with buildtab settings such as warnings, etc) when testing the + // self-hosted configuration this non-self-hosted one is based on. Also, + // by the same reason, we don't install tools or modules for + // non-self-hosted configurations. + // + // Also note that build system modules can only have external build-time + // tests (which is verified by bpkg-rep-fetch) and target packages cannot + // have external build-time tests (which we verify ourselves). + // + bool selfhost (tm.host && *tm.host); + + // Detect if the package is of the target, host, or module type. + // + auto requirement = [&tm] (const char* id) + { + return find_if (tm.requirements.begin (), + tm.requirements.end (), + [id] (const requirement_alternatives& r) + { + return r.size () == 1 && r[0] == id; + }) != tm.requirements.end (); + }; + + bool module_pkg (pkg.compare (0, 10, "libbuild2-") == 0); + bool bootstrap (module_pkg && requirement ("bootstrap")); + bool host_pkg (!module_pkg && requirement ("host")); + bool target_pkg (!module_pkg && !host_pkg); + + // Split external test packages into the runtime and build-time lists. + // + // Note that runtime and build-time test packages are always configured in + // different bpkg configurations, since they can depend on different + // versions of the same package. + // + small_vector runtime_tests; + small_vector buildtime_tests; + + for (test_dependency& t: tm.tests) + { + if (t.buildtime) + buildtime_tests.push_back (move (t)); + // + // @@ TMP Check for !module_pkg until 0.14.0 is out. + // + else if (target_pkg || (selfhost && !module_pkg)) + runtime_tests.push_back (move (t)); + } + + bool has_buildtime_tests (!buildtime_tests.empty ()); + bool has_runtime_tests (!runtime_tests.empty ()); + + // Abort if a target package has external build-time tests. // + if (target_pkg && has_buildtime_tests) + { + abort_operation ( + add_result ("configure"), + "build-time tests in package not marked with `requires: host`"); + + break; + } + + // Create the required build configurations. + // + // Note that if this is a target package, then we intentionally do not + // create host or module configuration letting the automatic private + // configuration creation to take its course (since that would probably be + // the most typical usage scenario). + // + dir_path target_conf ("build"); + dir_path host_conf ("build-host"); + dir_path module_conf ("build-module"); + + // Main package config. + // + const dir_path& main_pkg_conf (target_pkg ? target_conf : + host_pkg ? host_conf : + module_conf); + + // Create the target configuration if this is a target package or if the + // host/module package has external build-time tests. + // + bool create_target (target_pkg || has_buildtime_tests); + + // Create the host configuration if this is a host package. + // + // Also create it for the module package with external build-time tests. + // The idea is to be able to test a tool which might only be tested via + // the module. To be precise, we need to check that the tests package has + // a build-time dependency (on the tool) but that's not easy to do and so + // we will create a host configuration if a module has any build-time + // tests. + // + bool create_host (host_pkg || (module_pkg && has_buildtime_tests)); - // If this is a build system module, perform a "pre-step" by building it - // in a separate configuration reproducing the one used to build build2 - // itself. Note that the configuration and the environment options and - // variables are not passed to commands that may affect this - // configuration. + // Create the module configuration if the package is a build system + // module. // - bool module (pkg.compare (0, 10, "libbuild2-") == 0); - dir_path module_dir ("build-module"); + // Also create it for the host package with the external build-time tests, + // so that a single build2 configuration is used for both target and host + // packages (this is important in case they happen to use the same + // module). + // + bool create_module (module_pkg || (host_pkg && has_buildtime_tests)); - // If this is a build system module that requires bootstrap, then its - // importation into the dependent (test) projects cannot be configured and - // the corresponding config.import.* variable needs to be specified on the - // bpkg/build2 command line as a global override, whenever required. + // Root configuration through which we will be configuring the cluster + // (note: does not necessarily match the main package type). // - // Note that such a module must be explicitly marked with `requires: - // bootstrap` in its manifest. This can only be detected after the module - // is configured and its manifest available. + // In other words, this is configuration that will be specified for + // bpkg-pkg-build as the current configuration (via -d). It must be the + // configuration that links to all the other configurations. // - bool bootstrap (false); + const dir_path& root_conf (create_target ? target_conf : + create_host ? host_conf : + module_conf); - // Note that we will parse the package manifest right after the package is - // configured. + // Note that bpkg doesn't support configuring bootstrap module + // dependents well, not distinguishing such modules from regular ones + // (see pkg_configure() for details). Thus, we need to pass the + // !config.import.* global override wherever required ourselves. // - package_manifest pm; - path mf ("manifest"); + optional bootstrap_import; + + if (bootstrap) + bootstrap_import = "!config.import." + tm.name.variable () + "=" + + (rwd / main_pkg_conf).string (); - if (module) + // Configure. + // { - // Configure. + operation_result& r (add_result ("configure")); + + // Noop, just for the log record. + // + change_wd (trace, &r.log, rwd); + + // Create the target configuration. + // + // bpkg create // + if (create_target) { - operation_result& r (add_result ("configure")); - configure_result = &r; + step_id b (step_id::bpkg_create); // Breakpoint. + step_id s (step_id::bpkg_target_create); // Step. + step_id f (step_id::bpkg_create); // Fallback. - // Noop, just for the log record. - // - change_wd (trace, &r.log, rwd); + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create", + "-d", target_conf, + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (config_args, s, f)); - // b create() config.config.load=~build2 - // - // [bpkg.module.create] + if (!r.status) + break; + } + + // Create the host configuration. + // + if (create_host) + { + step_id b (step_id::bpkg_create); + step_id s (step_id::bpkg_host_create); + step_id f (step_id::bpkg_create); + + if (selfhost) + { + // bpkg create --type host + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create", + "-d", host_conf, + "--type", "host", + "--name", "host", + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (config_args, s, f)); + + if (!r.status) + break; + } + else + { + // b create() config.config.load=~host + // + // Note also that we suppress warnings about unused config.* values. + // + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create(" + host_conf.representation () + ",cc)", + "config.config.load=~host", + "config.config.persist+='config.*'@unused=drop"); + + if (!r.status) + break; + + // bpkg create --existing --type host + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "create", + "--existing", + "-d", host_conf, + "--type", "host", + "--name", "host"); + + if (!r.status) + break; + } + } + + // Create the module configuration. + // + if (create_module) + { + step_id b (step_id::bpkg_create); + step_id s (step_id::bpkg_module_create); + + // b create() config.config.load=~build2 [ ] // - // Note also that we suppress warnings about unused config.* values, - // such CLI configuration. + // Note also that we suppress warnings about unused config.* values. // // What if a module wants to use CLI? The current thinking is that we // will be "whitelisting" base (i.e., those that can plausibly be used @@ -1099,291 +1374,367 @@ build (size_t argc, const char* argv[]) // modules. So if and when we whitelist CLI, we will add it here, next // to cc. // + cstrings eas; + cstrings cas; + string mods; + + if (module_pkg) + { + for (const string& m: step_args (modules, s)) + { + if (!mods.empty ()) + mods += ' '; + + mods += m; + } + + eas = step_args (env_args, s); + cas = step_args (config_args, s); + } + else + mods = "cc"; + r.status |= run_b ( - step_id::bpkg_module_create, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", - "create(" + module_dir.representation () + ",cc)", + "create(" + module_conf.representation () + "," + mods + ")", "config.config.load=~build2", - "config.config.persist+='config.*'@unused=drop"); + "config.config.persist+='config.*'@unused=drop", + eas, + cas); if (!r.status) break; - change_wd (trace, &r.log, module_dir); - - // bpkg create --existing + // bpkg create --existing --type build2 // r.status |= run_bpkg ( - step_id::bpkg_module_create, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "create", - "--existing"); + "--existing", + "-d", module_conf, + "--type", "build2", + "--name", "module"); if (!r.status) break; + } + + // Link the configurations. + // + // bpkg link -d + // + { + step_id b (step_id::bpkg_link); + + if (create_target) + { + if (create_host) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", target_conf, + host_conf); + + if (!r.status) + break; + } + + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", target_conf, + module_conf); + + if (!r.status) + break; + } + } + + if (create_host) + { + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", host_conf, + module_conf); + + if (!r.status) + break; + } + } + } + + // Fetch repositories into the main package configuration and the target + // configuration for external build-time test, if any. + // + // bpkg add + // + { + step_id b (step_id::bpkg_configure_add); + step_id s (step_id::bpkg_configure_add); - // bpkg add - // - // bpkg.module.configure.add (bpkg.configure.add) - // r.status |= run_bpkg ( - step_id::bpkg_module_configure_add, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", - - step_args (env_args, - step_id::bpkg_module_configure_add, - step_id::bpkg_configure_add), - - step_args (config_args, - step_id::bpkg_module_configure_add, - step_id::bpkg_configure_add), - + "-d", main_pkg_conf, + step_args (env_args, s), + step_args (config_args, s), repo); if (!r.status) break; + } + + // bpkg fetch + // + { + step_id b (step_id::bpkg_configure_fetch); + step_id s (step_id::bpkg_configure_fetch); - // bpkg fetch - // - // bpkg.module.configure.fetch (bpkg.configure.fetch) - // r.status |= run_bpkg ( - step_id::bpkg_module_configure_fetch, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "fetch", - - step_args (env_args, - step_id::bpkg_module_configure_fetch, - step_id::bpkg_configure_fetch), - - step_args (config_args, - step_id::bpkg_module_configure_fetch, - step_id::bpkg_configure_fetch), - + "-d", main_pkg_conf, + step_args (env_args, s), + step_args (config_args, s), trust_ops); if (!r.status) break; + } - // bpkg build --configure-only / - // - // [bpkg.module.configure.build] + if (has_buildtime_tests) + { + // bpkg add // - r.status |= run_bpkg ( - step_id::bpkg_module_configure_build, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "build", - "--configure-only", - "--checkout-root", dist_root, - "--yes", - pkg_rev); - - if (!r.status) - break; + { + step_id b (step_id::bpkg_configure_add); + step_id s (step_id::bpkg_configure_add); - rm.status |= r.status; + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "add", + "-d", target_conf, + step_args (env_args, s), + step_args (config_args, s), + repo); - bool dist (exists (dist_src)); - const dir_path& src_dir (dist ? dist_src : pkg_dir); + if (!r.status) + break; + } - // Note that being unable to parse the package manifest is likely to - // be an infrastructure problem, given that the package has been - // successfully configured. + // bpkg fetch // - pm = parse_manifest (src_dir / mf, "package"); + { + step_id b (step_id::bpkg_configure_fetch); + step_id s (step_id::bpkg_configure_fetch); - bootstrap = find_if (pm.requirements.begin (), - pm.requirements.end (), - [] (const requirement_alternatives& r) - { - return r.size () == 1 && r[0] == "bootstrap"; - }) != pm.requirements.end (); + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "fetch", + "-d", target_conf, + step_args (env_args, s), + step_args (config_args, s), + trust_ops); - if (dist) - { - // Note that we reuse the configure operation log for the dist - // meta-operation. - // - if (!redist (step_id::bpkg_module_configure_build, - r, - dist_root, - pkg_dir)) + if (!r.status) break; - - rm.status |= r.status; } } - // Update. + // Configure all the packages using a single bpkg-pkg-build command. + // + // The overall command looks like this (but some parts may be omitted): + // + // bpkg build --configure-only + // { }+ + // { }+ { ... } + // { }+ { ... } + // + + // Add the main package args. // + // Also add the external runtime test packages here since they share the + // configuration directory with the main package. + // + strings pkg_args; { - operation_result& r (add_result ("update")); + step_id s (target_pkg ? step_id::bpkg_target_configure_build : + host_pkg ? step_id::bpkg_host_configure_build : + step_id::bpkg_module_configure_build); - // Noop, just for the log record to reduce the potential confusion for - // the combined log reader due to the configure operation log sharing - // (see above for details). - // - change_wd (trace, &r.log, current_directory ()); + step_id f (step_id::bpkg_configure_build); - // bpkg update + cstrings eas (step_args (env_args, s, f)); + cstrings cas (step_args (config_args, s, f)); + + // Main package configuration name. // - // [bpkg.module.update] + string conf_name (main_pkg_conf == root_conf ? "" : + host_pkg ? "host" : + "module"); + + // Add the main package. // - r.status |= run_bpkg ( - step_id::bpkg_module_update, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "update", - pkg); + { + if (!eas.empty () || !cas.empty () || !conf_name.empty ()) + { + pkg_args.push_back ("{"); - if (!r.status) - break; + if (!conf_name.empty ()) + { + pkg_args.push_back ("--config-name"); + pkg_args.push_back (conf_name); + } - rm.status |= r.status; - } + pkg_args.insert (pkg_args.end (), eas.begin (), eas.end ()); + pkg_args.insert (pkg_args.end (), cas.begin (), cas.end ()); - // Run the package internal tests if the test operation is supported by - // the project. - // - prj = prj_info (pkg_dir, true /* ext_mods */, "project"); + pkg_args.push_back ("}+"); + } - if (find (prj.operations.begin (), prj.operations.end (), "test") != - prj.operations.end ()) - { - operation_result& r (add_result ("test")); - test_result = &r; + pkg_args.push_back (pkg_rev); + } - // Use --package-cwd to help ported to build2 third-party packages a - // bit (see bpkg-pkg-test(1) for details). + // Add the runtime test packages. // - // Note that internal tests that load the module itself don't make - // much sense, thus we don't pass the config.import.* variable on - // the command line for modules that require bootstrap. - // - // bpkg test - // - // [bpkg.module.test] - // - r.status |= run_bpkg ( - step_id::bpkg_module_test, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "test", - "--package-cwd", - pkg); + if (has_runtime_tests) + { + bool og (!conf_name.empty () || + bootstrap_import || + !eas.empty () || + !cas.empty ()); - if (!r.status) - break; + if (og) + { + pkg_args.push_back ("{"); - rm.status |= r.status; - } - } + if (!conf_name.empty ()) + { + pkg_args.push_back ("--config-name"); + pkg_args.push_back (conf_name); + } - // The main phase. - // + if (bootstrap_import) + pkg_args.push_back (*bootstrap_import); - // Use the global override for modules that require bootstrap. - // - string module_import ( - module - ? ((bootstrap ? "!config.import." : "config.import.") + - tm.name.variable () + "=" + (rwd / module_dir).string ()) - : ""); + pkg_args.insert (pkg_args.end (), + make_move_iterator (eas.begin ()), + make_move_iterator (eas.end ())); - // Configure. - // - dir_path build_dir ("build"); // Configuration directory name. - dir_path pkg_config (rwd / (module ? module_dir : build_dir)); - { - operation_result& r (configure_result != nullptr - ? *configure_result - : add_result ("configure")); + pkg_args.insert (pkg_args.end (), + make_move_iterator (cas.begin ()), + make_move_iterator (cas.end ())); - change_wd (trace, &r.log, rwd); + pkg_args.push_back ("}+"); + } - // bpkg create + // Add test dependency package constraints (for example + // 'bar > 1.0.0') and group them if there are multiple of them. + // + if (og && runtime_tests.size () != 1) + pkg_args.push_back ("{"); + + for (auto t: runtime_tests) + pkg_args.push_back (t.string ()); + + if (og && runtime_tests.size () != 1) + pkg_args.push_back ("}"); + } + } + + // Add the external build-time test packages. // - // bpkg.create + // Note that if present, they are always configured in the root + // configuration and thus don't require --config-name. // + if (has_buildtime_tests) { - // If the package is a build system module, then make sure it is - // importable in this configuration (see above about bootstrap). - // - r.status |= run_bpkg ( - step_id::bpkg_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create", - "-d", build_dir.string (), - "--wipe", - step_args (modules, step_id::bpkg_create), - step_args (env_args, step_id::bpkg_create), - cargs, - module && !bootstrap ? module_import.c_str () : nullptr); + step_id s (step_id::bpkg_target_configure_build); + step_id f (step_id::bpkg_configure_build); - if (!r.status) - break; - } + cstrings eas (step_args (env_args, s, f)); + cstrings cas (step_args (config_args, s, f)); - change_wd (trace, &r.log, build_dir); + bool og (bootstrap_import || !eas.empty () || !cas.empty ()); - // bpkg add - // - // bpkg.configure.add - // - r.status |= run_bpkg ( - step_id::bpkg_configure_add, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "add", - step_args (env_args, step_id::bpkg_configure_add), - step_args (config_args, step_id::bpkg_configure_add), - repo); + if (og) + { + pkg_args.push_back ("{"); - if (!r.status) - break; + if (bootstrap_import) + pkg_args.push_back (*bootstrap_import); - // bpkg fetch - // - // bpkg.configure.fetch - // - r.status |= run_bpkg ( - step_id::bpkg_configure_fetch, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "fetch", - step_args (env_args, step_id::bpkg_configure_fetch), - step_args (config_args, step_id::bpkg_configure_fetch), - trust_ops); + pkg_args.insert (pkg_args.end (), + make_move_iterator (eas.begin ()), + make_move_iterator (eas.end ())); - if (!r.status) - break; + pkg_args.insert (pkg_args.end (), + make_move_iterator (cas.begin ()), + make_move_iterator (cas.end ())); - // bpkg build --configure-only - // / + pkg_args.push_back ("}+"); + } + + // Add test dependency package constraints and group them if there are + // multiple of them. + // + if (og && buildtime_tests.size () != 1) + pkg_args.push_back ("{"); + + // Strip the build-time mark. + // + for (auto t: buildtime_tests) + pkg_args.push_back (t.dependency::string ()); + + if (og && buildtime_tests.size () != 1) + pkg_args.push_back ("}"); + } + + // Finally, configure all the packages. // - // bpkg.configure.build + // Note that in the future we can run this command with the + // --rebuild-checksum option first to obtain the dependency tree + // checksum and bail out if it didn't change since the previous build. // - if (!module) // Note: the module is already built in the pre-step. { + step_id b (step_id::bpkg_configure_build); + step_id s (step_id::bpkg_global_configure_build); + r.status |= run_bpkg ( - step_id::bpkg_configure_build, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", @@ -1391,49 +1742,50 @@ build (size_t argc, const char* argv[]) "--configure-only", "--checkout-root", dist_root, "--yes", - step_args (env_args, step_id::bpkg_configure_build), - step_args (config_args, step_id::bpkg_configure_build), + "-d", root_conf, + step_args (env_args, s), + step_args (config_args, s), "--", - pkg_rev); + pkg_args); if (!r.status) break; + } - bool dist (exists (dist_src)); - const dir_path& src_dir (dist ? dist_src : pkg_dir); - - pm = parse_manifest (src_dir / mf, "package"); + change_wd (trace, &r.log, main_pkg_conf); - if (dist) - { - if (!redist (step_id::bpkg_configure_build, r, dist_root, pkg_dir)) - break; + // Redistribute the main package, if required (test packages will be + // handled later). + // + if (exists (dist_src)) + { + step_id b (step_id::bpkg_configure_build); - rm.status |= r.status; - } + if (!redist (b, r, dist_root, pkg_dir)) + break; } rm.status |= r.status; } - // Update. + // Update the main package. // - if (!module) // Note: the module is already built in the pre-step. { operation_result& r (add_result ("update")); // bpkg update // - // bpkg.update - // + step_id b (step_id::bpkg_update); + step_id s (step_id::bpkg_update); + r.status |= run_bpkg ( - step_id::bpkg_update, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "update", - step_args (env_args, step_id::bpkg_update), - step_args (config_args, step_id::bpkg_update), + step_args (env_args, s), + step_args (config_args, s), pkg); if (!r.status) @@ -1442,59 +1794,11 @@ build (size_t argc, const char* argv[]) rm.status |= r.status; } - // Run the package internal tests if the test operation is supported by - // the project, except for the build system module which is taken care of - // in the pre-step. - // - bool internal_tests; - - if (module) - { - internal_tests = false; - } - else - { - prj = prj_info (pkg_dir, true /* ext_mods */, "project"); - - internal_tests = find (prj.operations.begin (), - prj.operations.end (), - "test") != prj.operations.end (); - } - - // Run the package external tests, if specified. But first filter them - // against the test-exclude task manifest values using the package names. - // - // Note that a proper implementation should also make sure that the - // excluded test package version matches the version that will supposedly - // be configured by bpkg and probably abort the build if that's not the - // case. Such a mismatch can happen due to some valid reasons (the - // repository was updated since the task was issued, etc) and should - // probably be followed with automatic rebuild (the flake monitor idea). - // Anyway, this all requires additional thinking, so let's keep it simple - // for now. - // - // Filter the external test dependencies in place. - // - pm.tests.erase ( - remove_if (pm.tests.begin (), pm.tests.end (), - [&tm] (const test_dependency& td) - { - return find_if (tm.test_exclusions.begin (), - tm.test_exclusions.end (), - [&td] (const package& te) - { - return te.name == td.name; - }) != tm.test_exclusions.end (); - }), - pm.tests.end ()); - - bool external_tests (!pm.tests.empty ()); - - // Configure, re-distribute if comes from a version control-based - // repository, update, and test packages in the bpkg configuration in the + // Re-distribute if comes from a version control-based repository, update, + // and test external test packages in the bpkg configuration in the // current working directory. Optionally pass the config.import.* variable - // override and/or set the environment variables for bpkg processes. - // Return true if all operations for all packages succeed. + // override and/or set the environment variables for the bpkg processes. + // Return true if all operations for all packages succeeded. // // Pass true as the installed argument to use the test separate installed // phase step ids (bpkg.test-separate-installed.*) and the test separate @@ -1502,113 +1806,97 @@ build (size_t argc, const char* argv[]) // back to the main phase step ids (bpkg.*) when no environment/ // configuration arguments are specified for them. // - // Pass true as the sys_dep argument to configure the dependent package as - // a system dependency, which is normally required for testing modules and - // installed dependents. Note that bpkg configures the dependent package - // as a special dependency for the test package. - // auto test = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd, &step_args, &config_args, &env_args, - &pm, + &bootstrap_import, &redist] (operation_result& r, + const small_vector& tests, const dir_path& dist_root, bool installed, - bool sys_dep, - const char* import = nullptr, const small_vector& envvars = {}) { - for (const test_dependency& td: pm.tests) + const optional& import (!installed + ? bootstrap_import + : nullopt); + + for (const test_dependency& td: tests) { const string& pkg (td.name.string ()); - // Configure. - // - // bpkg build --configure-only - // '[ ]' - // - // bpkg.test-separate[-installed].configure.build (bpkg.configure.build) - // - step_id s (installed - ? step_id::bpkg_test_separate_installed_configure_build - : step_id::bpkg_test_separate_configure_build); - - r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "build", - "--configure-only", - "--checkout-root", dist_root, - "--yes", - step_args (env_args, s, step_id::bpkg_configure_build), - step_args (config_args, s, step_id::bpkg_configure_build), - import, - "--", - td.string (), - sys_dep ? ("?sys:" + pm.name.string ()).c_str () : nullptr); - - if (!r.status) - return false; - - // Note that re-distributing the test package is a bit tricky since we - // don't know its version and so cannot deduce its source directory - // name easily. We could potentially run the bpkg-status command after - // the package is configured and parse the output to obtain the - // version. Let's, however, keep it simple and find the source - // directory using the package directory name pattern. + // Re-distribute. // if (exists (dist_root)) - try { - dir_path pkg_dir; + // Note that re-distributing the test package is a bit tricky since + // we don't know its version and so cannot deduce its source + // directory name easily. We could potentially run the bpkg-status + // command after the package is configured and parse the output to + // obtain the version. Let's, however, keep it simple and find the + // source directory using the package directory name pattern. + // + try + { + dir_path pkg_dir; - path_search (dir_path (pkg + "-*/"), - [&pkg_dir] (path&& pe, const string&, bool interm) - { - if (!interm) - pkg_dir = path_cast (move (pe)); + path_search (dir_path (pkg + "-*/"), + [&pkg_dir] (path&& pe, const string&, bool interm) + { + if (!interm) + pkg_dir = path_cast (move (pe)); - return interm; - }, - dist_root); + return interm; + }, + dist_root); - if (!pkg_dir.empty () && - !redist (s, r, dist_root, pkg_dir, import, envvars)) - return false; - } - catch (const system_error& e) - { - fail << "unable to scan directory " << dist_root << ": " << e; + if (!pkg_dir.empty ()) + { + step_id b ( + installed + ? step_id::bpkg_test_separate_installed_configure_build + : step_id::bpkg_configure_build); + + if (!redist (b, r, dist_root, pkg_dir, import, envvars)) + return false; + } + } + catch (const system_error& e) + { + fail << "unable to scan directory " << dist_root << ": " << e; + } } // Update. // // bpkg update // - // bpkg.test-separate[-installed].update (bpkg.update) - // - s = installed - ? step_id::bpkg_test_separate_installed_update - : step_id::bpkg_test_separate_update; + { + step_id b (installed + ? step_id::bpkg_test_separate_installed_update + : step_id::bpkg_test_separate_update); - r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "update", - step_args (env_args, s, step_id::bpkg_update), - step_args (config_args, s, step_id::bpkg_update), - import, - pkg); + step_id s (installed + ? step_id::bpkg_test_separate_installed_update + : step_id::bpkg_test_separate_update); - if (!r.status) - return false; + step_id f (step_id::bpkg_update); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "update", + step_args (env_args, s, f), + step_args (config_args, s, f), + import, + pkg); + + if (!r.status) + return false; + } // Test. // @@ -1617,88 +1905,107 @@ build (size_t argc, const char* argv[]) // // bpkg test // - // bpkg.test-separate[-installed].test (bpkg.test) - // - s = installed - ? step_id::bpkg_test_separate_installed_test - : step_id::bpkg_test_separate_test; + { + step_id b (installed + ? step_id::bpkg_test_separate_installed_test + : step_id::bpkg_test_separate_test); - r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "test", - "--package-cwd", // See above for details. - step_args (env_args, s, step_id::bpkg_test), - step_args (config_args, s, step_id::bpkg_test), - import, - pkg); + step_id s (installed + ? step_id::bpkg_test_separate_installed_test + : step_id::bpkg_test_separate_test); - if (!r.status) - return false; + step_id f (step_id::bpkg_test); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "test", + "--package-cwd", // See above for details. + step_args (env_args, s, f), + step_args (config_args, s, f), + import, + pkg); + + if (!r.status) + return false; + } } return true; }; - if (internal_tests || external_tests) - { - operation_result& r (test_result != nullptr - ? *test_result - : add_result ("test")); + // Test the main package. + // + b_project_info prj (prj_info (pkg_dir, true /* ext_mods */, "project")); - // Noop, just for the log record to reduce the potential confusion for - // the combined log reader due to updating the build system module in a - // separate configuration (see above for details). - // - if (module) - change_wd (trace, &r.log, current_directory ()); + // Run the internal tests if the test operation is supported by the + // project but only for the target package or if the configuration is + // self-hosted. + // + bool has_internal_tests ((target_pkg || selfhost) && + find (prj.operations.begin (), + prj.operations.end (), + "test") != prj.operations.end ()); + + if (has_internal_tests || has_runtime_tests || has_buildtime_tests) + { + operation_result& r (add_result ("test")); // Run internal tests. // - if (internal_tests) // Note: false for modules (see above). + if (has_internal_tests) { - // bpkg test + // Use --package-cwd to help ported to build2 third-party packages a + // bit (see bpkg-pkg-test(1) for details). + // + // Note that internal tests that load the module itself don't make + // much sense, thus we don't pass the config.import.* variable on + // the command line for modules that require bootstrap. // - // bpkg.test + // bpkg test // + step_id b (step_id::bpkg_test); + step_id s (step_id::bpkg_test); + r.status |= run_bpkg ( - step_id::bpkg_test, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "test", - "--package-cwd", // See above for details. - step_args (env_args, step_id::bpkg_test), - step_args (config_args, step_id::bpkg_test), + "--package-cwd", + step_args (env_args, s), + step_args (config_args, s), pkg); if (!r.status) break; } - // Run external tests. + // External runtime tests. // // Note that we assume that these packages belong to the dependent // package's repository or its complement repositories, recursively. // Thus, we test them in the configuration used to build the dependent - // package (except for the build system module). + // package. // - if (external_tests) + if (has_runtime_tests) { - // The test separate phase. - // - if (!test (r, - dist_root, - false /* installed */, - module, - bootstrap ? module_import.c_str () : nullptr)) + if (!test (r, runtime_tests, dist_root, false /* installed */)) break; + } - // Back to the main phase. - // + // External build-time tests. + // + if (has_buildtime_tests) + { + change_wd (trace, &r.log, rwd / target_conf); + + if (!test (r, buildtime_tests, dist_root, false /* installed */)) + break; } rm.status |= r.status; @@ -1714,39 +2021,46 @@ build (size_t argc, const char* argv[]) if (install_root.empty ()) break; + // If this is not a self-hosted configuration, then skip installing host + // and module packages. + // + if (!target_pkg && !selfhost) + break; + // Now the overall plan is as follows: // // 1. Install the package. // // 2. If the package has subprojects that support the test operation, then // configure, build, and test them out of the source tree against the - // installed package. + // installed package using the build system directly. // // 3. If any of the test packages are specified, then configure, build, - // and test them in a separate bpkg configuration against the installed - // package. + // and test them in a separate bpkg configuration(s) against the + // installed package. // // 4. Uninstall the package. - // + // Install. // { operation_result& r (add_result ("install")); - change_wd (trace, &r.log, pkg_config); + change_wd (trace, &r.log, rwd / main_pkg_conf); // bpkg install // - // bpkg.install - // + step_id b (step_id::bpkg_install); + step_id s (step_id::bpkg_install); + r.status |= run_bpkg ( - step_id::bpkg_install, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "install", - step_args (env_args, step_id::bpkg_install), - step_args (config_args, step_id::bpkg_install), + step_args (env_args, s), + step_args (config_args, s), pkg); if (!r.status) @@ -1755,38 +2069,15 @@ build (size_t argc, const char* argv[]) rm.status |= r.status; } - // The test installed phase. - // - - // Make sure that the installed package executables are properly imported - // when configuring and running tests, unless we are testing the build - // system module (that supposedly doesn't install any executables). + // Run the internal tests if the project contains "testable" subprojects, + // but not for a module. // - small_vector envvars; + has_internal_tests = false; dir_paths subprj_dirs; // "Testable" package subprojects. - // We expect the build system modules to not have any testable subprojects - // but to have external tests package instead. - // - if (module) - internal_tests = false; - else + if (!module_pkg) { - // Note that we add the $config.install.root/bin directory at the - // beginning of the PATH environment variable value, so the installed - // executables are found first. - // - string paths ("PATH=" + (install_root / "bin").string ()); - - if (optional s = getenv ("PATH")) - { - paths += path::traits_type::path_separator; - paths += *s; - } - - envvars.push_back (move (paths)); - // Collect the "testable" subprojects. // for (const b_project_info::subproject& sp: prj.subprojects) @@ -1803,22 +2094,45 @@ build (size_t argc, const char* argv[]) subprj_dirs.push_back (sp.path); } - // If there are any "testable" subprojects, then configure them - // (sequentially) and test/build in parallel afterwards. - // - internal_tests = !subprj_dirs.empty (); + has_internal_tests = !subprj_dirs.empty (); } - if (internal_tests || external_tests) + if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { operation_result& r (add_result ("test-installed")); change_wd (trace, &r.log, rwd); + // Make sure that the installed package executables are properly + // imported when configuring and running tests, unless we are testing + // the build system module (that supposedly doesn't install any + // executables). + // + small_vector envvars; + + if (!module_pkg) + { + // Note that we add the $config.install.root/bin directory at the + // beginning of the PATH environment variable value, so the installed + // executables are found first. + // + string paths ("PATH=" + (install_root / "bin").string ()); + + if (optional s = getenv ("PATH")) + { + paths += path::traits_type::path_separator; + paths += *s; + } + + envvars.push_back (move (paths)); + } + // Run internal tests. // - if (internal_tests) + if (has_internal_tests) { + // Create the configuration. + // string mods; // build2 create meta-operation parameters. for (const string& m: step_args (modules, @@ -1830,26 +2144,30 @@ build (size_t argc, const char* argv[]) // b create(, ) // - // b.test-installed.create - // // Amalgamation directory that will contain configuration subdirectory // for package tests out of source tree build. // dir_path out_dir ("build-installed"); - r.status |= run_b ( - step_id::b_test_installed_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create('" + out_dir.representation () + "'" + mods + ")", - step_args (env_args, step_id::b_test_installed_create), - step_args (config_args, step_id::b_test_installed_create)); + { + step_id b (step_id::b_test_installed_create); + step_id s (step_id::b_test_installed_create); - if (!r.status) - break; + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create('" + out_dir.representation () + "'" + mods + ")", + step_args (env_args, s), + step_args (config_args, s)); - // Configure subprojects and create buildspecs for their testing. + if (!r.status) + break; + } + + // Configure testable subprojects sequentially and test/build in + // parallel afterwards. // strings test_specs; for (const dir_path& d: subprj_dirs) @@ -1857,16 +2175,17 @@ build (size_t argc, const char* argv[]) // b configure(@) // // - // b.test-installed.configure - // + step_id b (step_id::b_test_installed_configure); + step_id s (step_id::b_test_installed_configure); + dir_path subprj_src_dir (exists (dist_src) ? dist_src / d - : build_dir / pkg_dir / d); + : main_pkg_conf / pkg_dir / d); dir_path subprj_out_dir (out_dir / d); r.status |= run_b ( - step_id::b_test_installed_configure, + b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, @@ -1874,8 +2193,8 @@ build (size_t argc, const char* argv[]) "configure('" + subprj_src_dir.representation () + "'@'" + subprj_out_dir.representation () + "')", - step_args (env_args, step_id::b_test_installed_configure), - step_args (config_args, step_id::b_test_installed_configure)); + step_args (env_args, s), + step_args (config_args, s)); if (!r.status) break; @@ -1891,146 +2210,495 @@ build (size_t argc, const char* argv[]) // // b test()... // - // b.test-installed.test - // - r.status |= run_b ( - step_id::b_test_installed_test, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - test_specs, - step_args (env_args, step_id::b_test_installed_test), - step_args (config_args, step_id::b_test_installed_test)); + { + step_id b (step_id::b_test_installed_test); + step_id s (step_id::b_test_installed_test); - if (!r.status) - break; + r.status |= run_b ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + test_specs, + step_args (env_args, s), + step_args (config_args, s)); + + if (!r.status) + break; + } } - // Run external tests. + // Run runtime and build-time tests. + // + // Note that we only build runtime tests for target packages and for + // host packages in self-hosted configurations. // - if (external_tests) + if (has_runtime_tests || has_buildtime_tests) { - // Configure. + // Create the required build configurations. + // + dir_path target_conf ("build-installed-bpkg"); + dir_path host_conf ("build-installed-bpkg-host"); + dir_path module_conf ("build-installed-bpkg-module"); + + // Create the target configuration if this is a target package having + // external runtime tests or a host/module package having external + // build-time tests. + // + bool create_target (target_pkg || has_buildtime_tests); + + // Note that even if there are no runtime tests for a host/module + // package, we still need to create the host/build2 configuration to + // configure the system package in. + // + bool create_host (host_pkg || module_pkg); + bool create_module (module_pkg || (host_pkg && has_buildtime_tests)); + + // Note: a module package cannot have runtime tests and so the module + // configuration is only created to serve build-time tests. Thus, the + // host or target configuration is always created as well and the + // module configuration is never a root configuration. + // + assert (create_target || create_host); + + // Root configuration through which we will be configuring the + // cluster. + // + const dir_path& root_conf (create_target ? target_conf : host_conf); + + // Runtime tests configuration. Should only be used if there are any. + // + const dir_path& runtime_tests_conf (target_pkg + ? target_conf + : host_conf); + + // Create the target configuration. // // bpkg create // - // bpkg.test-installed.create (bpkg.create) + if (create_target) + { + step_id b (step_id::bpkg_test_separate_installed_create); + + // Note that here and below the _for_* step ids are determined by + // the main package type (and, yes, that means we will use the same + // step ids for target and host configuration -- that, however, + // should be ok since host configuration will only be created in + // the self-hosted case). + // + step_id s ( + target_pkg + ? step_id::bpkg_test_separate_installed_create_for_target + : host_pkg + ? step_id::bpkg_test_separate_installed_create_for_host + : step_id::bpkg_test_separate_installed_create_for_module); + + // Note: no fallback for modules. + // + optional f (!module_pkg + ? step_id::bpkg_test_separate_installed_create + : optional ()); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create", + "-d", target_conf, + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (config_args, s, f)); + + if (!r.status) + break; + } + + // Create the host configuration. // - dir_path config_dir ("build-installed-bpkg"); + if (create_host) + { + // bpkg create --type host + // + step_id b (step_id::bpkg_test_separate_installed_create); - r.status |= run_bpkg ( - step_id::bpkg_test_installed_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create", - "-d", config_dir.string (), - "--wipe", + step_id s ( + host_pkg + ? step_id::bpkg_test_separate_installed_create_for_host + : step_id::bpkg_test_separate_installed_create_for_module); + + // Note: no fallback for modules. + // + optional f (!module_pkg + ? step_id::bpkg_test_separate_installed_create + : optional ()); - step_args (modules, - step_id::bpkg_test_installed_create, - step_id::bpkg_create), + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create", + "-d", host_conf, + "--type", "host", + "--name", "host", + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (config_args, s, f)); - step_args (env_args, - step_id::bpkg_test_installed_create, - step_id::bpkg_create), + if (!r.status) + break; + } - step_args (config_args, - step_id::bpkg_test_installed_create, - step_id::bpkg_create)); + // Create the module configuration. + // + // Note that we never build any tests in it but only configure the + // system package. Note, however, that the host/module package + // build-time tests can potentially build some other modules here. + // + if (create_module) + { + // b create() config.config.load=~build2 + // + step_id b (step_id::bpkg_test_separate_installed_create); - if (!r.status) - break; + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-V", + "create(" + module_conf.representation () + ",cc)", + "config.config.load=~build2", + "config.config.persist+='config.*'@unused=drop"); - change_wd (trace, &r.log, config_dir); + if (!r.status) + break; - // bpkg add + // bpkg create --existing --type build2 + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "create", + "--existing", + "-d", module_conf, + "--type", "build2", + "--name", "module"); + + if (!r.status) + break; + } + + // Link the configurations. // - // bpkg.test-installed.configure.add (bpkg.configure.add) + // bpkg link -d // - r.status |= run_bpkg ( - step_id::bpkg_test_installed_configure_add, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "add", + { + step_id b (step_id::bpkg_test_separate_installed_link); - step_args (env_args, - step_id::bpkg_test_installed_configure_add, - step_id::bpkg_configure_add), + if (create_target) + { + if (create_host) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", target_conf, + host_conf); - step_args (config_args, - step_id::bpkg_test_installed_configure_add, - step_id::bpkg_configure_add), + if (!r.status) + break; + } - repo); + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", target_conf, + module_conf); - if (!r.status) - break; + if (!r.status) + break; + } + } - // bpkg fetch + if (create_host) + { + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "link", + "-d", host_conf, + module_conf); + + if (!r.status) + break; + } + } + } + + // Add and fetch the repositories. + // + if (has_runtime_tests) + { + // bpkg add + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_add); + step_id s (step_id::bpkg_test_separate_installed_configure_add); + step_id f (step_id::bpkg_configure_add); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "add", + "-d", runtime_tests_conf, + step_args (env_args, s, f), + step_args (config_args, s, f), + repo); + + if (!r.status) + break; + } + + // bpkg fetch + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_fetch); + step_id s (step_id::bpkg_test_separate_installed_configure_fetch); + step_id f (step_id::bpkg_configure_fetch); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "fetch", + "-d", runtime_tests_conf, + step_args (env_args, s, f), + step_args (config_args, s, f), + trust_ops); + + if (!r.status) + break; + } + } + + if (has_buildtime_tests) + { + // bpkg add + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_add); + step_id s (step_id::bpkg_test_separate_installed_configure_add); + step_id f (step_id::bpkg_configure_add); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "add", + "-d", target_conf, + step_args (env_args, s, f), + step_args (config_args, s, f), + repo); + + if (!r.status) + break; + } + + // bpkg fetch + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_fetch); + step_id s (step_id::bpkg_test_separate_installed_configure_fetch); + step_id f (step_id::bpkg_configure_fetch); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "fetch", + "-d", target_conf, + step_args (env_args, s, f), + step_args (config_args, s, f), + trust_ops); + + if (!r.status) + break; + } + } + + // Configure all the packages using a single bpkg-pkg-build command. // - // bpkg.test-installed.configure.fetch (bpkg.configure.fetch) + // bpkg build --configure-only + // { }+ { ... } + // ... + // ?sys: // - r.status |= run_bpkg ( - step_id::bpkg_test_installed_configure_fetch, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "fetch", + strings pkg_args; + + if (has_runtime_tests) + { + // Note that only host package runtime tests can (but not + // necessarily) be configured in a linked configuration and require + // --config-name to be specified for them. + // + assert (!module_pkg); - step_args (env_args, - step_id::bpkg_test_installed_configure_fetch, - step_id::bpkg_configure_fetch), + string conf_name (runtime_tests_conf == root_conf + ? "" + : "host"); - step_args (config_args, - step_id::bpkg_test_installed_configure_fetch, - step_id::bpkg_configure_fetch), + bool og (!conf_name.empty ()); - trust_ops); + if (og) + { + pkg_args.push_back ("{"); - if (!r.status) - break; + if (!conf_name.empty ()) + { + pkg_args.push_back ("--config-name"); + pkg_args.push_back (conf_name); + } - // The test separate installed phase. + pkg_args.push_back ("}+"); + } + + if (og && runtime_tests.size () != 1) + pkg_args.push_back ("{"); + + for (auto t: runtime_tests) + pkg_args.push_back (t.string ()); + + if (og && runtime_tests.size () != 1) + pkg_args.push_back ("}"); + } + + if (has_buildtime_tests) + { + // Strip the build-time mark. + // + for (auto t: buildtime_tests) + pkg_args.push_back (t.dependency::string ()); + } + + pkg_args.push_back ("?sys:" + pkg); + + dir_path dist_root (rwd / dir_path ("dist-installed")); + + // Finally, configure all the test packages. // - if (!test (r, - rwd / dir_path ("dist-installed"), - true /* installed */, - true /* sys_dep */, - nullptr /* import */, - envvars)) - break; + { + step_id b (step_id::bpkg_test_separate_installed_configure_build); + + step_id g (step_id::bpkg_global_configure_build); // Global. + + step_id s ( + target_pkg + ? step_id::bpkg_test_separate_installed_create_for_target + : host_pkg + ? step_id::bpkg_test_separate_installed_create_for_host + : step_id::bpkg_test_separate_installed_create_for_module); + + step_id f (step_id::bpkg_test_separate_installed_configure_build); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-v", + "build", + "--configure-only", + "--checkout-root", dist_root, + "--yes", + "-d", root_conf, + step_args (env_args, g), + step_args (env_args, s, f), + step_args (config_args, g), + step_args (config_args, s, f), + "--", + pkg_args); - // Back to the test installed phase. + if (!r.status) + break; + } + + // Run external runtime tests. // - } + if (has_runtime_tests) + { + const dir_path& runtime_tests_conf (target_pkg + ? target_conf + : host_conf); - rm.status |= r.status; + change_wd (trace, &r.log, runtime_tests_conf); + + if (!test (r, + runtime_tests, + dist_root, + true /* installed */, + envvars)) + break; + } + + // Run external build-time tests. + // + if (has_buildtime_tests) + { + change_wd (trace, &r.log, rwd / target_conf); + + if (!test (r, + buildtime_tests, + dist_root, + true /* installed */, + envvars)) + break; + } + + rm.status |= r.status; + } } - // Back to the main phase. - // // Uninstall. // { operation_result& r (add_result ("uninstall")); - change_wd (trace, &r.log, pkg_config); + change_wd (trace, &r.log, rwd / main_pkg_conf); // bpkg uninstall // - // bpkg.uninstall - // + step_id b (step_id::bpkg_uninstall); + step_id s (step_id::bpkg_uninstall); + r.status |= run_bpkg ( - step_id::bpkg_uninstall, + b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "uninstall", - step_args (env_args, step_id::bpkg_uninstall), - step_args (config_args, step_id::bpkg_uninstall), + step_args (env_args, s), + step_args (config_args, s), pkg); if (!r.status) @@ -2048,16 +2716,6 @@ build (size_t argc, const char* argv[]) rm.status |= r.status; // Merge last in case of a break. - // Also merge statuses of the configure and test operations, which logs - // can potentially be shared across multiple steps and which results may - // not be the last in the list. - // - if (configure_result != nullptr) - rm.status |= configure_result->status; - - if (test_result != nullptr) - rm.status |= test_result->status; - // Unless there is an error (or worse) encountered, log the special 'end' // step and, if this step is specified in the interactive manifest value, // ask the user if to continue the task execution. -- cgit v1.1