diff options
Diffstat (limited to 'bbot/worker/worker.cxx')
| -rw-r--r-- | bbot/worker/worker.cxx | 6047 |
1 files changed, 5025 insertions, 1022 deletions
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index eb7f50b..8fb7796 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -1,26 +1,28 @@ // file : bbot/worker.cxx -*- C++ -*- -// license : TBC; see accompanying LICENSE file +// license : MIT; see accompanying LICENSE file #ifndef _WIN32 # include <signal.h> // signal() #else -# include <libbutl/win32-utility.hxx> +# include <libbutl/win32-utility.hxx> // SetErrorMode(), Sleep() #endif #include <map> #include <regex> -#include <cstring> // strchr() +#include <cstring> // strchr(), strncmp() #include <sstream> #include <iostream> -#include <algorithm> // find(), find_if(), remove_if() -#include <libbutl/b.mxx> -#include <libbutl/pager.mxx> -#include <libbutl/prompt.mxx> -#include <libbutl/utility.mxx> // to_utf8() -#include <libbutl/timestamp.mxx> -#include <libbutl/filesystem.mxx> -#include <libbutl/string-parser.mxx> +#include <libbutl/b.hxx> +#include <libbutl/pager.hxx> +#include <libbutl/prompt.hxx> +#include <libbutl/utility.hxx> // to_utf8(), eof() +#include <libbutl/timestamp.hxx> +#include <libbutl/filesystem.hxx> +#include <libbutl/string-parser.hxx> +#include <libbutl/manifest-serializer.hxx> + +#include <libbutl/json/parser.hxx> #include <libbbot/manifest.hxx> @@ -60,7 +62,7 @@ namespace bbot const size_t tftp_get_retries (3); // Task request retries (see startup()). } -bool +static bool exists (const dir_path& d) try { @@ -71,6 +73,17 @@ catch (const system_error& e) fail << "unable to stat path " << d << ": " << e << endf; } +static bool +exists (const path& f) +try +{ + return file_exists (f, true /* follow_symlinks */); +} +catch (const system_error& e) +{ + fail << "unable to stat path " << f << ": " << e << endf; +} + static dir_path current_directory () try @@ -82,11 +95,23 @@ catch (const system_error& e) fail << "unable to obtain current directory: " << e << endf; } -static dir_path -change_wd (tracer& t, string* log, const dir_path& d, bool create = false) -try +static void +#ifndef _WIN32 +mk_p (tracer& t, string* log, const dir_path& d, bool sudo = false) { - if (create) + if (sudo) + { + if (log != nullptr) + *log += "sudo mkdir -p " + d.representation () + '\n'; + + run_io (t, 0, 1, 2, "sudo", "mkdir", "-p", d); + } + else +#else +mk_p (tracer& t, string* log, const dir_path& d, bool = false) +{ +#endif + try { if (verb >= 3) t << "mkdir -p " << d; @@ -96,21 +121,55 @@ try try_mkdir_p (d); } + catch (const system_error& e) + { + fail << "unable to create directory " << d << ": " << e << endf; + } +} - dir_path r (current_directory ()); +static void +mk (tracer& t, string* log, const dir_path& d) +try +{ + if (verb >= 3) + t << "mkdir " << d; + if (log != nullptr) + *log += "mkdir " + d.representation () + '\n'; + + try_mkdir (d); +} +catch (const system_error& e) +{ + fail << "unable to create directory " << d << ": " << e << endf; +} + +static bool +empty (const dir_path& d) +try +{ + return dir_empty (d); +} +catch (const system_error& e) +{ + fail << "unable to scan directory " << d << ": " << e << endf; +} + +static void +cp_into (tracer& t, string* log, const path& p, const dir_path& d) +try +{ if (verb >= 3) - t << "cd " << d; + t << "cp " << p << ' ' << d; if (log != nullptr) - *log += "cd " + d.representation () + '\n'; + *log += "cp " + p.string () + ' ' + d.representation () + '\n'; - dir_path::current_directory (d); - return r; + cpfile_into (p, d); } catch (const system_error& e) { - fail << "unable to change current directory to " << d << ": " << e << endf; + fail << "unable to copy file " << p << " into " << d << ": " << e << endf; } static void @@ -133,6 +192,25 @@ catch (const system_error& e) } static void +mv_into (tracer& t, string* log, const path& from, const dir_path& into) +try +{ + if (verb >= 3) + t << "mv " << from << ' ' << into; + + if (log != nullptr) + *log += "mv " + from.representation () + ' ' + into.representation () + + "\n"; + + mventry_into (from, into); +} +catch (const system_error& e) +{ + fail << "unable to move entry '" << from << "' into '" << into << "': " << e + << endf; +} + +static void rm_r (tracer& t, string* log, const dir_path& d) try { @@ -149,77 +227,330 @@ catch (const system_error& e) fail << "unable to remove directory " << d << ": " << e << endf; } +static dir_path +change_wd (tracer& t, string* log, const dir_path& d, bool create = false) +try +{ + if (create) + mk_p (t, log, d); + + dir_path r (current_directory ()); + + if (verb >= 3) + t << "cd " << d; + + if (log != nullptr) + *log += "cd " + d.representation () + '\n'; + + dir_path::current_directory (d); + return r; +} +catch (const system_error& e) +{ + fail << "unable to change current directory to " << d << ": " << e << endf; +} + // Step IDs. // +// NOTE: keep ids ordered according to the sequence of steps and remember to +// update unreachable breakpoint checks if changing anything here. +// enum class step_id { - bpkg_module_create, - bpkg_module_configure_add, - bpkg_module_configure_fetch, - bpkg_module_configure_build, - bpkg_module_update, - bpkg_module_test, - bpkg_create, + // Common fallbacks for bpkg_*_create/b_test_installed_create and + // bpkg_*_configure_build/b_test_installed_configure, respectively. Note: + // not breakpoints. + // + b_create, + b_configure, + + // Note that bpkg_module_* options are only used if the main package is a + // build system module (using just ~build2 otherwise). They also have no + // fallback (build system modules are just too different to try to handle + // them together with target and host; e.g., install root). However, + // bpkg_module_create is complemented with arguments from un-prefixed step + // ids, the same way as other *.create[_for_*] steps (note that un-prefixed + // steps are not fallbacks, they are always added first). + // + bpkg_create, // Breakpoint and base. + bpkg_target_create, //: b_create, bpkg_create + bpkg_host_create, //: b_create, bpkg_create + bpkg_module_create, //: no fallback + + bpkg_link, + bpkg_configure_add, bpkg_configure_fetch, - bpkg_configure_build, + + // Global (as opposed to package-specific) bpkg-pkg-build options (applies + // to all *_configure_build* steps). Note: not a breakpoint. + // + bpkg_global_configure_build, + + // Note that bpkg_configure_build serves as a breakpoint for the + // bpkg-pkg-build call that configures (at once) the main package and all + // its external tests. + // + bpkg_configure_build, // Breakpoint and base. + bpkg_target_configure_build, //: b_configure, bpkg_configure_build + bpkg_host_configure_build, //: b_configure, bpkg_configure_build + bpkg_module_configure_build, //: b_configure, bpkg_configure_build + bpkg_update, bpkg_test, - bpkg_test_separate_configure_build, - bpkg_test_separate_update, - bpkg_test_separate_test, + + // Note that separate test packages are configured as part of the + // bpkg_configure_build step above with options taken from + // bpkg_{target,host}_configure_build, depending on tests package type. + // + bpkg_test_separate_update, //: bpkg_update + bpkg_test_separate_test, //: bpkg_test + + // Note that we only perform the installation tests if this is a target + // package or a self-hosted configuration. Also note that this step is + // considered disabled if any of the bpkg_bindist_* steps is explicitly + // enabled. + // bpkg_install, - b_test_installed_create, - b_test_installed_configure, + + bbot_install_ldconfig, // Note: disabled by default. + + // Note that the bpkg_bindist_* steps are mutually exclusive and the latest + // status change for them (via the leading +/- characters in the prefix) + // overrides all the previous ones. Disabled by default. + // + bpkg_bindist_debian, + bpkg_bindist_fedora, + bpkg_bindist_archive, + + // Note that this step is considered disabled unless one of the + // bpkg_bindist_* steps is explicitly enabled. Note: not a breakpoint. + // + bbot_sys_install, + + bbot_sys_install_apt_get_update, + bbot_sys_install_apt_get_install, + bbot_sys_install_dnf_install, + bbot_sys_install_tar_extract, + + bbot_sys_install_ldconfig, // Note: disabled by default. + + // Note: skipped for modules. + // + b_test_installed_create, //: b_create + b_test_installed_configure, //: b_configure b_test_installed_test, - bpkg_test_installed_create, - bpkg_test_installed_configure_add, - bpkg_test_installed_configure_fetch, - bpkg_test_separate_installed_configure_build, - bpkg_test_separate_installed_update, - bpkg_test_separate_installed_test, + + // Note that for a host package this can involve both run-time and build- + // time tests (which means we may also need a shared configuration for + // modules). + // + // The *_for_{target,host,module} denote main package type, not + // configuration being created, which will always be target (more precisely, + // target or host, but host only in a self-hosted case, which means it's + // the same as target). + // + // Note that if this is a non-self-hosted configuration, we can only end up + // here if building target package and so can just use *_create and *_build + // values in buildtabs. + // + bpkg_test_separate_installed_create, // Breakpoint and base. + bpkg_test_separate_installed_create_for_target, //: bpkg_test_separate_installed_create + bpkg_test_separate_installed_create_for_host, //: bpkg_test_separate_installed_create + bpkg_test_separate_installed_create_for_module, //: no fallback + + bpkg_test_separate_installed_link, // breakpoint only + bpkg_test_separate_installed_configure_add, //: bpkg_configure_add + bpkg_test_separate_installed_configure_fetch, //: bpkg_configure_fetch + + bpkg_test_separate_installed_configure_build, // Breakpoint and base. + bpkg_test_separate_installed_configure_build_for_target, //: bpkg_test_separate_installed_configure_build + bpkg_test_separate_installed_configure_build_for_host, //: bpkg_test_separate_installed_configure_build + bpkg_test_separate_installed_configure_build_for_module, //: bpkg_test_separate_installed_configure_build + + bpkg_test_separate_installed_update, //: bpkg_update + bpkg_test_separate_installed_test, //: bpkg_test + + bbot_sys_uninstall_apt_get_remove, + bbot_sys_uninstall_dnf_remove, + bpkg_uninstall, + + bbot_bindist_upload, // Note: disabled by default, not a breakpoint. + + // Note that this step is considered disabled unless the upload/ directory + // is not empty. Note: not a breakpoint. + // + bbot_upload, + + bbot_upload_tar_create, + bbot_upload_tar_list, + end }; static const strings step_id_str { - "bpkg.module.create", - "bpkg.module.configure.add", - "bpkg.module.configure.fetch", - "bpkg.module.configure.build", - "bpkg.module.update", - "bpkg.module.test", + "b.create", + "b.configure", + "bpkg.create", + "bpkg.target.create", + "bpkg.host.create", + "bpkg.module.create", + + "bpkg.link", + "bpkg.configure.add", "bpkg.configure.fetch", + + "bpkg.global.configure.build", + "bpkg.configure.build", + "bpkg.target.configure.build", + "bpkg.host.configure.build", + "bpkg.module.configure.build", + "bpkg.update", "bpkg.test", - "bpkg.test-separate.configure.build", + "bpkg.test-separate.update", "bpkg.test-separate.test", + "bpkg.install", + "bbot.install.ldconfig", + + "bpkg.bindist.debian", + "bpkg.bindist.fedora", + "bpkg.bindist.archive", + + "bbot.sys-install", + "bbot.sys-install.apt-get.update", + "bbot.sys-install.apt-get.install", + "bbot.sys-install.dnf.install", + "bbot.sys-install.tar.extract", + "bbot.sys-install.ldconfig", + "b.test-installed.create", "b.test-installed.configure", "b.test-installed.test", - "bpkg.test-installed.create", - "bpkg.test-installed.configure.add", - "bpkg.test-installed.configure.fetch", + + "bpkg.test-separate-installed.create", + "bpkg.test-separate-installed.create_for_target", + "bpkg.test-separate-installed.create_for_host", + "bpkg.test-separate-installed.create_for_module", + + "bpkg.test-separate-installed.link", + "bpkg.test-separate-installed.configure.add", + "bpkg.test-separate-installed.configure.fetch", + "bpkg.test-separate-installed.configure.build", + "bpkg.test-separate-installed.configure.build_for_target", + "bpkg.test-separate-installed.configure.build_for_host", + "bpkg.test-separate-installed.configure.build_for_module", + "bpkg.test-separate-installed.update", "bpkg.test-separate-installed.test", + + "bbot.sys-uninstall.apt-get.remove", + "bbot.sys-uninstall.dnf.remove", + "bpkg.uninstall", + + "bbot.bindist.upload", + + "bbot.upload", + "bbot.upload.tar.create", + "bbot.upload.tar.list", + "end"}; +static inline const string& +to_string (step_id s) +{ + return step_id_str[static_cast<size_t> (s)]; +} + using std::regex; namespace regex_constants = std::regex_constants; using regexes = vector<regex>; +// UTF-8-sanitize and log the line. Also print it to a tracer, if specified, +// or to stderr otherwise at verbosity level 3 or higher. +// +static void +log_line (string&& l, string& log, tracer* trace = nullptr) +{ + if (verb >= 3) + { + if (trace != nullptr) + *trace << l; + else + text << l; + } + + to_utf8 (l, '?', codepoint_types::graphic, U"\n\r\t"); + + log += l; + log += '\n'; +} + +#ifndef _WIN32 +const char* comment_begin ("#"); +#else +const char* comment_begin ("rem"); +#endif + +static void +log_step_id (tracer& t, string& log, step_id id) +{ + string ts (to_string (system_clock::now (), + "%Y-%m-%d %H:%M:%S %Z", + true /* special */, + true /* local */)); + + const string& sid (to_string (id)); + + l3 ([&]{t << "step id: " << sid << ' ' << ts;}); + + log += comment_begin; + log += " step id: "; + log += sid; + log += ' '; + log += ts; + log += '\n'; +} + +// Add the specified string to the log as a comment. Unless the string is +// empty (e.g., a blank line to separate comments), also trace it. +// +static void +log_comment (tracer& t, string& log, const string& s) +{ + if (!s.empty ()) + l3 ([&]{t << s;}); + + log += comment_begin; + + if (!s.empty ()) + { + log += ' '; + log += s; + } + + log += '\n'; +} + // Run the worker script command. Name is used for logging and diagnostics // only. Match lines read from the command's stderr against the regular // expressions and return the warning result status (instead of success) in // case of a match. Save the executed command into last_cmd. // +// Redirect stdout to stderr if the out_* arguments are not specified (out_str +// is NULL and out_file is empty; must never be specified both). Otherwise, +// save the process output into the variable referenced by out_str, if +// specified, and to the file referenced by out_file otherwise. Note: in the +// former case assumes that the output will always fit into the pipe buffer. +// // If bkp_step is present and is equal to the command step, then prior to // running this command ask the user if to continue or abort the task // execution. If bkp_status is present, then ask for that if the command @@ -228,36 +559,28 @@ using regexes = vector<regex>; // For the special end step no command is executed. In this case only the user // is potentially prompted and the step is traced/logged. // +// If specified, the pre-run callback is called after the step id is logged +// but before the command is logged/executed. +// +using pre_run_function = void (); + template <typename... A> static result_status run_cmd (step_id step, tracer& t, - string& log, const regexes& warn_detect, + string& log, + const function<pre_run_function>& pre_run, + optional<string>* out_str, const path& out_file, + const regexes& warn_detect, const string& name, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const process_env& pe, A&&... a) { - // UTF-8-sanitize and log the diagnostics. Also print the raw diagnostics - // to stderr at verbosity level 3 or higher. - // - auto add = [&log, &t] (string&& s, bool trace = true) - { - if (verb >= 3) - { - if (trace) - t << s; - else - text << s; - } - - to_utf8 (s, '?', codepoint_types::graphic, U"\n\r\t"); - - log += s; - log += '\n'; - }; + assert (out_str == nullptr || out_file.empty ()); string next_cmd; @@ -266,60 +589,54 @@ run_cmd (step_id step, // struct abort {}; - auto prompt = [&last_cmd, &next_cmd, &add] (const string& what) + auto prompt = [&aux_env, &last_cmd, &next_cmd, &t, &log] (const string& what) { diag_record dr (text); dr << '\n' << what << '\n' - << " current dir: " << current_directory () << '\n' - << " environment: " << ops.env_script () << ' ' << ops.env_target (); + << " current dir: " << current_directory () << '\n' + << " environment: " << ops.env_script () << ' ' << ops.env_target (); + + if (!aux_env.empty ()) + { + dr << '\n' + << " auxiliary environment:"; + + for (const string& e: aux_env) + dr << '\n' + << " " << e; + } if (!last_cmd.empty ()) dr << '\n' - << " last command: " << last_cmd; + << " last command: " << last_cmd; if (!next_cmd.empty ()) dr << '\n' - << " next command: " << next_cmd; + << " next command: " << next_cmd; dr.flush (); if (!yn_prompt ( "continue execution (or you may shutdown the machine)? [y/n]")) { - add ("execution aborted by interactive user"); + log_line ("execution aborted by interactive user", log, &t); throw abort (); } }; - auto prompt_step = [step, &t, &log, &bkp_step, &prompt] () + auto prompt_step = [step, &t, &log, &bkp_step, &prompt, &pre_run] () { - const string& sid (step_id_str[static_cast<size_t> (step)]); - // Prompt the user if the breakpoint is reached. // if (bkp_step && *bkp_step == step) - prompt (sid + " step reached"); + prompt (to_string (step) + " step reached"); - string ts (to_string (system_clock::now (), - "%Y-%m-%d %H:%M:%S %Z", - true /* special */, - true /* local */)); + log_step_id (t, log, step); - // Log the step id and the command to be executed. - // - l3 ([&]{t << "step id: " << sid << ' ' << ts;}); - -#ifndef _WIN32 - log += "# step id: "; -#else - log += "rem step id: "; -#endif - log += sid; - log += ' '; - log += ts; - log += '\n'; + if (pre_run) + pre_run (); }; try @@ -335,6 +652,8 @@ run_cmd (step_id step, prompt_step (); + // Log the command to be executed. + // t (c, n); log += next_cmd; @@ -347,27 +666,47 @@ run_cmd (step_id step, { try { - fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate. + // Redirect stdout to stderr, if the caller is not interested in it. + // + // Text mode seems appropriate. + // + fdpipe out_pipe (out_str != nullptr ? fdopen_pipe () : fdpipe ()); + fdpipe err_pipe (fdopen_pipe ()); + + // If the output file is specified, then open "half-pipe". + // + if (!out_file.empty ()) + try + { + out_pipe.out = fdopen (out_file, + fdopen_mode::out | fdopen_mode::create); + } + catch (const io_error& e) + { + fail << "unable to open " << out_file << ": " << e; + } process pr ( - process_start_callback (cmdc, - fdopen_null (), // Never reads from stdin. - 2, // 1>&2 - pipe, - pe, - forward<A> (a)...)); + process_start_callback ( + cmdc, + fdopen_null (), // Never reads from stdin. + out_pipe.out != nullfd ? out_pipe.out.get () : 2, + err_pipe, + pe, + forward<A> (a)...)); - pipe.out.close (); + out_pipe.out.close (); + err_pipe.out.close (); { // Skip on exception. // - ifdstream is (move (pipe.in), fdstream_mode::skip); + ifdstream is (move (err_pipe.in), + fdstream_mode::skip, + ifdstream::badbit); - for (string l; is.peek () != ifdstream::traits_type::eof (); ) + for (string l; !eof (getline (is, l)); ) { - getline (is, l); - // Match the log line with the warning-detecting regular // expressions until the first match. // @@ -392,17 +731,27 @@ run_cmd (step_id step, } } - add (move (l), false /* trace */); + log_line (move (l), log); } } if (!pr.wait ()) { const process_exit& e (*pr.exit); - add (name + " " + to_string (e)); + log_line (name + ' ' + to_string (e), log, &t); r = e.normal () ? result_status::error : result_status::abnormal; } + // Only read the buffered output if the process terminated normally. + // + if (out_str != nullptr && pr.exit->normal ()) + { + // Note: shouldn't throw since the output is buffered. + // + ifdstream is (move (out_pipe.in)); + *out_str = is.read_text (); + } + last_cmd = move (next_cmd); if (bkp_status && r >= *bkp_status) @@ -434,6 +783,61 @@ run_cmd (step_id step, } } +template <typename... A> +static result_status +run_cmd (step_id step, + tracer& t, + string& log, + optional<string>* out_str, const path& out_file, + const regexes& warn_detect, + const string& name, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const process_env& pe, + A&&... a) +{ + return run_cmd (step, + t, + log, + nullptr /* pre_run */, + out_str, out_file, + warn_detect, + name, + bkp_step, bkp_status, aux_env, last_cmd, + pe, + forward<A> (a)...); +} + +template <typename V, typename... A> +static result_status +run_bpkg (step_id step, + const V& envvars, + tracer& t, + string& log, + const function<pre_run_function>& pre_run, + optional<string>& out, + const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const char* verbosity, + const string& cmd, A&&... a) +{ + return run_cmd (step, + t, + log, + pre_run, + &out, path () /* out_file */, + warn_detect, + "bpkg " + cmd, + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("bpkg", envvars), + verbosity, cmd, forward<A> (a)...); +} + template <typename V, typename... A> static result_status run_bpkg (step_id step, @@ -442,15 +846,68 @@ run_bpkg (step_id step, string& log, const regexes& warn_detect, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const char* verbosity, + const string& cmd, A&&... a) +{ + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file */, + warn_detect, + "bpkg " + cmd, + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("bpkg", envvars), + verbosity, cmd, forward<A> (a)...); +} + +template <typename... A> +static result_status +run_bpkg (step_id step, + tracer& t, + string& log, + const function<pre_run_function>& pre_run, + optional<string>& out, + const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const char* verbosity, + const string& cmd, A&&... a) +{ + const char* const* envvars (nullptr); + + return run_bpkg (step, + envvars, + t, + log, + pre_run, + out, + warn_detect, + bkp_step, bkp_status, aux_env, last_cmd, + verbosity, cmd, forward<A> (a)...); +} + +template <typename V, typename... A> +static result_status +run_bpkg (step_id step, + const V& envvars, + tracer& t, + string& log, const path& out, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const char* verbosity, const string& cmd, A&&... a) { return run_cmd (step, t, - log, warn_detect, + log, nullptr /* out_str */, out, warn_detect, "bpkg " + cmd, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, process_env ("bpkg", envvars), verbosity, cmd, forward<A> (a)...); } @@ -459,9 +916,32 @@ template <typename... A> static result_status run_bpkg (step_id step, tracer& t, + string& log, const path& out, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const char* verbosity, + const string& cmd, A&&... a) +{ + const char* const* envvars (nullptr); + + return run_bpkg (step, + envvars, + t, + log, out, warn_detect, + bkp_step, bkp_status, aux_env, last_cmd, + verbosity, cmd, forward<A> (a)...); +} + +template <typename... A> +static result_status +run_bpkg (step_id step, + tracer& t, string& log, const regexes& warn_detect, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const char* verbosity, const string& cmd, A&&... a) @@ -472,7 +952,7 @@ run_bpkg (step_id step, envvars, t, log, warn_detect, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, verbosity, cmd, forward<A> (a)...); } @@ -484,6 +964,7 @@ run_b (step_id step, string& log, const regexes& warn_detect, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const char* verbosity, const strings& buildspecs, A&&... a) @@ -498,10 +979,11 @@ run_b (step_id step, } return run_cmd (step, - t, - log, warn_detect, + t, log, + nullptr /* out_str */, path () /* out_file */, + warn_detect, name, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, process_env ("b", envvars), verbosity, buildspecs, forward<A> (a)...); } @@ -514,15 +996,17 @@ run_b (step_id step, string& log, const regexes& warn_detect, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const char* verbosity, const string& buildspec, A&&... a) { return run_cmd (step, - t, - log, warn_detect, + t, log, + nullptr /* out_str */, path () /* out_file */, + warn_detect, "b " + buildspec, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, process_env ("b", envvars), verbosity, buildspec, forward<A> (a)...); } @@ -534,6 +1018,7 @@ run_b (step_id step, string& log, const regexes& warn_detect, const optional<step_id>& bkp_step, const optional<result_status>& bkp_status, + const strings& aux_env, string& last_cmd, const char* verbosity, const string& buildspec, A&&... a) @@ -543,10 +1028,131 @@ run_b (step_id step, envvars, t, log, warn_detect, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, verbosity, buildspec, forward<A> (a)...); } +template <typename... A> +static result_status +run_ldconfig (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + A&&... a) +{ + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo ldconfig", + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("sudo"), + "ldconfig", forward<A> (a)...); +} + +template <typename... A> +static result_status +run_apt_get (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const string& cmd, A&&... a) +{ + // Note: dumps some of its diagnostics to stdout. + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo apt-get " + cmd, + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("sudo"), + "apt-get", cmd, forward<A> (a)...); +} + +template <typename... A> +static result_status +run_dnf (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + const string& cmd, A&&... a) +{ + // Note: dumps some of its diagnostics to stdout. + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo dnf " + cmd, + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("sudo"), + "dnf", cmd, forward<A> (a)...); +} + +#ifndef _WIN32 +template <typename... A> +static result_status +run_tar (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + bool sudo, + A&&... a) +{ + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + sudo ? "sudo tar" : "tar", + bkp_step, bkp_status, aux_env, last_cmd, + process_env (sudo ? "sudo" : "tar"), + sudo ? "tar" : nullptr, forward<A> (a)...); +} +#else +template <typename... A> +static result_status +run_tar (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional<step_id>& bkp_step, + const optional<result_status>& bkp_status, + const strings& aux_env, + string& last_cmd, + bool /* sudo */, + A&&... a) +{ + // Note: using bsdtar which can unpack .zip archives (and also not an MSYS + // executable). + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "bsdtar", + bkp_step, bkp_status, aux_env, last_cmd, + process_env ("bsdtar"), + forward<A> (a)...); +} +#endif + // Upload compressed manifest to the specified TFTP URL with curl. Issue // diagnostics and throw failed on invalid manifest or process management // errors and throw io_error for input/output errors or non-zero curl exit. @@ -567,6 +1173,11 @@ upload_manifest (tracer& trace, // other hand, uploading from a file appears to work reliably (we still // get an odd error on Windows from time to time with larger uploads). // + // Let's not break lines in the manifest values not to further increase + // the size of the manifest encoded representation. Also here we don't + // care much about readability of the manifest since it will only be read + // by the bbot agent anyway. + // #if 0 // Note: need to add compression support if re-enable this. tftp_curl c (trace, @@ -577,7 +1188,7 @@ upload_manifest (tracer& trace, "--tftp-blksize", tftp_blksize, "--max-time", tftp_put_timeout); - manifest_serializer s (c.out, url); + manifest_serializer s (c.out, url, true /* long_lines */); m.serialize (s); c.out.close (); #else @@ -585,9 +1196,9 @@ upload_manifest (tracer& trace, try { tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4")); - ofdstream ofs (tmp.path); + ofdstream ofs (tmp.path, fdopen_mode::binary); olz4stream ozs (ofs, 9, 5 /* 256KB */, nullopt /* content_size */); - manifest_serializer s (ozs, tmp.path.string ()); + manifest_serializer s (ozs, tmp.path.string (), true /* long_lines */); m.serialize (s); ozs.close (); ofs.close (); @@ -632,24 +1243,39 @@ upload_manifest (tracer& trace, } } +static strings +parse_auxiliary_environment (const string&, const char*); // See below. + +static const string worker_checksum ("5"); // Logic version. + static int bbot:: build (size_t argc, const char* argv[]) { - using namespace bpkg; + using std::map; + using std::multimap; using string_parser::unquote; + using serializer = manifest_serializer; + using serialization = manifest_serialization; + + using namespace bpkg; + tracer trace ("build"); // Our overall plan is as follows: // // 1. Parse the task manifest (it should be in CWD). // - // 2. Run bpkg to create the configuration, add the repository, and - // configure, build, test, optionally install, test installed and - // uninstall the package all while saving the logs in the result manifest. + // 2. Run bpkg to create the package/tests configurations, add the + // repository to them, and configure, build, test, optionally install or + // alternatively bindist and sys-install, test installed, and + // (sys-)uninstall the package all while saving the logs in the result + // manifest. // - // 3. Upload the result manifest. + // 3. Upload the result manifest and, optionally, the build artifacts. + // + // NOTE: consider updating worker_checksum if making any logic changes. // // Note also that we are being "watched" by the startup version of us which // will upload an appropriate result in case we exit with an error. So here @@ -659,42 +1285,35 @@ build (size_t argc, const char* argv[]) task_manifest tm ( parse_manifest<task_manifest> (path ("task.manifest"), "task")); + // Reset the dependency checksum if the task's worker checksum doesn't match + // the current one. + // + if (!tm.worker_checksum || *tm.worker_checksum != worker_checksum) + tm.dependency_checksum = nullopt; + result_manifest rm { tm.name, tm.version, result_status::success, - operation_results {} + operation_results {}, + worker_checksum, + nullopt /* dependency_checksum */ }; - // Reserve storage large enough to hold all the potential operation results - // without reallocations. Note that this is not an optimization but is - // required to make sure that element references are not invalidated when - // new results are added. - // - size_t max_results (6); - rm.results.reserve (max_results); - - auto add_result = [&rm, max_results] (string o) -> operation_result& + auto add_result = [&rm] (string o) -> operation_result& { - assert (rm.results.size () < max_results); - rm.results.push_back ( operation_result {move (o), result_status::success, ""}); return rm.results.back (); }; - // Note that we don't consider the build system module configuring and - // testing during the "pre-step" as separate operations and share the - // operation logs with the "main" configure and test steps (see below). - // Thus, we save pointers to the added result objects for the subsequent - // use. - // - operation_result* configure_result (nullptr); - operation_result* test_result (nullptr); - dir_path rwd; // Root working directory. + // Archive of the build artifacts for upload. + // + path upload_archive ("upload.tar"); + // Resolve the breakpoint specified by the interactive manifest value into // the step id or the result status breakpoint. If the breakpoint is // invalid, then log the error and abort the build. Note that we reuse the @@ -704,8 +1323,49 @@ build (size_t argc, const char* argv[]) optional<result_status> bkp_status; string last_cmd; // Used in the user prompt. + // Parse the auxiliary environment, if present, to dump it into the + // configure operation log and to use it in the interactive build user + // prompt. Note that this environment is already set by the parent process. + // + strings aux_env; + for (;;) // The "breakout" loop. { + auto fail_operation = [&trace] (operation_result& r, + const string& e, + result_status s, + const string& name = "", + uint64_t line = 0, + uint64_t column = 0) + { + string prefix; + + if (!name.empty ()) + { + prefix += name; + prefix += ':'; + + if (line != 0) + { + prefix += to_string (line); + prefix += ':'; + + if (column != 0) + { + prefix += to_string (column); + prefix += ':'; + } + } + + prefix += ' '; + } + + l3 ([&]{trace << prefix << e;}); + + r.log += prefix + "error: " + e + '\n'; + r.status = s; + }; + // Regular expressions that detect different forms of build2 toolchain // warnings. Accidently (or not), they also cover GCC and Clang warnings // (for the English locale). @@ -723,7 +1383,7 @@ build (size_t argc, const char* argv[]) for (const string& re: tm.unquoted_warning_regex ()) wre.emplace_back (re, f); - if (tm.interactive) + if (tm.interactive && *tm.interactive != "none") { const string& b (*tm.interactive); @@ -745,171 +1405,571 @@ build (size_t argc, const char* argv[]) if (!bkp_step && !bkp_status) { - string e ("invalid interactive build breakpoint '" + b + "'"); - - l3 ([&]{trace << e;}); - - operation_result& r (add_result ("configure")); - - r.log = "error: " + e + '\n'; - r.status = result_status::abort; + fail_operation (add_result ("configure"), + "invalid interactive build breakpoint '" + b + '\'', + result_status::abort); break; } } + // Parse the auxiliary environment, if present. + // + if (tm.auxiliary_environment) + { + // Note: cannot throw since has already been called successfully by the + // parent process. + // + aux_env = parse_auxiliary_environment (*tm.auxiliary_environment, + comment_begin); + } + // Split the argument into prefix (empty if not present) and unquoted - // value. Return nullopt if the prefix is invalid. + // value (absent if not present) and determine the step status. If the + // prefix is present and is prepended with the '+'/'-' character, then the + // respective step needs to be enabled/disabled. Return nullopt if the + // prefix is invalid. + // + // Note that arguments with absent values are normally used to + // enable/disable steps and are omitted from the command lines. // - auto parse_arg = [] (const string& a) -> optional<pair<string, string>> + struct argument + { + string prefix; + + // Absent if the argument value is an empty unquoted string. + // + optional<string> value; + + // True - enable, false - disable, nullopt - neutral. + // + optional<bool> step_status; + }; + + auto parse_arg = [] (const string& a) -> optional<argument> { size_t p (a.find_first_of (":=\"'")); + auto value = [] (const string& v) + { + return !v.empty () ? unquote (v) : optional<string> (); + }; + if (p == string::npos || a[p] != ':') // No prefix. - return make_pair (string (), unquote (a)); + return argument {string (), value (a), nullopt}; + + string prefix (a, 0, p); + + optional<bool> enable; + if (prefix[0] == '+' || prefix[0] == '-') + { + enable = (prefix[0] == '+'); + + prefix.erase (0, 1); + + if (prefix != "bpkg.update" && + prefix != "bpkg.test" && + prefix != "bpkg.test-separate.update" && + prefix != "bpkg.test-separate.test" && + prefix != "bpkg.install" && + prefix != "bbot.install.ldconfig" && + prefix != "bpkg.bindist.debian" && + prefix != "bpkg.bindist.fedora" && + prefix != "bpkg.bindist.archive" && + prefix != "bbot.sys-install" && + prefix != "bbot.sys-install.ldconfig" && + prefix != "b.test-installed.test" && + prefix != "bpkg.test-separate-installed.update" && + prefix != "bpkg.test-separate-installed.test" && + prefix != "bbot.bindist.upload" && + prefix != "bbot.upload") + { + return nullopt; // Prefix is invalid. + } + } for (const string& id: step_id_str) { - if (a.compare (0, p, id, 0, p) == 0 && - (id.size () == p || (id.size () > p && id[p] == '.'))) - return make_pair (a.substr (0, p), unquote (a.substr (p + 1))); + size_t n (prefix.size ()); + if (id.compare (0, n, prefix) == 0 && + (id.size () == n || (id.size () > n && id[n] == '.'))) + return argument {move (prefix), value (a.substr (p + 1)), enable}; } return nullopt; // Prefix is invalid. }; - // Enter split arguments into a map. Those without a prefix are - // entered for the *.create steps. + // Keep track of explicitly enabled/disabled steps. + // + map<string, bool> step_statuses; + + // Return true if the step is explicitly enabled via a +<prefix>:[<value>] + // environment/configuration argument. // - auto add_arg = [] (std::multimap<string, string>& args, - pair<string, string>&& a) + auto step_enabled = [&step_statuses] (step_id step) -> bool { - if (!a.first.empty ()) - args.emplace (move (a)); - else + auto i (step_statuses.find (to_string (step))); + return i != step_statuses.end () && i->second; + }; + + // Return true if the step is explicitly disabled via a -<prefix>:[<value>] + // environment/configuration argument. + // + auto step_disabled = [&step_statuses] (step_id step) -> bool + { + auto i (step_statuses.find (to_string (step))); + return i != step_statuses.end () && !i->second; + }; + + // Save a step status. + // + // Note that since the bpkg.bindist.* steps are mutually exclusive we only + // keep the latest status change (see above for details). + // + auto step_status = [&step_statuses] (const string& step, bool status) + { + if (step.compare (0, 13, "bpkg.bindist.") == 0) { - args.emplace ("bpkg.create", a.second); - args.emplace ("b.test-installed.create", a.second); - args.emplace ("bpkg.test-installed.create", move (a.second)); + step_statuses.erase ("bpkg.bindist.debian"); + step_statuses.erase ("bpkg.bindist.fedora"); + step_statuses.erase ("bpkg.bindist.archive"); } + + step_statuses[step] = status; }; - // Parse configuration arguments. Report failures to the bbot controller. + // Parse the environment, target configuration, and build package + // configuration arguments. + // + // NOTE: keep this parsing order intact so that, for example, a build + // package configuration argument can override step status specified + // by a target configuration argument. + // + + // Parse environment arguments. + // + multimap<string, string> modules; + multimap<string, string> env_args; + + for (size_t i (1); i != argc; ++i) + { + const char* a (argv[i]); + optional<argument> v (parse_arg (a)); + + if (!v) + fail << "invalid environment argument prefix in '" << a << "'"; + + bool mod (v->value && + (*v->value)[0] != '-' && + v->value->find ('=') == string::npos); + + if (mod && + !v->prefix.empty () && + v->prefix != "b.create" && + v->prefix != "bpkg.create" && + v->prefix != "bpkg.target.create" && + v->prefix != "bpkg.host.create" && + v->prefix != "bpkg.module.create" && + v->prefix != "b.test-installed.create" && + v->prefix != "bpkg.test-separate-installed.create" && + v->prefix != "bpkg.test-separate-installed.create_for_target" && + v->prefix != "bpkg.test-separate-installed.create_for_host" && + v->prefix != "bpkg.test-separate-installed.create_for_module") + fail << "invalid module prefix in '" << a << "'"; + + if (v->step_status) + step_status (v->prefix, *v->step_status); + + if (v->value) + (mod ? modules : env_args).emplace (make_pair (move (v->prefix), + move (*v->value))); + } + + // Parse target configuration arguments. Report failures to the bbot + // controller. // - std::multimap<string, string> config_args; + multimap<string, string> tgt_args; - for (const string& c: tm.config) + for (const string& c: tm.target_config) { - optional<pair<string, string>> v (parse_arg (c)); + optional<argument> v (parse_arg (c)); if (!v) { rm.status |= result_status::abort; - l3 ([&]{trace << "invalid configuration argument prefix in " + l3 ([&]{trace << "invalid target configuration argument prefix in " << "'" << c << "'";}); break; } - if (v->second[0] != '-' && v->second.find ('=') == string::npos) + if (v->value && + (*v->value)[0] != '-' && + v->value->find ('=') == string::npos) { rm.status |= result_status::abort; - l3 ([&]{trace << "invalid configuration argument '" << c << "'";}); + l3 ([&]{trace << "invalid target configuration argument '" << c + << "'";}); break; } - add_arg (config_args, move (*v)); + if (v->step_status) + step_status (v->prefix, *v->step_status); + + if (v->value) + tgt_args.emplace (make_pair (move (v->prefix), move (*v->value))); } if (!rm.status) break; - // Parse environment arguments. + // Parse the build package configuration represented as a whitespace + // separated list of the following potentially quoted bpkg-pkg-build + // command arguments: // - std::multimap<string, string> modules; - std::multimap<string, string> env_args; - - for (size_t i (1); i != argc; ++i) + // <option>... + // <config-var>... + // ([{ <config-var>... }+] (?[sys:]|sys:)<pkg-name>[<version-spec>])... + // ( { <config-var>... }+ <pkg-name>)... + // + // If the package configuration is specified, then parse it into the + // following lists/maps: + // + // - The prefixed global options and configuration variables map + // (pkg_args). Added to the command lines at the corresponding steps + // after potential environment and target configuration arguments. + // + // - The unprefixed global options list (pkg_config_opts). Specified after + // all the prefixed global options on the bpkg-pkg-build command line at + // the bpkg.configure.build step. + // + // - The main package-specific configuration variables list + // (pkg_config_vars). Specified for the main package only on the + // bpkg-pkg-build command line at the bpkg.configure.build step, + // wherever it is configured. Also specified on the b-configure command + // line at the b.test-installed.configure step. + // + // - The main package-specific dependency packages list + // (pkg_config_main_deps), potentially with their own configuration + // variables (but not options). Only configured where the main package + // is configured with the bpkg-pkg-build command line at the + // bpkg.configure.build step. + // + // - The global system dependency packages list (pkg_config_glob_deps). + // Configured in all configurations with the bpkg-pkg-build command line + // at the bpkg.configure.build step. + // + // - The main and external test package-specific configuration variables + // map (pkg_config_pkgs). Specified on the bpkg-pkg-build command lines + // at the bpkg.configure.build and + // bpkg.test-separate-installed.configure.build steps. Package names + // other than the main and external test package names are silently + // ignored. + // + multimap<string, string> pkg_args; + strings pkg_config_opts; + strings pkg_config_vars; + vector<pair<string, strings>> pkg_config_main_deps; // ?<pkg>, sys:<pkg> + vector<pair<string, strings>> pkg_config_glob_deps; // ?sys:<pkg> + map<string, strings> pkg_config_pkgs; // <pkg> + + if (!tm.package_config.empty ()) { - const char* a (argv[i]); - optional<pair<string, string>> v (parse_arg (a)); + struct abort {}; - if (!v) - fail << "invalid environment argument prefix in '" << a << "'"; + auto fail = [&tm, &add_result, &fail_operation] (const string& d, + bool throw_abort = true) + { + fail_operation (add_result ("configure"), + "invalid package configuration: " + d + + "\n info: package configuration: '" + + tm.package_config + '\'', + result_status::abort); - bool mod (v->second[0] != '-' && v->second.find ('=') == string::npos); + if (throw_abort) + throw abort (); + }; - if (mod && !v->first.empty () && - v->first != "bpkg.create" && - v->first != "b.test-installed.create" && - v->first != "bpkg.test-installed.create") - fail << "invalid module prefix in '" << a << "'"; + try + { + strings argsv (string_parser::parse_quoted (tm.package_config, + false /* unquote */)); + + cli::vector_scanner scanv (argsv); + cli::group_scanner args (scanv); + + while (args.more ()) + { + string a (args.next ()); + + // Unless the argument is an unquoted dependency (starts with `?` or + // `sys:`), first try to interpret it as a prefixed option/variable + // and/or step id status (enabled/disabled). + // + if (!(a[0] == '?' || a.compare (0, 4, "sys:") == 0)) + { + optional<argument> v (parse_arg (a)); + + // Note that we only assume an argument as prefixed if the prefix + // is a known step id. Otherwise, we interpret the argument as + // unprefixed global option, variable, or a package spec. + // + if (v && !v->prefix.empty ()) + { + if (v->value && + (*v->value)[0] != '-' && + v->value->find ('=') == string::npos) + fail ("invalid prefixed argument '" + a + '\''); - add_arg (mod ? modules : env_args, move (*v)); + if (args.group ().more ()) + fail ("unexpected options group for prefixed argument '" + a + + '\''); + + if (v->step_status) + step_status (v->prefix, *v->step_status); + + if (v->value) + pkg_args.emplace (make_pair (move (v->prefix), + move (*v->value))); + + continue; + } + } + + a = unquote (a); + + // Return true if the argument is an option. + // + // Note that options with values can only be specified using + // the single argument notation. + // + auto opt = [] (const string& a) + { + // Make sure that -- or - is always followed by some characters. + // + return a.compare (0, 2, "--") == 0 ? a.size () > 2 : + a[0] == '-' ? a.size () > 1 : + false ; + }; + + // Return true if the argument is a configuration variable. + // + auto var = [] (const string& a) + { + // Note: we need to be careful not to misinterpret + // '?libfoo == 1.0.0' as a variable. + // + return a.compare (0, 7, "config.") == 0 && + a.find ('=') != string::npos; + }; + + bool o (opt (a)); + bool v (var (a)); + + cli::scanner& ag (args.group ()); + + if (o) // Option. + { + if (ag.more ()) + fail ("unexpected options group for option '" + a + '\''); + + pkg_config_opts.push_back (move (a)); + } + else if (v) // Configuration variable. + { + if (ag.more ()) + fail ("unexpected options group for configuration variable '" + + a + '\''); + + pkg_config_vars.push_back (move (a)); + } + else // Dependency or build-to-hold package. + { + // Note that we consider a system package as a dependency + // regardless whether it is prefixed with '?' or not. + // + strings vars; + while (ag.more ()) + { + string da (unquote (ag.next ())); + if (!var (da)) + fail ("argument is not a configuration variable for " + "dependency " + a + ": '" + da + '\''); + + vars.push_back (move (da)); + } + + // Add the system dependency packages (prefixed with `?sys:`) to + // a separate list, to specify them globally on the + // bpkg-pkg-build command line for configuring them in all the + // (being) created configurations. + // + // Note, though, that we will handle the build-to-hold system + // packages (prefixed with `sys:`) in the same way as non system + // dependencies, since such an auto-configuration is only + // supported by bpkg-pkg-build for system dependencies. In the + // future, we may support that on the bbot worker level by, for + // example, specifying all the configurations manually for the + // build-to-hold system packages and also specifying them as a + // system dependencies globally. We need to be careful to make + // sure that these dependencies are also auto-configured for the + // private configurations potentially created by bpkg-pkg-build. + // + // Also note that in the future we may allow package-specific + // --config-uuid options to only configure such packages in the + // specified configurations. We may also invent the special + // 00000000-0000-0000-0000-000000000005 configuration id to, for + // example, only configure them at the + // bpkg.test-separate-installed.configure.build step. + // + if (a.compare (0, 5, "?sys:") == 0) // Global system dependency. + { + pkg_config_glob_deps.push_back (make_pair (move (a), + move (vars))); + } + else if (a[0] == '?' || // Main package dependency. + a.compare (0, 4, "sys:") == 0) + { + pkg_config_main_deps.push_back (make_pair (move (a), + move (vars))); + } + else // Build-to-hold package. + { + if (vars.empty ()) + fail ("no configuration variables specified for package '" + + a + '\''); + + auto i (pkg_config_pkgs.find (a)); + + if (i == pkg_config_pkgs.end ()) + { + pkg_config_pkgs.emplace (move (a), move (vars)); + } + else + { + strings& vs (i->second); + vs.insert (vs.end (), + make_move_iterator (vars.begin ()), + make_move_iterator (vars.end ())); + } + } + } + } + } + catch (const cli::exception& e) + { + fail (e.what (), false /* throw_abort */); + break; + } + catch (const string_parser::invalid_string& e) + { + fail (e.what (), false /* throw_abort */); + break; + } + catch (const abort&) + { + break; + } } - // Return command arguments for the specified step id. Arguments with more - // specific prefixes come last. + // Return command arguments for the specified step id, complementing + // *.create[_for_*] steps with un-prefixed arguments. If no arguments are + // specified for the step then use the specified fallbacks, potentially + // both. Arguments with more specific prefixes come last. Optionally, + // search for arguments starting from the specified step id rather than + // from the least specific one (tool id). // - auto step_args = [] (const std::multimap<string, string>& args, + auto step_args = [] (const multimap<string, string>& args, step_id step, - optional<step_id> fallback = nullopt) -> strings + optional<step_id> fallback1 = nullopt, + optional<step_id> fallback2 = nullopt, + optional<step_id> start_step = nullopt) -> cstrings { - strings r; - const string& sid (step_id_str[static_cast<size_t> (step)]); + cstrings r; - // If no arguments found for the step id, then use the fallback step id, - // if specified. + // Add arguments for a specified, potentially empty, prefix. // - const string& s (args.find (sid) == args.end () && fallback - ? step_id_str[static_cast<size_t> (*fallback)] - : sid); - - for (size_t n (0);; ++n) + auto add_args = [&args, &r] (const string& prefix) { - n = s.find ('.', n); - - auto range ( - args.equal_range (n == string::npos ? s : string (s, 0, n))); + auto range (args.equal_range (prefix)); for (auto i (range.first); i != range.second; ++i) - r.emplace_back (i->second); + r.emplace_back (i->second.c_str ()); + }; - if (n == string::npos) + // Add un-prefixed arguments if this is one of the *.create[_for_*] + // steps. + // + switch (step) + { + case step_id::b_create: + case step_id::bpkg_create: + case step_id::bpkg_target_create: + case step_id::bpkg_host_create: + case step_id::bpkg_module_create: + case step_id::b_test_installed_create: + case step_id::bpkg_test_separate_installed_create: + case step_id::bpkg_test_separate_installed_create_for_target: + case step_id::bpkg_test_separate_installed_create_for_host: + case step_id::bpkg_test_separate_installed_create_for_module: + { + add_args (""); break; + } + default: break; } - return r; - }; + auto add_step_args = [&add_args] (step_id step, + optional<step_id> start_step = nullopt) + { + const string& s (to_string (step)); - // Search for config.install.root variable. If it is present and has a - // non-empty value, then test the package installation and uninstall. Note - // that passing [null] value would be meaningless, so we don't recognize - // it as a special one. While at it, cache the bpkg.create args for later - // use. - // - dir_path install_root; - strings cargs (step_args (config_args, step_id::bpkg_create)); - { - size_t n (19); - auto space = [] (char c) {return c == ' ' || c == '\t';}; + size_t n; - for (const string& s: reverse_iterate (cargs)) - { - if (s.compare (0, n, "config.install.root") == 0 && - (s[n] == '=' || space (s[n]))) + if (start_step) { - while (space (s[n])) ++n; // Skip spaces. - if (s[n] == '=') ++n; // Skip the equal sign. - while (space (s[n])) ++n; // Skip spaces. + const string& ss (to_string (*start_step)); - // Note that the config.install.root variable value may - // potentially be quoted. - // - install_root = dir_path (unquote (string (s, n, s.size () - n))); - break; + assert (s.size () >= ss.size () && + s.compare (0, ss.size (), ss) == 0 && + (s.size () == ss.size () || s[ss.size ()] == '.')); + + n = ss.size (); } + else + n = 0; + + for (;; ++n) + { + n = s.find ('.', n); + + add_args (n == string::npos ? s : string (s, 0, n)); + + if (n == string::npos) + break; + } + }; + + // If no arguments found for the step id, then use the fallback step + // ids, if specified. + // + if (args.find (to_string (step)) != args.end ()) + { + add_step_args (step, start_step); } - } + else + { + // Note that if we ever need to specify fallback pairs with common + // ancestors, we may want to suppress duplicate ancestor step ids. + // + if (fallback1) + add_step_args (*fallback1); + + if (fallback2) + add_step_args (*fallback2); + } + + return r; + }; // bpkg-rep-fetch trust options. // @@ -935,6 +1995,7 @@ build (size_t argc, const char* argv[]) const version& ver (tm.version); const string repo (tm.repository.string ()); const dir_path pkg_dir (pkg + '-' + ver.string ()); + const string pkg_var (tm.name.variable ()); // Specify the revision explicitly for the bpkg-build command not to end // up with a race condition building the latest revision rather than the @@ -951,7 +2012,7 @@ build (size_t argc, const char* argv[]) // Query the project's build system information with `b info`. // auto prj_info = [&trace] (const dir_path& d, - bool ext_mods, + b_info_flags fl, const char* what) { // Note that the `b info` diagnostics won't be copied into any of the @@ -962,7 +2023,7 @@ build (size_t argc, const char* argv[]) // try { - return b_info (d, ext_mods, verb, trace); + return b_info (d, fl, verb, trace); } catch (const b_error& e) { @@ -974,10 +2035,14 @@ build (size_t argc, const char* argv[]) } }; - b_project_info prj; // Package project information. - rwd = current_directory (); + // Create directory for the build artifacts to archive and upload. + // + dir_path upload_dir ("upload"); + + mk (trace, nullptr /* log */, upload_dir); + // If the package comes from a version control-based repository, then we // will also test its dist meta-operation. Specifically, we will checkout // the package outside the configuration directory passing --checkout-root @@ -988,6 +2053,11 @@ build (size_t argc, const char* argv[]) dir_path dist_root (rwd / dir_path ("dist")); dir_path dist_src (dist_root / pkg_dir); + dir_path dist_install_root (rwd / dir_path ("dist-install")); + dir_path dist_install_src (dist_install_root / pkg_dir); + + dir_path dist_installed_root (rwd / dir_path ("dist-installed")); + // Redistribute the package source directory (pkg_dir) checked out into // the directory other than the configuration directory (dist_root) and // replace it with the newly created distribution. Assume that the current @@ -996,12 +2066,12 @@ build (size_t argc, const char* argv[]) // for the build2 process. Return true if the dist meta-operation // succeeds. // - auto redist = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd] + auto redist = [&trace, &wre, &bkp_step, &bkp_status, &aux_env, &last_cmd] (step_id step, operation_result& r, const dir_path& dist_root, const dir_path& pkg_dir, // <name>-<version> - const char* import = nullptr, + const optional<string>& import = nullopt, const small_vector<string, 1>& envvars = {}) { // Temporarily change the current directory to the distribution root @@ -1026,7 +2096,7 @@ build (size_t argc, const char* argv[]) step, envvars, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, "-v", "config.dist.root=" + redist_root.string (), import, @@ -1045,998 +2115,3763 @@ build (size_t argc, const char* argv[]) return true; }; - // The module phase. + // Note that if this is not a self-hosted configuration, then we do not + // build external runtime tests nor run internal for host or module + // packages because the assumption is that they have been built/run (and + // with buildtab settings such as warnings, etc) when testing the + // self-hosted configuration this non-self-hosted one is based on. Also, + // by the same reason, we don't install tools or modules nor generate the + // binary distribution packages for them for non-self-hosted + // configurations. + // + // Actually, it could make sense to build and install tools and module + // from a target configuration in this case. But that means for a + // non-self-hosted configuration a tool/module may want to test two + // things: its output build and its own build, which means we would need a + // way to control which of the two things (or both) are to be tested + // (think of two cross-compiler configurations, Emscripten and MinGW: for + // the former a source code generator would normally only want to test the + // output while for the latter -- both; maybe we could have a `cross-host` + // class, meaning that the configuration is not host itself but its target + // is). In any case, seeing that there is no way to verify such own build + // works, we ignore this for now. + // + // Also note that build system modules can only have external build-time + // tests (which is verified by bpkg-rep-fetch) and target packages cannot + // have external build-time tests (which we verify ourselves). + // + bool selfhost (tm.host && *tm.host); + + // Detect if the package is of the target, host, or module type. + // + auto requirement = [&tm] (const char* id) + { + return find_if (tm.requirements.begin (), + tm.requirements.end (), + [id] (const requirement_alternatives& r) + { + if (r.size () == 1) + { + const requirement_alternative& a (r[0]); + return find (a.begin (), a.end (), id) != a.end (); + } + + return false; + }) != tm.requirements.end (); + }; + + bool module_pkg (pkg.compare (0, 10, "libbuild2-") == 0); + bool bootstrap (module_pkg && requirement ("bootstrap")); + bool host_pkg (!module_pkg && requirement ("host")); + bool target_pkg (!module_pkg && !host_pkg); + + // Don't generate binary packages for tools or modules for non-self-hosted + // configurations (see above for details). + // + optional<step_id> bindist; + + if (target_pkg || selfhost) + { + if (step_enabled (step_id::bpkg_bindist_debian)) + bindist = step_id::bpkg_bindist_debian; + else if (step_enabled (step_id::bpkg_bindist_fedora)) + bindist = step_id::bpkg_bindist_fedora; + else if (step_enabled (step_id::bpkg_bindist_archive)) + bindist = step_id::bpkg_bindist_archive; + } + + bool sys_install (bindist && !step_disabled (step_id::bbot_sys_install)); + bool bindist_upload (bindist && step_enabled (step_id::bbot_bindist_upload)); + + // Unless a bpkg.bindist.* step is enabled or bpkg.install step is + // disabled, search for config.install.root variable. If it is present and + // has a non-empty value, then test the package installation and + // uninstall. Note that passing [null] value would be meaningless, so we + // don't recognize it as a special one. + // + // Note that the host package can only be installed for a self-hosted + // configuration, using bpkg configuration of the target type. + // + // Also note that the module package is always installed for a self-hosted + // configuration (and never otherwise), using config.install.root + // specified for ~build2 configuration. + // + // If present, indicates that the install, test installed, and uninstall + // operations need to be tested. + // + // Note that the main package may not support the install operation. We, + // however, can only detect that after the package is configured. If + // that's the case, we will disable the steps which may not be performed + // for such a package (bpkg.install, bpkg.bindist.*, etc) later, after the + // package is configured. + // + optional<dir_path> install_root; + + // While building and running tests against the installation created + // either from source or from the archive distribution package we will + // make the bin/ subdirectory of config.install.root, if specified, the + // first entry in the PATH environment variable, except for build system + // modules which supposedly don't install any executables. + // + // Note that normally the config.install.root is expected to be prefixed + // with the bpkg.target.create or, as a fallback, b.create or bpkg.create + // step ids. However, for testing of the relocatable installations it can + // be desirable to extract the archive distribution package content at the + // bbot.sys-install.tar.extract step into a different installation + // directory. If that's the case, then this directory needs to also be + // specified as bbot.sys-install:config.install.root. If specified, this + // directory will be preferred as a base for forming the bin/ directory + // path. + // + optional<dir_path> install_bin; + + auto config_install_root = [&step_args, &tgt_args] + (step_id s, + optional<step_id> f1 = nullopt, + optional<step_id> f2 = nullopt) + -> optional<dir_path> + { + size_t n (19); + auto space = [] (char c) {return c == ' ' || c == '\t';}; + + for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2))) + { + if (strncmp (a, "config.install.root", n) == 0 && + (a[n] == '=' || space (a[n]))) + { + while (space (a[n])) ++n; // Skip spaces. + if (a[n] == '=') ++n; // Skip the equal sign. + while (space (a[n])) ++n; // Skip spaces. + + // Note that the config.install.root variable value may potentially + // be quoted. + // + return dir_path (unquote (a + n)); + } + } + + return nullopt; + }; + + if ((target_pkg || selfhost) && + !bindist && + !step_disabled (step_id::bpkg_install)) + { + if (!module_pkg) + { + install_root = config_install_root (step_id::bpkg_target_create, + step_id::b_create, + step_id::bpkg_create); + + if (install_root) + install_bin = *install_root / dir_path ("bin"); + } + else + install_root = dir_path (); + } + + // Split external test packages into the runtime and build-time lists. + // + // Note that runtime and build-time test packages are always configured in + // different bpkg configurations, since they can depend on different + // versions of the same package. + // + small_vector<test_dependency, 1> runtime_tests; + small_vector<test_dependency, 1> buildtime_tests; + + for (test_dependency& t: tm.tests) + { + if (t.buildtime) + buildtime_tests.push_back (move (t)); + else if (target_pkg || selfhost) + runtime_tests.push_back (move (t)); + } + + bool has_buildtime_tests (!buildtime_tests.empty ()); + bool has_runtime_tests (!runtime_tests.empty ()); + + // Abort if a target package has external build-time tests. + // + if (target_pkg && has_buildtime_tests) + { + fail_operation ( + add_result ("configure"), + "build-time tests in package not marked with `requires: host`", + result_status::abort); + + break; + } + + // Create the required build configurations. + // + // Note that if this is a target package, then we intentionally do not + // create host or module configuration letting the automatic private + // configuration creation to take its course (since that would probably be + // the most typical usage scenario). + // + // Also note that we may need a separate target configuration to build the + // host package for installation. This is required to avoid a potential + // conflict between the main package and a tool it may try to run during + // the build. We also do the same for module packages which, while cannot + // have build-time dependencies, could have private code generators. This + // configuration needs to have the target type (so that it uses any + // build-time dependencies from build-host/module configurations). Note + // also that we currently only do this for self-hosted configuration + // (since we don't install otherwise, see above). + // + dir_path target_conf ("build"); + dir_path host_conf ("build-host"); + dir_path module_conf ("build-module"); + dir_path install_conf ("build-install"); + + // Main package config. + // + const dir_path& main_pkg_conf (target_pkg ? target_conf : + host_pkg ? host_conf : + module_conf); + + // Create the target configuration if this is a target package or if the + // host/module package has external build-time tests. // + bool create_target (target_pkg || has_buildtime_tests); - // If this is a build system module, perform a "pre-step" by building it - // in a separate configuration reproducing the one used to build build2 - // itself. Note that the configuration and the environment options and - // variables are not passed to commands that may affect this - // configuration. + // Create the host configuration if this is a host package. + // + // Also create it for the module package with external build-time tests. + // The idea is to be able to test a tool which might only be tested via + // the module. To be precise, we need to check that the tests package has + // a build-time dependency (on the tool) but that's not easy to do and so + // we will create a host configuration if a module has any build-time + // tests. + // + bool create_host (host_pkg || (module_pkg && has_buildtime_tests)); + + // Create the module configuration if the package is a build system + // module. + // + // Also create it for the host package with the external build-time tests, + // so that a single build2 configuration is used for both target and host + // packages (this is important in case they happen to use the same + // module). // - bool module (pkg.compare (0, 10, "libbuild2-") == 0); - dir_path module_dir ("build-module"); + bool create_module (module_pkg || (host_pkg && has_buildtime_tests)); - // If this is a build system module that requires bootstrap, then its - // importation into the dependent (test) projects cannot be configured and - // the corresponding config.import.* variable needs to be specified on the - // bpkg/build2 command line as a global override, whenever required. + // Create the configuration for installing the main package (potentially + // as a part of generating binary distribution package) of the host or + // module type, unless it's not supposed to be installed. // - // Note that such a module must be explicitly marked with `requires: - // bootstrap` in its manifest. This can only be detected after the module - // is configured and its manifest available. + bool create_install (!target_pkg && (install_root || bindist)); + + // Configuration where the package will be installed from. // - bool bootstrap (false); + dir_path effective_install_conf ( + rwd / (create_install ? install_conf : main_pkg_conf)); - // Note that we will parse the package manifest right after the package is - // configured. + // Root configuration through which we will be configuring the cluster + // (note: does not necessarily match the main package type). + // + // In other words, this is configuration that will be specified for + // bpkg-pkg-build as the current configuration (via -d). It must be the + // configuration that links to all the other configurations, except + // install. + // + // Note that the install configuration, if present, is either the + // cluster's "second root" (for a host package) or is an independent + // cluster (for a module package). In either case it needs to additionally + // be specified as a current configuration on the command line. + // + const dir_path& root_conf (create_target ? target_conf : + create_host ? host_conf : + module_conf); + + // Note that bpkg doesn't support configuring bootstrap module + // dependents well, not distinguishing such modules from regular ones + // (see pkg_configure() for details). Thus, we need to pass the + // !config.import.* global override wherever required ourselves. // - package_manifest pm; - path mf ("manifest"); + // Also note that since this override is global, it may only be specified + // globally on the bpkg command line (as opposed to package-specific + // overrides). + // + optional<string> bootstrap_import; + + if (bootstrap) + bootstrap_import = "!config.import." + pkg_var + '=' + + (rwd / main_pkg_conf).string (); - if (module) + // Configure. + // { - // Configure. + operation_result* pr (&add_result ("configure")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + // If we have auxiliary environment, show it in the logs. // + if (!aux_env.empty ()) { - operation_result& r (add_result ("configure")); - configure_result = &r; + for (const string& e: aux_env) + { + r.log += e; + r.log += '\n'; + } - // Noop, just for the log record. + // Add a trailing blank line to separate this from the rest. // - change_wd (trace, &r.log, rwd); + r.log += '\n'; + } - // b create(<dir>) config.config.load=~build2 - // - // [bpkg.module.create] - // - // Note also that we suppress warnings about unused config.* values, - // such CLI configuration. - // - // What if a module wants to use CLI? The current thinking is that we - // will be "whitelisting" base (i.e., those that can plausibly be used - // by multiple modules) libraries and tools for use by build system - // modules. So if and when we whitelist CLI, we will add it here, next - // to cc. - // - r.status |= run_b ( - step_id::bpkg_module_create, + // Noop, just for the log record. + // + change_wd (trace, &r.log, rwd); + + // If we end up with multiple current configurations (root and install) + // then when running the bpkg-pkg-build command we need to specify the + // configuration for each package explicitly via --config-uuid. + // + // While it's tempting to use the --config-name option instead of + // --config-uuid, that wouldn't work well for multiple current + // configurations. For --config-name the configuration search is carried + // out among configurations explicitly linked to the main configuration + // only. That's in contrast to --config-uuid, when the whole + // configuration cluster is searched (see bpkg-pkg-build implementation + // for details). + // + // Let's not generate random UUIDs but use some predefined values which + // we can easily recognize in the build logs. + // + const char* target_uuid ("00000000-0000-0000-0000-000000000001"); + const char* host_uuid ("00000000-0000-0000-0000-000000000002"); + const char* module_uuid ("00000000-0000-0000-0000-000000000003"); + const char* install_uuid ("00000000-0000-0000-0000-000000000004"); + + // Let's however distinguish the target package as a simple common case + // and simplify the configuration creation and packages configuration + // commands making them more readable in the build log. For this simple + // case only one configuration needs to be created explicitly and so it + // doesn't need the UUID. Also there is no need in any package-specific + // options for the bpkg-pkg-build command in this case. + // + // Create the target configuration. + // + // bpkg create <env-modules> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + if (create_target) + { + step_id b (step_id::bpkg_create); // Breakpoint. + step_id s (step_id::bpkg_target_create); // Step. + step_id f1 (step_id::b_create); // First fallback. + step_id f2 (step_id::bpkg_create); // Second fallback. + + r.status |= run_bpkg ( + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, "-V", - "create(" + module_dir.representation () + ",cc)", - "config.config.load=~build2", - "config.config.persist+='config.*'@unused=drop"); + "create", + "-d", target_conf, + !target_pkg ? cstrings ({"--uuid", target_uuid}) : cstrings (), + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2), + step_args (pkg_args, s, f1, f2)); if (!r.status) break; + } - change_wd (trace, &r.log, module_dir); + // Create the host configurations. + // + if (create_host) + { + step_id b (step_id::bpkg_create); + + if (host_pkg && selfhost) + { + // Create the host configuration. + // + { + step_id s (step_id::bpkg_host_create); + step_id f1 (step_id::b_create); + step_id f2 (step_id::bpkg_create); - // bpkg create --existing + // bpkg create --type host <env-modules> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create", + "-d", host_conf, + "--type", "host", + "--uuid", host_uuid, + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2), + step_args (pkg_args, s, f1, f2)); + + if (!r.status) + break; + } + + // Create the install configuration. + // + // bpkg create <env-modules> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + if (create_install) + { + step_id s (step_id::bpkg_target_create); + step_id f1 (step_id::b_create); + step_id f2 (step_id::bpkg_create); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create", + "-d", install_conf, + "--uuid", install_uuid, + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2), + step_args (pkg_args, s, f1, f2)); + + if (!r.status) + break; + } + } + else + { + // b create(<dir>) config.config.load=~host + // + // Note also that we suppress warnings about unused config.* values. + // + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create(" + host_conf.representation () + ",cc)", + "config.config.load=~host", + "config.config.persist+='config.*'@unused=drop"); + + if (!r.status) + break; + + // bpkg create --existing --type host + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "create", + "--existing", + "-d", host_conf, + "--type", "host", + "--uuid", host_uuid); + + if (!r.status) + break; + } + } + + // Create the module configurations. + // + if (create_module) + { + step_id b (step_id::bpkg_create); + + // Create the module configuration. // - r.status |= run_bpkg ( - step_id::bpkg_module_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "create", - "--existing"); + { + // b create(<dir>) config.config.load=~build2 [<env-config-args> + // <tgt-config-args> + // <pkg-config-args>] + // + // Note also that we suppress warnings about unused config.* values. + // + // What if a module wants to use CLI? The current thinking is that + // we will be "whitelisting" base (i.e., those that can plausibly be + // used by multiple modules) libraries and tools for use by build + // system modules. So if and when we whitelist CLI, we will add it + // here, next to cc. + // + string mods; + cstrings eas; + cstrings cas; + cstrings pas; - if (!r.status) - break; + if (module_pkg && selfhost) + { + step_id s (step_id::bpkg_module_create); + + for (const char* m: step_args (modules, s)) + { + if (!mods.empty ()) + mods += ' '; + + mods += m; + } + + eas = step_args (env_args, s); + cas = step_args (tgt_args, s); + pas = step_args (pkg_args, s); + } + else + mods = "cc"; + + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create(" + module_conf.representation () + ',' + mods + ')', + "config.config.load=~build2", + "config.config.persist+='config.*'@unused=drop", + eas, + cas, + pas); - // bpkg add <env-config-args> <config-args> <repository-url> + if (!r.status) + break; + + // bpkg create --existing --type build2 + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "create", + "--existing", + "-d", module_conf, + "--type", "build2", + "--uuid", module_uuid); + + if (!r.status) + break; + } + + // Create the install configuration. // - // bpkg.module.configure.add (bpkg.configure.add) + if (create_install && module_pkg) + { + step_id s (step_id::bpkg_module_create); + + string mods; + for (const char* m: step_args (modules, s)) + { + if (!mods.empty ()) + mods += ' '; + + mods += m; + } + + // b create(<dir>) config.config.load=~build2 [<env-config-args> + // <tgt-config-args> + // <pkg-config-args>] + // + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create(" + install_conf.representation () + ',' + mods + ')', + "config.config.load=~build2", + "config.config.persist+='config.*'@unused=drop", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s)); + + if (!r.status) + break; + + // bpkg create --existing + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "create", + "--existing", + "-d", install_conf, + "--uuid", install_uuid); + + if (!r.status) + break; + } + } + + // Link the configurations. + // + // bpkg link -d <dir> <dir> + // + { + step_id b (step_id::bpkg_link); + + if (create_target) + { + if (create_host) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", target_conf, + host_conf); + + if (!r.status) + break; + } + + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", target_conf, + module_conf); + + if (!r.status) + break; + } + } + + if (create_host) + { + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", host_conf, + module_conf); + + if (!r.status) + break; + } + } + + // Link the install configuration only for the host package. Note that + // the module package may not have build-time dependencies and so + // doesn't need configurations for them. // + if (create_install && host_pkg) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", install_conf, + host_conf); + + if (!r.status) + break; + + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", install_conf, + module_conf); + + if (!r.status) + break; + } + } + } + + // Fetch repositories into the main package configuration, the target + // configuration for external build-time tests, if any, and the install + // configuration, if present. + // + // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args> + // <repository-url> + // + { + step_id b (step_id::bpkg_configure_add); + step_id s (step_id::bpkg_configure_add); + r.status |= run_bpkg ( - step_id::bpkg_module_configure_add, + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, "-v", "add", - - step_args (env_args, - step_id::bpkg_module_configure_add, - step_id::bpkg_configure_add), - - step_args (config_args, - step_id::bpkg_module_configure_add, - step_id::bpkg_configure_add), - + "-d", main_pkg_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), repo); if (!r.status) break; + } + + // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args> + // <trust-options> + // + { + step_id b (step_id::bpkg_configure_fetch); + step_id s (step_id::bpkg_configure_fetch); - // bpkg fetch <env-config-args> <config-args> <trust-options> - // - // bpkg.module.configure.fetch (bpkg.configure.fetch) - // r.status |= run_bpkg ( - step_id::bpkg_module_configure_fetch, + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, "-v", "fetch", - - step_args (env_args, - step_id::bpkg_module_configure_fetch, - step_id::bpkg_configure_fetch), - - step_args (config_args, - step_id::bpkg_module_configure_fetch, - step_id::bpkg_configure_fetch), - + "-d", main_pkg_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), trust_ops); if (!r.status) break; + } - // bpkg build --configure-only <package-name>/<package-version> - // - // [bpkg.module.configure.build] + if (create_install) + { + // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args> + // <repository-url> // - r.status |= run_bpkg ( - step_id::bpkg_module_configure_build, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "build", - "--configure-only", - "--checkout-root", dist_root, - "--yes", - pkg_rev); - - if (!r.status) - break; + { + step_id b (step_id::bpkg_configure_add); + step_id s (step_id::bpkg_configure_add); - rm.status |= r.status; + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "add", + "-d", install_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + repo); - bool dist (exists (dist_src)); - const dir_path& src_dir (dist ? dist_src : pkg_dir); + if (!r.status) + break; + } - // Note that being unable to parse the package manifest is likely to - // be an infrastructure problem, given that the package has been - // successfully configured. + // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args> + // <trust-options> // - pm = parse_manifest<package_manifest> (src_dir / mf, "package"); + { + step_id b (step_id::bpkg_configure_fetch); + step_id s (step_id::bpkg_configure_fetch); - bootstrap = find_if (pm.requirements.begin (), - pm.requirements.end (), - [] (const requirement_alternatives& r) - { - return r.size () == 1 && r[0] == "bootstrap"; - }) != pm.requirements.end (); + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "fetch", + "-d", install_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + trust_ops); + + if (!r.status) + break; + } + } - if (dist) + if (has_buildtime_tests) + { + // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args> + // <repository-url> + // { - // Note that we reuse the configure operation log for the dist - // meta-operation. - // - if (!redist (step_id::bpkg_module_configure_build, - r, - dist_root, - pkg_dir)) + step_id b (step_id::bpkg_configure_add); + step_id s (step_id::bpkg_configure_add); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "add", + "-d", target_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + repo); + + if (!r.status) break; + } - rm.status |= r.status; + // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args> + // <trust-options> + // + { + step_id b (step_id::bpkg_configure_fetch); + step_id s (step_id::bpkg_configure_fetch); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "fetch", + "-d", target_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + trust_ops); + + if (!r.status) + break; } } - // Update. + // Configure all the packages using a single bpkg-pkg-build command. + // + // First, prepare the common and package arguments. + // + // If no variables are specified in the package configuration, then add + // the config.<pkg>.develop=false variable for the main package instead + // to trigger its package skeleton creation and loading. Also add this + // variable for the external test packages for the same purpose. This + // way we make sure that these packages can be used as dependencies of + // dependents with configuration clauses. // + // Also add the dependency packages specified in the package + // configuration, if any, to configurations where the main package is + // being configured. + // + // Should we also add the dependency packages to configurations where + // the test packages are being configured? It feels like we shouldn't. + // Moreover, in the future we may decide to support specifying tests + // package configuration in the tests manifest value or some such. In + // this case a test package may have its own dependencies to be + // configured. What we could probably do now, is to never share a bpkg + // configuration between the main package and the tests packages if we + // configure any dependencies in it. Note that such dependencies may + // potentially be unsatisfactory for the test packages (unsatisfactory + // version, etc). This, however, seems rather far fetched so let's keep + // it simple for now. + // + strings common_args; + strings pkgs; + + if (target_pkg) // The simple common case (see above)? { - operation_result& r (add_result ("update")); + // The overall command looks like this (but some parts may be omitted): + // + // bpkg build --configure-only <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <pkg-config-opts> + // -- + // { <pkg-config-vars>|config.<pkg-name>.develop=false }+ <pkg> + // { <rtt-config-vars>|config.<runtime-test-name>.develop=false }+ <runtime-test>... + // { <dep-config-vars> }+ <main-dep>... + // <main-dep>... + // <glob-dep>... + // + step_id s (step_id::bpkg_target_configure_build); + step_id f1 (step_id::b_configure); + step_id f2 (step_id::bpkg_configure_build); + + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); + cstrings pas (step_args (pkg_args, s, f1, f2)); + + common_args.push_back ("--checkout-root"); + common_args.push_back (dist_root.string ()); + + common_args.insert (common_args.end (), eas.begin (), eas.end ()); + common_args.insert (common_args.end (), cas.begin (), cas.end ()); + common_args.insert (common_args.end (), pas.begin (), pas.end ()); - // Noop, just for the log record to reduce the potential confusion for - // the combined log reader due to the configure operation log sharing - // (see above for details). + // Add the main package. // - change_wd (trace, &r.log, current_directory ()); + pkgs.push_back ("{"); - // bpkg update <package-name> + // @@ config.<pkg>.develop=false // - // [bpkg.module.update] + // Only add the config.<pkg>.develop variable if there are no package + // configuration variables specified. // - r.status |= run_bpkg ( - step_id::bpkg_module_update, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "update", - pkg); + auto i (pkg_config_pkgs.find (tm.name.string ())); - if (!r.status) - break; + if (!pkg_config_vars.empty () || i != pkg_config_pkgs.end ()) + { + if (!pkg_config_vars.empty ()) + pkgs.insert (pkgs.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); - rm.status |= r.status; - } + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); + } +#if 1 + else + pkgs.push_back ("config." + pkg_var + ".develop=false"); +#endif - // Run the package internal tests if the test operation is supported by - // the project. - // - prj = prj_info (pkg_dir, true /* ext_mods */, "project"); + pkgs.push_back ("}+"); + + pkgs.push_back (pkg_rev); + + // Add the runtime test packages. + // + for (const auto& t: runtime_tests) + { + pkgs.push_back ("{"); + + // @@ config.<pkg>.develop=false + // + // Only add the config.<pkg>.develop variable if there are no + // package configuration variables specified. + // + auto i (pkg_config_pkgs.find (t.name.string ())); + + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); +#if 1 + else + pkgs.push_back ("config." + t.name.variable () + ".develop=false"); +#endif - if (find (prj.operations.begin (), prj.operations.end (), "test") != - prj.operations.end ()) + pkgs.push_back ("}+"); + + // Add test dependency package constraints (strip the potential + // reflection variable assignment; for example 'bar > 1.0.0'). + // + pkgs.push_back (t.dependency::string ()); + } + + // Add the main package dependencies. + // + for (const pair<string, strings>& d: pkg_config_main_deps) + { + if (!d.second.empty ()) + { + pkgs.push_back ("{"); + pkgs.insert (pkgs.end (), d.second.begin (), d.second.end ()); + pkgs.push_back ("}+"); + } + + pkgs.push_back (d.first); + } + } + else { - operation_result& r (add_result ("test")); - test_result = &r; + // The overall command looks like this (but some parts may be omitted): + // + // bpkg build --configure-only <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <pkg-config-opts> + // -- + // { <build-config> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <pkg-config-vars>|config.<pkg-name>.develop=false }+ <pkg> + // + // { <build-config> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <rtt-config-vars>|config.<runtime-test-name>.develop=false }+ <runtime-test>... + // + // { <install-config> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <pkg-config-vars> }+ <pkg> + // + // { <target-config> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <btt-config-vars>|config.<buildtime-test-name>.develop=false }+ <buildtime-test>... + // + // { <build-config> <install-config> <dep-config-vars> }+ <main-dep>... + // { <build-config> <install-config> }+ { <main-dep>... } + // <glob-dep>... + // - // Use --package-cwd to help ported to build2 third-party packages a - // bit (see bpkg-pkg-test(1) for details). + // Main package configuration name. // - // Note that internal tests that load the module itself don't make - // much sense, thus we don't pass the config.import.* variable on - // the command line for modules that require bootstrap. + const char* conf_uuid (host_pkg ? host_uuid : module_uuid); + + // Add the main package args. // - // bpkg test <package-name> + // Also add the external runtime test packages here since they share + // the configuration directory with the main package. // - // [bpkg.module.test] + { + step_id s (target_pkg ? step_id::bpkg_target_configure_build : + host_pkg ? step_id::bpkg_host_configure_build : + step_id::bpkg_module_configure_build); + + step_id f1 (step_id::b_configure); + step_id f2 (step_id::bpkg_configure_build); + + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); + cstrings pas (step_args (pkg_args, s, f1, f2)); + + // Add the main package. + // + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (conf_uuid); + + pkgs.push_back ("--checkout-root"); + pkgs.push_back (dist_root.string ()); + + pkgs.insert (pkgs.end (), eas.begin (), eas.end ()); + pkgs.insert (pkgs.end (), cas.begin (), cas.end ()); + pkgs.insert (pkgs.end (), pas.begin (), pas.end ()); + + // @@ config.<pkg>.develop=false + // + // Only add the config.<pkg>.develop variable if there are no + // package configuration variables specified. + // + auto i (pkg_config_pkgs.find (tm.name.string ())); + + if (!pkg_config_vars.empty () || i != pkg_config_pkgs.end ()) + { + if (!pkg_config_vars.empty ()) + pkgs.insert (pkgs.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); + + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); + } +#if 1 + else + pkgs.push_back ("config." + pkg_var + ".develop=false"); +#endif + + pkgs.push_back ("}+"); + + pkgs.push_back (pkg_rev); + } + + // Add the runtime test packages. + // + for (const auto& t: runtime_tests) + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (conf_uuid); + + pkgs.push_back ("--checkout-root"); + pkgs.push_back (dist_root.string ()); + + pkgs.insert (pkgs.end (), eas.begin (), eas.end ()); + pkgs.insert (pkgs.end (), cas.begin (), cas.end ()); + pkgs.insert (pkgs.end (), pas.begin (), pas.end ()); + + // @@ config.<pkg>.develop=false + // + // Only add the config.<pkg>.develop variable if there are no + // package configuration variables specified. + // + auto i (pkg_config_pkgs.find (t.name.string ())); + + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); +#if 1 + else + pkgs.push_back ("config." + t.name.variable () + ".develop=false"); +#endif + + pkgs.push_back ("}+"); + + // Strip the potential reflection variable assignment. + // + pkgs.push_back (t.dependency::string ()); + } + } + + // Add the main package configured in the install configuration and + // the external build-time test packages. // + { + step_id s (step_id::bpkg_target_configure_build); + step_id f1 (step_id::b_configure); + step_id f2 (step_id::bpkg_configure_build); + + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); + cstrings pas (step_args (pkg_args, s, f1, f2)); + + // Add the main package. + // + if (create_install) + { + common_args.push_back ("-d"); + common_args.push_back (install_conf.string ()); + + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (install_uuid); + + // Note that we do another re-distribution (with a separate + // --checkout-root) in case the package is missing file that + // are only used during installation. + // + pkgs.push_back ("--checkout-root"); + pkgs.push_back (dist_install_root.string ()); + + pkgs.insert (pkgs.end (), eas.begin (), eas.end ()); + pkgs.insert (pkgs.end (), cas.begin (), cas.end ()); + pkgs.insert (pkgs.end (), pas.begin (), pas.end ()); + + pkgs.insert (pkgs.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); + + auto i (pkg_config_pkgs.find (tm.name.string ())); + + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); + + pkgs.push_back ("}+"); + + pkgs.push_back (pkg_rev); + } + + // Add the build-time test packages. + // + // @@ config.<pkg>.develop=false + // + for (const auto& t: buildtime_tests) + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (target_uuid); + + pkgs.push_back ("--checkout-root"); + pkgs.push_back (dist_root.string ()); + + pkgs.insert (pkgs.end (), eas.begin (), eas.end ()); + pkgs.insert (pkgs.end (), cas.begin (), cas.end ()); + pkgs.insert (pkgs.end (), pas.begin (), pas.end ()); + + // @@ config.<pkg>.develop=false + // + // Only add the config.<pkg>.develop variable if there are no + // package configuration variables specified. + // + auto i (pkg_config_pkgs.find (t.name.string ())); + + if (i != pkg_config_pkgs.end ()) + pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ()); +#if 1 + else + pkgs.push_back ("config." + t.name.variable () + ".develop=false"); +#endif + + pkgs.push_back ("}+"); + + // Strip the build-time mark and potential reflection variable + // assignment. + // + pkgs.push_back (t.dependency::string ()); + } + } + + // Add the main package dependencies to those configurations where + // the main package is configured. + // + { + // Add dependencies which have some configuration variables + // specified and count the number of others. + // + size_t no_vars (0); + for (const pair<string, strings>& d: pkg_config_main_deps) + { + if (!d.second.empty ()) + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (conf_uuid); + + if (create_install) + { + pkgs.push_back ("--config-uuid"); + pkgs.push_back (install_uuid); + } + + pkgs.insert (pkgs.end (), d.second.begin (), d.second.end ()); + + pkgs.push_back ("}+"); + + pkgs.push_back (d.first); + } + else + ++no_vars; + } + + // Add dependencies which have no configuration variables specified. + // + if (no_vars != 0) + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-uuid"); + pkgs.push_back (conf_uuid); + + if (create_install) + { + pkgs.push_back ("--config-uuid"); + pkgs.push_back (install_uuid); + } + + pkgs.push_back ("}+"); + + if (no_vars != 1) + pkgs.push_back ("{"); + + for (const pair<string, strings>& d: pkg_config_main_deps) + { + if (d.second.empty ()) + pkgs.push_back (d.first); + } + + if (no_vars != 1) + pkgs.push_back ("}"); + } + } + } + + // Add the global system dependencies. + // + for (const pair<string, strings>& d: pkg_config_glob_deps) + pkgs.push_back (d.first); + + // Finally, configure all the packages. + // + { + step_id b (step_id::bpkg_configure_build); + step_id s (step_id::bpkg_global_configure_build); + + optional<string> dependency_checksum; + + // Only log configuration UUIDs if they are specified on the command + // line. + // + function<pre_run_function> log_uuids ( + [&r, &trace, + target_uuid, host_uuid, module_uuid, install_uuid, + target_pkg] () + { + if (!target_pkg) + { + auto log = [&r, &trace] (const char* uuid, const char* name) + { + string s (uuid); + s += " - "; + s += name; + + log_comment (trace, r.log, s); + }; + + log_comment (trace, r.log, ""); + + log (target_uuid, "target"); + log (host_uuid, "host"); + log (module_uuid, "module"); + log (install_uuid, "install"); + + log_comment (trace, r.log, ""); + } + }); + r.status |= run_bpkg ( - step_id::bpkg_module_test, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + b, + trace, r.log, + log_uuids, + dependency_checksum, + wre, + bkp_step, bkp_status, aux_env, last_cmd, "-v", - "test", - "--package-cwd", - pkg); + "build", + "--configure-only", + "--rebuild-checksum", + tm.dependency_checksum ? *tm.dependency_checksum : "", + "--yes", + "-d", root_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + common_args, + pkg_config_opts, + (has_runtime_tests || has_buildtime_tests + ? bootstrap_import + : nullopt), + "--", + pkgs); + + // The dependency checksum is tricky, here are the possibilities: + // + // - absent: bpkg terminated abnormally (or was not executed due to + // a breakpoint) -- nothing to do here. + // + // - empty: bpkg terminated normally with error before calculating the + // checksum -- nothing to do here either. + // + // - one line: bpkg checksum that we want. + // + // - many lines: someone else (e.g., buildfile) printed to stdout, + // which we consider an error. + // + if (dependency_checksum && !dependency_checksum->empty ()) + { + string& s (*dependency_checksum); + + // Make sure that the output contains a single line, and bail out + // with the error status if that's not the case. + // + if (s.find ('\n') == s.size () - 1) + { + s.pop_back (); + + // If the dependency checksum didn't change, then save it to the + // result manifest, clean the logs and bail out with the skip + // result status. + // + if (tm.dependency_checksum && *tm.dependency_checksum == s) + { + l3 ([&]{trace << "skip";}); + + rm.status = result_status::skip; + rm.dependency_checksum = move (s); + rm.results.clear (); + break; + } + + // Save the (new) dependency checksum to the result manifest. + // + // Also note that we save the checksum if bpkg failed after the + // checksum was printed. As a result, we won't be rebuilding the + // package until the error is fixed (in a package or worker) and + // the checksum changes, which feels like a proper behavior. + // + rm.dependency_checksum = move (s); + } + else + fail_operation (r, + "unexpected bpkg output:\n'" + s + '\'', + result_status::error); + } if (!r.status) break; + } - rm.status |= r.status; + // Redistribute the main package in both build and install + // configurations, if required (test packages will be handled later). + // + if (exists (dist_src)) + { + change_wd (trace, &r.log, main_pkg_conf); + + step_id b (step_id::bpkg_configure_build); + + if (!redist (b, r, dist_root, pkg_dir)) + break; } - } - // The main phase. - // + if (exists (dist_install_src)) + { + change_wd (trace, &r.log, rwd / install_conf); - // Use the global override for modules that require bootstrap. - // - string module_import ( - module - ? ((bootstrap ? "!config.import." : "config.import.") + - tm.name.variable () + "=" + (rwd / module_dir).string ()) - : ""); + step_id b (step_id::bpkg_configure_build); - // Configure. + if (!redist (b, r, dist_install_root, pkg_dir)) + break; + } + + rm.status |= r.status; + } + +#ifdef _WIN32 + // Give Windows a chance to (presumably) scan any files we may have just + // unpacked. Failed that, if we try to overwrite any such file (e.g., a + // generated header) we may end up with a permission denied error. Note + // also that this is in addition to the 2 seconds retry we have in our + // fdopen() implementation, which is not always enough. // - dir_path build_dir ("build"); // Configuration directory name. - dir_path pkg_config (rwd / (module ? module_dir : build_dir)); + Sleep (5000); +#endif + + auto fail_unreached_breakpoint = [&bkp_step, &fail_operation] + (operation_result& r) { - operation_result& r (configure_result != nullptr - ? *configure_result - : add_result ("configure")); + assert (bkp_step); - change_wd (trace, &r.log, rwd); + fail_operation (r, + "interactive build breakpoint " + + to_string (*bkp_step) + " cannot be reached", + result_status::abort); + }; - // bpkg create <env-modules> <env-config-args> <config-args> - // - // bpkg.create + // Note that if the bpkg.update step is disabled, we also skip all the + // test and install related steps. + // + if (!step_disabled (step_id::bpkg_update)) + { + // Update the main package. // { - // If the package is a build system module, then make sure it is - // importable in this configuration (see above about bootstrap). + operation_result* pr (&add_result ("update")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + change_wd (trace, &r.log, rwd / main_pkg_conf); + + // bpkg update <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> // + step_id b (step_id::bpkg_update); + step_id s (step_id::bpkg_update); + r.status |= run_bpkg ( - step_id::bpkg_create, + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create", - "-d", build_dir.string (), - "--wipe", - step_args (modules, step_id::bpkg_create), - step_args (env_args, step_id::bpkg_create), - cargs, - module && !bootstrap ? module_import.c_str () : nullptr); + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "update", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pkg); if (!r.status) break; + + rm.status |= r.status; } - change_wd (trace, &r.log, build_dir); + b_project_info prj ( + prj_info (pkg_dir, + b_info_flags::ext_mods | b_info_flags::subprojects, + "project")); - // bpkg add <env-config-args> <config-args> <repository-url> + // If the package turned out to be non-installable, then disable all the + // steps which may not be performed for such a package. // - // bpkg.configure.add - // - r.status |= run_bpkg ( - step_id::bpkg_configure_add, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "add", - step_args (env_args, step_id::bpkg_configure_add), - step_args (config_args, step_id::bpkg_configure_add), - repo); - - if (!r.status) - break; + if (find (prj.operations.begin (), + prj.operations.end (), + "install") == prj.operations.end ()) + { + install_root = nullopt; + bindist = nullopt; + sys_install = false; + bindist_upload = false; + } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // Re-distribute if comes from a version control-based repository, + // update, and test external test packages in the bpkg configuration in + // the current working directory. Optionally pass the config.import.* + // variable override and/or set the environment variables for the bpkg + // processes. Return true if all operations for all packages succeeded. // - // bpkg.configure.fetch + // Pass true as the installed argument to use the test separate installed + // phase step ids (bpkg.test-separate-installed.*) and the test separate + // phase step ids (bpkg.test-separate.*) otherwise. In both cases fall + // back to the main phase step ids (bpkg.*) when no environment/ + // configuration arguments are specified for them. // - r.status |= run_bpkg ( - step_id::bpkg_configure_fetch, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "fetch", - step_args (env_args, step_id::bpkg_configure_fetch), - step_args (config_args, step_id::bpkg_configure_fetch), - trust_ops); + auto test = [&trace, &wre, + &bkp_step, &bkp_status, &aux_env, &last_cmd, + &step_args, &env_args, &tgt_args, &pkg_args, + &bootstrap_import, + &redist] + (operation_result& r, + const small_vector<test_dependency, 1>& tests, + const dir_path& dist_root, + bool installed, + bool update_only, + const small_vector<string, 1>& envvars = {}) + { + const optional<string>& import (!installed + ? bootstrap_import + : nullopt); - if (!r.status) - break; + for (const test_dependency& td: tests) + { + const string& pkg (td.name.string ()); + + // Re-distribute. + // + if (exists (dist_root)) + { + // Note that re-distributing the test package is a bit tricky + // since we don't know its version and so cannot deduce its + // source directory name easily. We could potentially run the + // bpkg-status command after the package is configured and parse + // the output to obtain the version. Let's, however, keep it + // simple and find the source directory using the package + // directory name pattern. + // + try + { + dir_path pkg_dir; + + // Note: doesn't follow symlinks. + // + path_search (dir_path (pkg + "-*/"), + [&pkg_dir] (path&& pe, const string&, bool interm) + { + if (!interm) + pkg_dir = path_cast<dir_path> (move (pe)); + + return interm; + }, + dist_root, + path_match_flags::none); + + if (!pkg_dir.empty ()) + { + step_id b ( + installed + ? step_id::bpkg_test_separate_installed_configure_build + : step_id::bpkg_configure_build); + + if (!redist (b, r, dist_root, pkg_dir, import, envvars)) + return false; + } + } + catch (const system_error& e) + { + fail << "unable to scan directory " << dist_root << ": " << e; + } + } + + // Update. + // + // bpkg update <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> + // + { + step_id b (installed + ? step_id::bpkg_test_separate_installed_update + : step_id::bpkg_test_separate_update); + + step_id s (b); + + step_id f (step_id::bpkg_update); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "update", + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + import, + pkg); + + if (!r.status) + return false; + } - // bpkg build --configure-only <env-config-args> <config-args> - // <package-name>/<package-version> + // Test. + // + // Note that we assume that the package supports the test operation + // since this is its main purpose. + // + // bpkg test <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> + // + if (!update_only) + { + step_id b (installed + ? step_id::bpkg_test_separate_installed_test + : step_id::bpkg_test_separate_test); + + step_id s (b); + + step_id f (step_id::bpkg_test); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "test", + "--package-cwd", // See above for details. + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + import, + pkg); + + if (!r.status) + return false; + } + } + + return true; + }; + + // Test the main package. // - // bpkg.configure.build + // Run the internal tests if the test operation is supported by the + // project but only for the target package or if the configuration is + // self-hosted. // - if (!module) // Note: the module is already built in the pre-step. + bool has_internal_tests ((target_pkg || selfhost) && + find (prj.operations.begin (), + prj.operations.end (), + "test") != prj.operations.end ()); + + if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { - r.status |= run_bpkg ( - step_id::bpkg_configure_build, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "build", - "--configure-only", - "--checkout-root", dist_root, - "--yes", - step_args (env_args, step_id::bpkg_configure_build), - step_args (config_args, step_id::bpkg_configure_build), - "--", - pkg_rev); + operation_result* pr (&add_result ("test")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE - if (!r.status) - break; + // Run internal tests. + // + if (has_internal_tests && !step_disabled (step_id::bpkg_test)) + { + // Use --package-cwd to help ported to build2 third-party packages a + // bit (see bpkg-pkg-test(1) for details). + // + // Note that internal tests that load the module itself don't make + // much sense, thus we don't pass the config.import.* variable on + // the command line for modules that require bootstrap. + // + // bpkg test <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> + // + step_id b (step_id::bpkg_test); + step_id s (step_id::bpkg_test); - bool dist (exists (dist_src)); - const dir_path& src_dir (dist ? dist_src : pkg_dir); + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "test", + "--package-cwd", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pkg); - pm = parse_manifest<package_manifest> (src_dir / mf, "package"); + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the bpkg.test step but the package + // has no internal tests or this step is disabled. + // + else if (bkp_step && *bkp_step == step_id::bpkg_test) + { + fail_unreached_breakpoint (r); + break; + } - if (dist) + // External tests. + // + // Note that if the bpkg.test-separate.update step is disabled, we + // also skip bpkg.test-separate.test. + // + if ((has_runtime_tests || has_buildtime_tests) && + !step_disabled (step_id::bpkg_test_separate_update)) { - if (!redist (step_id::bpkg_configure_build, r, dist_root, pkg_dir)) + bool update_only (step_disabled (step_id::bpkg_test_separate_test)); + + // Fail if the breakpoint refers to the bpkg.test-separate.test step + // but this step is disabled. + // + if (update_only && + bkp_step && + *bkp_step == step_id::bpkg_test_separate_test) + { + fail_unreached_breakpoint (r); break; + } - rm.status |= r.status; + // External runtime tests. + // + // Note that we assume that these packages belong to the dependent + // package's repository or its complement repositories, recursively. + // Thus, we test them in the configuration used to build the + // dependent package. + // + if (has_runtime_tests) + { + if (!test (r, + runtime_tests, + dist_root, + false /* installed */, + update_only)) + break; + } + + // External build-time tests. + // + if (has_buildtime_tests) + { + change_wd (trace, &r.log, rwd / target_conf); + + if (!test (r, + buildtime_tests, + dist_root, + false /* installed */, + update_only)) + break; + } + } + // + // Fail if the breakpoint refers to some of the bpkg.test-separate.* + // steps but the package either has no external tests or the + // bpkg.test-separate.update step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bpkg_test_separate_update && + *bkp_step <= step_id::bpkg_test_separate_test) + { + fail_unreached_breakpoint (r); + break; } + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to some of the test steps but the + // package has no tests. + // + else if (bkp_step && + *bkp_step >= step_id::bpkg_test && + *bkp_step <= step_id::bpkg_test_separate_test) + { + fail_unreached_breakpoint (add_result ("test")); + break; } - rm.status |= r.status; - } + // Install from source. + // + if (install_root) + { + operation_result* pr (&add_result ("install")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE - // Update. - // - if (!module) // Note: the module is already built in the pre-step. - { - operation_result& r (add_result ("update")); + change_wd (trace, &r.log, effective_install_conf); + + // Note that for a host or module package we don't need the target + // configuration anymore, if present. So let's free up the space a + // little bit. + // + if (!target_pkg && create_target) + rm_r (trace, &r.log, rwd / target_conf); + + // Install. + // + { + // bpkg install <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> + // + step_id b (step_id::bpkg_install); + step_id s (step_id::bpkg_install); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "install", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pkg); + + if (!r.status) + break; + } + + // Run ldconfig. + // + if (step_enabled (step_id::bbot_install_ldconfig)) + { + // sudo ldconfig <env-config-args> <tgt-config-args> <pkg-config-args> + // + step_id b (step_id::bbot_install_ldconfig); + step_id s (step_id::bbot_install_ldconfig); + step_id ss (step_id::bbot_install_ldconfig); + + r.status |= run_ldconfig ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss)); - // bpkg update <env-config-args> <config-args> <package-name> + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the bbot.install.ldconfig step but + // this step is disabled. + // + else if (bkp_step && *bkp_step == step_id::bbot_install_ldconfig) + { + fail_unreached_breakpoint (r); + break; + } + + rm.status |= r.status; + } // - // bpkg.update + // Fail if the breakpoint refers to the bpkg.install related steps but + // the package is not supposed to be installed from source. // - r.status |= run_bpkg ( - step_id::bpkg_update, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "update", - step_args (env_args, step_id::bpkg_update), - step_args (config_args, step_id::bpkg_update), - pkg); - - if (!r.status) + else if (bkp_step && + *bkp_step >= step_id::bpkg_install && + *bkp_step <= step_id::bbot_install_ldconfig) + { + fail_unreached_breakpoint (add_result ("install")); break; + } - rm.status |= r.status; - } + // Generate the binary distribution package. + // + // Note that if bbot.bindist.upload step is enabled, it makes sense to + // only copy the generated binary distribution files to the + // upload/bindist/<distribution>/ directory after the binary + // distribution packages are testes, i.e. after the potential + // bbot.sys-uninstall.* steps. + // + // The following bindist_* structures contain a subset of members of the + // corresponding structures described in the STRUCTURED RESULT section + // of the bpkg-pkg-bindist(1) man page. Note: needed later for + // uninstall, upload. + // + struct bindist_os_release + { + string name_id; + optional<string> version_id; + }; - // Run the package internal tests if the test operation is supported by - // the project, except for the build system module which is taken care of - // in the pre-step. - // - bool internal_tests; + struct bindist_file + { + string type; + bbot::path path; // Absolute and normalized. + optional<string> system_name; + }; - if (module) - { - internal_tests = false; - } - else - { - prj = prj_info (pkg_dir, true /* ext_mods */, "project"); + struct bindist_package + { + string name; + string version; + optional<string> system_version; + vector<bindist_file> files; + }; - internal_tests = find (prj.operations.begin (), - prj.operations.end (), - "test") != prj.operations.end (); - } + struct bindist_result_type + { + string distribution; + string architecture; + bindist_os_release os_release; + bindist_package package; + vector<bindist_package> dependencies; + }; - // Run the package external tests, if specified. But first filter them - // against the test-exclude task manifest values using the package names. - // - // Note that a proper implementation should also make sure that the - // excluded test package version matches the version that will supposedly - // be configured by bpkg and probably abort the build if that's not the - // case. Such a mismatch can happen due to some valid reasons (the - // repository was updated since the task was issued, etc) and should - // probably be followed with automatic rebuild (the flake monitor idea). - // Anyway, this all requires additional thinking, so let's keep it simple - // for now. - // - // Filter the external test dependencies in place. - // - pm.tests.erase ( - remove_if (pm.tests.begin (), pm.tests.end (), - [&tm] (const test_dependency& td) - { - return find_if (tm.test_exclusions.begin (), - tm.test_exclusions.end (), - [&td] (const package& te) - { - return te.name == td.name; - }) != tm.test_exclusions.end (); - }), - pm.tests.end ()); - - bool external_tests (!pm.tests.empty ()); - - // Configure, re-distribute if comes from a version control-based - // repository, update, and test packages in the bpkg configuration in the - // current working directory. Optionally pass the config.import.* variable - // override and/or set the environment variables for bpkg processes. - // Return true if all operations for all packages succeed. - // - // Pass true as the installed argument to use the test separate installed - // phase step ids (bpkg.test-separate-installed.*) and the test separate - // phase step ids (bpkg.test-separate.*) otherwise. In both cases fall - // back to the main phase step ids (bpkg.*) when no environment/ - // configuration arguments are specified for them. - // - // Pass true as the sys_dep argument to configure the dependent package as - // a system dependency, which is normally required for testing modules and - // installed dependents. Note that bpkg configures the dependent package - // as a special dependency for the test package. - // - auto test = [&trace, &wre, - &bkp_step, &bkp_status, &last_cmd, - &step_args, &config_args, &env_args, - &pm, - &redist] - (operation_result& r, - const dir_path& dist_root, - bool installed, - bool sys_dep, - const char* import = nullptr, - const small_vector<string, 1>& envvars = {}) - { - for (const test_dependency& td: pm.tests) + bindist_result_type bindist_result; + + const dir_path& bindist_conf ( + create_install ? install_conf : main_pkg_conf); + + // Make it absolute for the sake of diagnostics. + // + path bindist_result_file (rwd / bindist_conf / "bindist-result.json"); + + if (bindist) { - const string& pkg (td.name.string ()); + operation_result* pr (&add_result ("bindist")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + // Fail if the breakpoint refers to a bpkg.bindist.* step but this + // step differs from the enabled one. + // + if (bkp_step && + (*bkp_step == step_id::bpkg_bindist_debian || + *bkp_step == step_id::bpkg_bindist_fedora || + *bkp_step == step_id::bpkg_bindist_archive) && + *bkp_step != *bindist) + { + fail_unreached_breakpoint (r); + break; + } + + change_wd (trace, &r.log, rwd); - // Configure. + // Note that for a host or module package we don't need the target + // configuration anymore, if present. So let's free up the space a + // little bit. // - // bpkg build --configure-only <env-config-args> <config-args> - // '<package-name>[ <version-constraint>]' + if (!target_pkg && create_target) + rm_r (trace, &r.log, rwd / target_conf); + + string distribution; + dir_path output_root; + + switch (*bindist) + { + case step_id::bpkg_bindist_debian: + { + distribution = "debian"; + output_root = dir_path ("bindist"); + break; + } + case step_id::bpkg_bindist_fedora: + { + distribution = "fedora"; + break; + } + case step_id::bpkg_bindist_archive: + { + distribution = "archive"; + output_root = dir_path ("bindist"); + break; + } + default: assert (false); + } + + // bpkg bindist --distribution <distribution> + // <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> // - // bpkg.test-separate[-installed].configure.build (bpkg.configure.build) + // Note that if we are installing the result, we need to generate + // packages for all the dependencies unless they are included in the + // package (with --recursive). The way we are going to arrange for + // this is by specifying --recursive=separate first and letting any + // user --recursive option override that. // - step_id s (installed - ? step_id::bpkg_test_separate_installed_configure_build - : step_id::bpkg_test_separate_configure_build); + step_id b (*bindist); + step_id s (*bindist); r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + b, + trace, r.log, bindist_result_file, wre, + bkp_step, bkp_status, aux_env, last_cmd, "-v", - "build", - "--configure-only", - "--checkout-root", dist_root, - "--yes", - step_args (env_args, s, step_id::bpkg_configure_build), - step_args (config_args, s, step_id::bpkg_configure_build), - import, - "--", - td.string (), - sys_dep ? ("?sys:" + pm.name.string ()).c_str () : nullptr); + "bindist", + "--distribution", distribution, + sys_install ? cstrings ({"--recursive", "separate"}) : cstrings (), + "--structured-result", "json", + (!output_root.empty () + ? cstrings ({"--output-root", output_root.string ().c_str ()}) + : cstrings ()), + "-d", bindist_conf, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pkg); if (!r.status) - return false; + break; - // Note that re-distributing the test package is a bit tricky since we - // don't know its version and so cannot deduce its source directory - // name easily. We could potentially run the bpkg-status command after - // the package is configured and parse the output to obtain the - // version. Let's, however, keep it simple and find the source - // directory using the package directory name pattern. + // Parse the structured result JSON. // - if (exists (dist_root)) try { - dir_path pkg_dir; + ifdstream is (bindist_result_file); + json::parser p (is, bindist_result_file.string ()); + + using event = json::event; + + auto bad_json = [&p] (string d) + { + throw json::invalid_json_input (p.input_name, + p.line (), + p.column (), + p.position (), + move (d)); + }; + + // Parse bindist_os_release object. + // + auto parse_os_release = [&p] () + { + // enter: after begin_object + // leave: after end_object + + bindist_os_release r; + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "name_id") + { + r.name_id = p.next_expect_string (); + } + else if (n == "version_id") + { + r.version_id = p.next_expect_string (); + } + else + p.next_expect_value_skip (); + } + + return r; + }; + + // Parse a bindist_file object. + // + auto parse_file = [&p, &bad_json] () + { + // enter: after begin_object + // leave: after end_object + + bindist_file r; + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); - path_search (dir_path (pkg + "-*/"), - [&pkg_dir] (path&& pe, const string&, bool interm) - { - if (!interm) - pkg_dir = path_cast<dir_path> (move (pe)); + if (n == "type") + { + r.type = p.next_expect_string (); + } + else if (n == "path") + { + try + { + r.path = + path (p.next_expect_string ()).complete ().normalize (); + } + catch (const invalid_path& e) + { + bad_json ("invalid package file path '" + e.path + "'"); + } + } + else if (n == "system_name") + { + r.system_name = p.next_expect_string (); + } + else + p.next_expect_value_skip (); + } - return interm; - }, - dist_root); + return r; + }; - if (!pkg_dir.empty () && - !redist (s, r, dist_root, pkg_dir, import, envvars)) - return false; + // Parse a bindist_package object. + // + auto parse_package = [&p, &parse_file] () + { + // enter: after begin_object + // leave: after end_object + + bindist_package r; + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "name") + { + r.name = p.next_expect_string (); + } + else if (n == "version") + { + r.version = p.next_expect_string (); + } + else if (n == "system_version") + { + r.system_version = p.next_expect_string (); + } + else if (n == "files") + { + p.next_expect (event::begin_array); + + while (p.next_expect (event::begin_object, event::end_array)) + r.files.push_back (parse_file ()); + } + else + p.next_expect_value_skip (); + } + + return r; + }; + + // Parse the bindist_result. + // + // Note that if the bbot.bindist.upload step is enabled, then we + // require bindist_result.os_release.version_id to be present. This + // way the uploaded binary package can be published for a specific + // version of the distribution. + // + p.next_expect (event::begin_object); + + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "distribution") + { + bindist_result.distribution = p.next_expect_string (); + + if (bindist_result.distribution != distribution) + bad_json ("expected distribution '" + distribution + + "' instead of '" + bindist_result.distribution + "'"); + } + else if (n == "architecture") + { + bindist_result.architecture = p.next_expect_string (); + } + else if (n == "os_release") + { + p.next_expect (event::begin_object); + bindist_result.os_release = parse_os_release (); + + if (!bindist_result.os_release.version_id && bindist_upload) + bad_json ("version_id must be present if bbot.bindist.upload " + "step is enabled"); + } + else if (n == "package") + { + p.next_expect (event::begin_object); + bindist_result.package = parse_package (); + } + else if (n == "dependencies") + { + p.next_expect (event::begin_array); + + while (p.next_expect (event::begin_object, event::end_array)) + bindist_result.dependencies.push_back (parse_package ()); + } + else + p.next_expect_value_skip (); + } + } + catch (const json::invalid_json_input& e) + { + fail_operation ( + r, + string ("invalid bpkg-pkg-bindist result json input: ") + + e.what (), + result_status::abort, + e.name, + e.line, + e.column); + + // Fall through. } - catch (const system_error& e) + catch (const io_error& e) { - fail << "unable to scan directory " << dist_root << ": " << e; + fail << "unable to read " << bindist_result_file << ": " << e; } - // Update. + if (!r.status) + break; + + log_line ("generated " + distribution + " package for " + pkg + '/' + + ver.string () + ':', + r.log); + + for (const bindist_file& f: bindist_result.package.files) + log_line (" " + f.path.string (), r.log); + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to a bpkg.bindist.* step but this step + // is disabled. + // + else if (bkp_step && + (*bkp_step == step_id::bpkg_bindist_debian || + *bkp_step == step_id::bpkg_bindist_fedora || + *bkp_step == step_id::bpkg_bindist_archive)) + { + fail_unreached_breakpoint (add_result ("bindist")); + break; + } + + // Install from the binary distribution package generated on a + // bpkg.bindist.* step. + // + if (sys_install) + { + operation_result* pr (&add_result ("sys-install")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + // Fail if the breakpoint refers to the bbot.sys-install step since + // it has no specific command associated. + // + if (bkp_step && *bkp_step == step_id::bbot_sys_install) + { + fail_unreached_breakpoint (r); + break; + } + + // Noop, just for the log record. + // + change_wd (trace, &r.log, rwd); + + // Collect the binary package files. // - // bpkg update <env-config-args> <config-args> <package-name> + // Specifically, for now we consider files with the system name + // specified as package files. // - // bpkg.test-separate[-installed].update (bpkg.update) + cstrings pfs; + + auto add_package_files = [&pfs] (const vector<bindist_file>& bfs) + { + for (const bindist_file& f: bfs) + { + if (f.system_name) + pfs.push_back (f.path.string ().c_str ()); + } + }; + + add_package_files (bindist_result.package.files); + + for (const bindist_package& d: bindist_result.dependencies) + add_package_files (d.files); + + // Install for the `debian` distribution. // - s = installed - ? step_id::bpkg_test_separate_installed_update - : step_id::bpkg_test_separate_update; + if (*bindist == step_id::bpkg_bindist_debian) + { + // Update package index. + // + { + // sudo apt-get update <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + step_id b (step_id::bbot_sys_install_apt_get_update); + step_id s (step_id::bbot_sys_install_apt_get_update); + step_id ss (step_id::bbot_sys_install_apt_get_update); + + r.status |= run_apt_get ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "update", + "--assume-yes", + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss)); + + if (!r.status) + break; + } - r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "update", - step_args (env_args, s, step_id::bpkg_update), - step_args (config_args, s, step_id::bpkg_update), - import, - pkg); + // Install. + // + { + // sudo apt-get install <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <distribution-package-file>... + // + // Note that apt-get install requires a directory separator for an + // argument to be treated as a file rather than name. The paths we + // pass are absolute. + // + step_id b (step_id::bbot_sys_install_apt_get_install); + step_id s (step_id::bbot_sys_install_apt_get_install); + step_id ss (step_id::bbot_sys_install_apt_get_install); + + r.status |= run_apt_get ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "install", + "--assume-yes", + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss), + pfs); + + if (!r.status) + break; + } + } + // + // Fail if the breakpoint refers to a bbot.sys-install.apt_get.* step + // but the distribution is other than `debian`. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install_apt_get_update && + *bkp_step <= step_id::bbot_sys_install_apt_get_install) + { + fail_unreached_breakpoint (r); + break; + } + // + // Install for the `fedora` distribution. + // + else if (*bindist == step_id::bpkg_bindist_fedora) + { + // sudo dnf install <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <distribution-package-file>... + // + step_id b (step_id::bbot_sys_install_dnf_install); + step_id s (step_id::bbot_sys_install_dnf_install); + step_id ss (step_id::bbot_sys_install_dnf_install); - if (!r.status) - return false; + r.status |= run_dnf ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "install", + "--refresh", + "--assumeyes", + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss), + pfs); - // Test. + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to a bbot.sys-install.dnf.* step but + // the distribution is other than `fedora`. // - // Note that we assume that the package supports the test operation - // since this is its main purpose. + else if (bkp_step && *bkp_step == step_id::bbot_sys_install_dnf_install) + { + fail_unreached_breakpoint (r); + break; + } + // + // Install for the `archive` distribution. // - // bpkg test <env-config-args> <config-args> <package-name> + // Since there is no easy way to extract from multiple archives with a + // single command, we run tar in a loop. // - // bpkg.test-separate[-installed].test (bpkg.test) + // Note that it is assumed that the --directory and --strip-components + // options are passed via <*-config-args>. The extracted executables + // can be arranged to be found by setting config.install.root for + // bpkg.target.create, etc (the same way as for installing from + // source). // - s = installed - ? step_id::bpkg_test_separate_installed_test - : step_id::bpkg_test_separate_test; + else if (*bindist == step_id::bpkg_bindist_archive) + { + // If the bbot.sys-install:config.install.root variable is + // specified, then make sure the directory it refers to exists by + // the time we run `tar -xf`, so that this command doesn't fail + // trying to extract into a non-existent directory. Note that we do + // that regardless whether the package is a build system module or + // not. + // + optional<dir_path> ir ( + config_install_root (step_id::bbot_sys_install)); - r.status |= run_bpkg ( - s, - envvars, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "test", - "--package-cwd", // See above for details. - step_args (env_args, s, step_id::bpkg_test), - step_args (config_args, s, step_id::bpkg_test), - import, - pkg); + if (ir) + mk_p (trace, &r.log, *ir, true /* sudo */); - if (!r.status) - return false; - } + if (!module_pkg) + { + if (!ir) + ir = config_install_root (step_id::bpkg_target_create, + step_id::b_create, + step_id::bpkg_create); - return true; - }; + if (ir) + install_bin = *ir / dir_path ("bin"); + } - if (internal_tests || external_tests) - { - operation_result& r (test_result != nullptr - ? *test_result - : add_result ("test")); + for (const char* f: pfs) + { + // [sudo] tar -xf <distribution-package-file> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + step_id b (step_id::bbot_sys_install_tar_extract); + step_id s (step_id::bbot_sys_install_tar_extract); + step_id ss (step_id::bbot_sys_install_tar_extract); + + r.status |= run_tar ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + true /* sudo */, + "-xf", + f, + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss)); + + if (!r.status) + break; + } - // Noop, just for the log record to reduce the potential confusion for - // the combined log reader due to updating the build system module in a - // separate configuration (see above for details). - // - if (module) - change_wd (trace, &r.log, current_directory ()); + if (!r.status) + break; - // Run internal tests. - // - if (internal_tests) // Note: false for modules (see above). - { - // bpkg test <env-config-args> <config-args> <package-name> + // Run ldconfig. + // + if (step_enabled (step_id::bbot_sys_install_ldconfig)) + { + // sudo ldconfig <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + step_id b (step_id::bbot_sys_install_ldconfig); + step_id s (step_id::bbot_sys_install_ldconfig); + step_id ss (step_id::bbot_sys_install_ldconfig); + + r.status |= run_ldconfig ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss)); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the bbot.sys-install.ldconfig + // step but this step is disabled. + // + else if (bkp_step && *bkp_step == step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (r); + break; + } + } // - // bpkg.test + // Fail if the breakpoint refers to a + // bbot.sys-install.{tar.extract,ldconfig} step but the distribution + // is other than `archive`. // - r.status |= run_bpkg ( - step_id::bpkg_test, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "test", - "--package-cwd", // See above for details. - step_args (env_args, step_id::bpkg_test), - step_args (config_args, step_id::bpkg_test), - pkg); - - if (!r.status) + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install_tar_extract && + *bkp_step <= step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (r); break; + } + else + assert (false); + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to a bbot.sys-install.* step but this + // step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install && + *bkp_step <= step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (add_result ("sys-install")); + break; } - // Run external tests. + // Now, if the package is installed, either from source or from the + // binary distribution package, the overall plan is as follows: + // + // 1. If the package has subprojects that support the test operation, + // then configure, build, and test them out of the source tree + // against the installed package using the build system directly. // - // Note that we assume that these packages belong to the dependent - // package's repository or its complement repositories, recursively. - // Thus, we test them in the configuration used to build the dependent - // package (except for the build system module). + // 2. If any of the test packages are specified, then configure, build, + // and test them in a separate bpkg configuration(s) against the + // installed package. // - if (external_tests) + if (install_root || sys_install) { - // The test separate phase. + // Run the internal tests if the project contains "testable" + // subprojects, but not for a module. // - if (!test (r, - dist_root, - false /* installed */, - module, - bootstrap ? module_import.c_str () : nullptr)) - break; + has_internal_tests = false; + + dir_paths subprj_dirs; // "Testable" package subprojects. - // Back to the main phase. + // Collect the "testable" subprojects. // - } + if (!module_pkg) + { + assert (!rm.results.empty ()); - rm.status |= r.status; - } + // Result of the install or sys-install operation. + // + operation_result& r (rm.results.back ()); - // Install the package, optionally test the installation and uninstall - // afterwards. - // - // These operations are triggered by presence of config.install.root - // configuration variable having a non-empty value for - // bpkg.configure.create step. - // - if (install_root.empty ()) - break; + change_wd (trace, &r.log, effective_install_conf); - // Now the overall plan is as follows: - // - // 1. Install the package. - // - // 2. If the package has subprojects that support the test operation, then - // configure, build, and test them out of the source tree against the - // installed package. - // - // 3. If any of the test packages are specified, then configure, build, - // and test them in a separate bpkg configuration against the installed - // package. - // - // 4. Uninstall the package. - // - // Install. - // - { - operation_result& r (add_result ("install")); + for (const b_project_info::subproject& sp: prj.subprojects) + { + // Retrieve the subproject information similar to how we've done it + // for the package. + // + b_project_info si (prj_info (pkg_dir / sp.path, + b_info_flags::ext_mods, + "subproject")); - change_wd (trace, &r.log, pkg_config); + const strings& ops (si.operations); + if (find (ops.begin (), ops.end (), "test") != ops.end ()) + subprj_dirs.push_back (sp.path); + } - // bpkg install <env-config-args> <config-args> <package-name> - // - // bpkg.install - // - r.status |= run_bpkg ( - step_id::bpkg_install, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "install", - step_args (env_args, step_id::bpkg_install), - step_args (config_args, step_id::bpkg_install), - pkg); + has_internal_tests = !subprj_dirs.empty (); + } - if (!r.status) - break; + if (has_internal_tests || has_runtime_tests || has_buildtime_tests) + { + operation_result* pr (&add_result ("test-installed")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE - rm.status |= r.status; - } + change_wd (trace, &r.log, rwd); - // The test installed phase. - // + // Make sure that the installed package executables are properly + // imported when configuring and running tests, unless we are testing + // the build system module (that supposedly doesn't install any + // executables). + // + small_vector<string, 1> envvars; - // Make sure that the installed package executables are properly imported - // when configuring and running tests, unless we are testing the build - // system module (that supposedly doesn't install any executables). - // - small_vector<string, 1> envvars; + if (install_bin) + { + // Note that we add the $config.install.root/bin directory at the + // beginning of the PATH environment variable value, so the + // installed executables are found first. + // + const string& ib (install_bin->string ()); - dir_paths subprj_dirs; // "Testable" package subprojects. + log_comment (trace, r.log, + "add " + ib + " to PATH environment variable"); - // We expect the build system modules to not have any testable subprojects - // but to have external tests package instead. - // - if (module) - internal_tests = false; - else - { - // Note that we add the $config.install.root/bin directory at the - // beginning of the PATH environment variable value, so the installed - // executables are found first. - // - string paths ("PATH=" + (install_root / "bin").string ()); + string paths ("PATH=" + ib); - if (optional<string> s = getenv ("PATH")) - { - paths += path::traits_type::path_separator; - paths += *s; - } + if (optional<string> s = getenv ("PATH")) + { + paths += path::traits_type::path_separator; + paths += *s; + } - envvars.push_back (move (paths)); + envvars.push_back (move (paths)); + } - // Collect the "testable" subprojects. - // - for (const b_project_info::subproject& sp: prj.subprojects) - { - // Retrieve the subproject information similar to how we've done it - // for the package. - // - b_project_info si (prj_info (pkg_dir / sp.path, - true /* ext_mods */, - "subproject")); + // Run internal tests. + // + if (has_internal_tests) + { + // Create the configuration. + // + // b create(<dir>, <env-modules>) <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + // Amalgamation directory that will contain configuration + // subdirectory for package tests out of source tree build. + // + dir_path out_dir ("build-installed"); - const strings& ops (si.operations); - if (find (ops.begin (), ops.end (), "test") != ops.end ()) - subprj_dirs.push_back (sp.path); - } + { + step_id b (step_id::b_test_installed_create); + step_id s (step_id::b_test_installed_create); + step_id f (step_id::b_create); - // If there are any "testable" subprojects, then configure them - // (sequentially) and test/build in parallel afterwards. - // - internal_tests = !subprj_dirs.empty (); - } + string mods; // build2 create meta-operation parameters. - if (internal_tests || external_tests) - { - operation_result& r (add_result ("test-installed")); + for (const char* m: step_args (modules, s, f)) + { + mods += mods.empty () ? ", " : " "; + mods += m; + } - change_wd (trace, &r.log, rwd); + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create('" + out_dir.representation () + '\'' + mods + ')', + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f)); + + if (!r.status) + break; + } - // Run internal tests. - // - if (internal_tests) - { - string mods; // build2 create meta-operation parameters. + // Configure testable subprojects sequentially and test/build in + // parallel afterwards. + // + // It feels right to configure internal tests also passing the + // main package configuration variables, since they may need to + // align with the main package setup (enable some testscripts, + // etc). + // + strings test_specs; + for (const dir_path& d: subprj_dirs) + { + // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <pkg-vars> + // + step_id b (step_id::b_test_installed_configure); + step_id s (step_id::b_test_installed_configure); + step_id f (step_id::b_configure); + + dir_path subprj_src_dir (exists (dist_src) + ? dist_src / d + : main_pkg_conf / pkg_dir / d); + + dir_path subprj_out_dir (out_dir / d); + + r.status |= run_b ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "configure('" + + subprj_src_dir.representation () + "'@'" + + subprj_out_dir.representation () + "')", + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + pkg_config_vars); + + if (!r.status) + break; + + test_specs.push_back ( + "test('" + subprj_out_dir.representation () + "')"); + } + + if (!r.status) + break; + + // Build/test subprojects. + // + // b test(<subprj-out-dir>)... <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + if (!step_disabled (step_id::b_test_installed_test)) + { + step_id b (step_id::b_test_installed_test); + step_id s (step_id::b_test_installed_test); + + r.status |= run_b ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + test_specs, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s)); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the b.test-installed.test step + // but this step is disabled. + // + else if (bkp_step && *bkp_step == step_id::b_test_installed_test) + { + fail_unreached_breakpoint (r); + break; + } + } + // + // Fail if the breakpoint refers to some of the b.test-installed.* + // steps but the package doesn't have any internal tests. + // + else if (bkp_step && + *bkp_step >= step_id::b_test_installed_create && + *bkp_step <= step_id::b_test_installed_test) + { + fail_unreached_breakpoint (r); + break; + } + + // Run runtime and build-time tests. + // + // Note that we only build runtime tests for target packages and for + // host packages in self-hosted configurations. + // + if (has_runtime_tests || has_buildtime_tests) + { + // Create the required build configurations. + // + dir_path target_conf ("build-installed-bpkg"); + dir_path host_conf ("build-installed-bpkg-host"); + dir_path module_conf ("build-installed-bpkg-module"); + + // Create the target configuration if this is a target package + // having external runtime tests or a host/module package having + // external build-time tests. + // + bool create_target (target_pkg || has_buildtime_tests); + + // Note that even if there are no runtime tests for a host/module + // package, we still need to create the host/build2 configuration + // to configure the system package in. + // + bool create_host (host_pkg || module_pkg); + + bool create_module (module_pkg || + (host_pkg && has_buildtime_tests)); + + // Note: a module package cannot have runtime tests and so the + // module configuration is only created to serve build-time tests. + // Thus, the host or target configuration is always created as + // well and the module configuration is never a root + // configuration. + // + assert (create_target || create_host); + + // Root configuration through which we will be configuring the + // cluster. + // + const dir_path& root_conf (create_target ? target_conf : host_conf); + + // Runtime tests configuration. Should only be used if there are + // any. + // + const dir_path& runtime_tests_conf (target_pkg + ? target_conf + : host_conf); + + // Create the target configuration. + // + // bpkg create <env-modules> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + if (create_target) + { + step_id b (step_id::bpkg_test_separate_installed_create); + + // Note that here and below the _for_* step ids are determined + // by the main package type (and, yes, that means we will use + // the same step ids for target and host configuration -- that, + // however, should be ok since host configuration will only be + // created in the self-hosted case). + // + step_id s ( + target_pkg + ? step_id::bpkg_test_separate_installed_create_for_target + : host_pkg + ? step_id::bpkg_test_separate_installed_create_for_host + : step_id::bpkg_test_separate_installed_create_for_module); + + // Note: no fallback for modules. + // + optional<step_id> f (!module_pkg + ? step_id::bpkg_test_separate_installed_create + : optional<step_id> ()); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create", + "-d", target_conf, + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f)); + + if (!r.status) + break; + } + + // Create the host configuration. + // + if (create_host) + { + // bpkg create --type host <env-modules> <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + step_id b (step_id::bpkg_test_separate_installed_create); + + step_id s (host_pkg + ? step_id::bpkg_test_separate_installed_create_for_host + : step_id::bpkg_test_separate_installed_create_for_module); + + // Note: no fallback for modules. + // + optional<step_id> f (!module_pkg + ? step_id::bpkg_test_separate_installed_create + : optional<step_id> ()); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create", + "-d", host_conf, + "--type", "host", + "--name", "host", + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f)); + + if (!r.status) + break; + } + + // Create the module configuration. + // + // Note that we never build any tests in it but only configure the + // system package. Note, however, that the host/module package + // build-time tests can potentially build some other modules here. + // + if (create_module) + { + // b create(<dir>) config.config.load=~build2 + // + step_id b (step_id::bpkg_test_separate_installed_create); + + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-V", + "create(" + module_conf.representation () + ",cc)", + "config.config.load=~build2", + "config.config.persist+='config.*'@unused=drop"); + + if (!r.status) + break; + + // bpkg create --existing --type build2 + // + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "create", + "--existing", + "-d", module_conf, + "--type", "build2", + "--name", "module"); + + if (!r.status) + break; + } + + // Link the configurations. + // + // bpkg link -d <dir> <dir> + // + { + step_id b (step_id::bpkg_test_separate_installed_link); + + if (create_target) + { + if (create_host) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", target_conf, + host_conf); + + if (!r.status) + break; + } + + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", target_conf, + module_conf); + + if (!r.status) + break; + } + } + + if (create_host) + { + if (create_module) + { + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "link", + "-d", host_conf, + module_conf); + + if (!r.status) + break; + } + } + } + + // Add and fetch the repositories. + // + if (has_runtime_tests) + { + // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args> + // <repository-url> + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_add); + step_id s (step_id::bpkg_test_separate_installed_configure_add); + step_id f (step_id::bpkg_configure_add); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "add", + "-d", runtime_tests_conf, + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + repo); + + if (!r.status) + break; + } + + // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args> + // <trust-options> + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_fetch); + step_id s (step_id::bpkg_test_separate_installed_configure_fetch); + step_id f (step_id::bpkg_configure_fetch); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "fetch", + "-d", runtime_tests_conf, + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + trust_ops); + + if (!r.status) + break; + } + } + + if (has_buildtime_tests) + { + // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args> + // <repository-url> + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_add); + step_id s (step_id::bpkg_test_separate_installed_configure_add); + step_id f (step_id::bpkg_configure_add); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "add", + "-d", target_conf, + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + repo); + + if (!r.status) + break; + } + + // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args> + // <trust-options> + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_fetch); + step_id s (step_id::bpkg_test_separate_installed_configure_fetch); + step_id f (step_id::bpkg_configure_fetch); + + r.status |= run_bpkg ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "fetch", + "-d", target_conf, + step_args (env_args, s, f), + step_args (tgt_args, s, f), + step_args (pkg_args, s, f), + trust_ops); + + if (!r.status) + break; + } + } + + // Configure all the packages using a single bpkg-pkg-build command. + // + // bpkg build --configure-only <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // { <config> <rtt-config-vars> }+ <runtime-test>... + // { <config> }+ { <runtime-test>... } + // { <btt-config-vars> }+ <buildtime-test>... + // ?sys:<pkg> + // <glob-dep>... + // + strings pkgs; + + if (has_runtime_tests) + { + // Note that only host package runtime tests can (but not + // necessarily) be configured in a linked configuration and + // require --config-name to be specified for them. + // + assert (!module_pkg); + + string conf_name (runtime_tests_conf == root_conf + ? "" + : "host"); + + // If there are any runtime tests with configuration variables, + // then add them to the command line as following: + // + // { --config-name <name> <config-var>... }+ <runtime-test>... + // + // Also count the number of runtime tests without configuration + // variables. + // + size_t no_vars (0); + for (const auto& t: runtime_tests) + { + auto i (pkg_config_pkgs.find (t.name.string ())); + + if (i != pkg_config_pkgs.end ()) + { + pkgs.push_back ("{"); + + if (!conf_name.empty ()) + { + pkgs.push_back ("--config-name"); + pkgs.push_back (conf_name); + } + + pkgs.insert (pkgs.end (), + i->second.begin (), i->second.end ()); + + pkgs.push_back ("}+"); + + // Strip the potential reflection variable assignment. + // + pkgs.push_back (t.dependency::string ()); + } + else + ++no_vars; + } + + // If there are any runtime tests without configuration + // variables, then add them to the command line as following: + // + // { --config-name <name> }+ { <runtime-test>... } + // + if (no_vars != 0) + { + bool og (!conf_name.empty ()); + + if (og) + { + pkgs.push_back ("{"); + + pkgs.push_back ("--config-name"); + pkgs.push_back (conf_name); + + pkgs.push_back ("}+"); + } + + if (og && no_vars != 1) + pkgs.push_back ("{"); + + for (const auto& t: runtime_tests) + { + if (pkg_config_pkgs.find (t.name.string ()) == + pkg_config_pkgs.end ()) + { + // Strip the potential reflection variable assignment. + // + pkgs.push_back (t.dependency::string ()); + } + } + + if (og && no_vars != 1) + pkgs.push_back ("}"); + } + } + + if (has_buildtime_tests) + { + for (const auto& t: buildtime_tests) + { + auto i (pkg_config_pkgs.find (t.name.string ())); + + if (i != pkg_config_pkgs.end ()) + { + pkgs.push_back ("{"); + + pkgs.insert (pkgs.end (), + i->second.begin (), i->second.end ()); + + pkgs.push_back ("}+"); + } + + // Strip the build-time mark and potential reflection variable + // assignment. + // + pkgs.push_back (t.dependency::string ()); + } + } + + pkgs.push_back ("?sys:" + pkg_rev); + + // Add the global system dependencies. + // + for (const pair<string, strings>& d: pkg_config_glob_deps) + pkgs.push_back (d.first); + + // Finally, configure all the test packages. + // + { + step_id b (step_id::bpkg_test_separate_installed_configure_build); + + step_id g (step_id::bpkg_global_configure_build); // Global. + + step_id s ( + target_pkg + ? step_id::bpkg_test_separate_installed_configure_build_for_target + : host_pkg + ? step_id::bpkg_test_separate_installed_configure_build_for_host + : step_id::bpkg_test_separate_installed_configure_build_for_module); + + step_id f (step_id::bpkg_test_separate_installed_configure_build); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-v", + "build", + "--configure-only", + "--checkout-root", dist_installed_root, + "--yes", + "-d", root_conf, + step_args (env_args, g), + step_args (env_args, s, f), + step_args (tgt_args, g), + step_args (tgt_args, s, f), + step_args (pkg_args, g), + step_args (pkg_args, s, f), + "--", + pkgs); + + if (!r.status) + break; + } + +#ifdef _WIN32 + Sleep (5000); // See above. +#endif - for (const string& m: step_args (modules, - step_id::b_test_installed_create)) + // Note that if bpkg.test-separate-installed.update step is + // disabled, we also skip bpkg.test-separate-installed.test. + // + if (!step_disabled (step_id::bpkg_test_separate_installed_update)) + { + bool update_only ( + step_disabled (step_id::bpkg_test_separate_installed_test)); + + // Fail if the breakpoint refers to the + // bpkg.test-separate-installed.test step but this step is + // disabled. + // + if (update_only && + bkp_step && + *bkp_step == step_id::bpkg_test_separate_installed_test) + { + fail_unreached_breakpoint (r); + break; + } + + // External runtime tests. + // + if (has_runtime_tests) + { + const dir_path& runtime_tests_conf (target_pkg + ? target_conf + : host_conf); + + change_wd (trace, &r.log, runtime_tests_conf); + + if (!test (r, + runtime_tests, + dist_installed_root, + true /* installed */, + update_only, + envvars)) + break; + } + + // External build-time tests. + // + if (has_buildtime_tests) + { + change_wd (trace, &r.log, rwd / target_conf); + + if (!test (r, + buildtime_tests, + dist_installed_root, + true /* installed */, + update_only, + envvars)) + break; + } + } + // + // Fail if the breakpoint refers to some of the + // bpkg.test-separate-installed.{update,test} steps but the + // bpkg.test-separate-installed.update step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bpkg_test_separate_installed_update && + *bkp_step <= step_id::bpkg_test_separate_installed_test) + { + fail_unreached_breakpoint (r); + break; + } + } + // + // Fail if the breakpoint refers to some of the + // bpkg.test-separate-installed.* steps but the package has no + // external tests. + // + else if (bkp_step && + *bkp_step >= step_id::bpkg_test_separate_installed_create && + *bkp_step <= step_id::bpkg_test_separate_installed_test) + { + fail_unreached_breakpoint (r); + break; + } + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to some of the test installed steps + // but the package has no tests. + // + else if (bkp_step && + *bkp_step >= step_id::b_test_installed_create && + *bkp_step <= step_id::bpkg_test_separate_installed_test) { - mods += mods.empty () ? ", " : " "; - mods += m; + fail_unreached_breakpoint (add_result ("test-installed")); + break; } + } + // + // Fail if the breakpoint refers to some of the test installed steps but + // the package is not supposed to be installed neither from source nor + // from the binary distribution package. + // + else if (bkp_step && + *bkp_step >= step_id::b_test_installed_create && + *bkp_step <= step_id::bpkg_test_separate_installed_test) + { + fail_unreached_breakpoint (add_result ("test-installed")); + break; + } - // b create(<dir>, <env-modules>) <env-config-args> <config-args> - // - // b.test-installed.create + // Uninstall, if installed from the binary distribution package. + // + // Note: noop for the archive distribution. + // + if (sys_install && + (*bindist == step_id::bpkg_bindist_debian || + *bindist == step_id::bpkg_bindist_fedora)) + { + operation_result* pr (&add_result ("sys-uninstall")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + // Noop, just for the log record. // - // Amalgamation directory that will contain configuration subdirectory - // for package tests out of source tree build. + change_wd (trace, &r.log, rwd); + + // Collect the binary package system names. // - dir_path out_dir ("build-installed"); + cstrings pns; - r.status |= run_b ( - step_id::b_test_installed_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create('" + out_dir.representation () + "'" + mods + ")", - step_args (env_args, step_id::b_test_installed_create), - step_args (config_args, step_id::b_test_installed_create)); + auto add_package_names = [&pns] (const vector<bindist_file>& bfs) + { + for (const bindist_file& f: bfs) + { + if (f.system_name) + pns.push_back (f.system_name->c_str ()); + } + }; - if (!r.status) - break; + add_package_names (bindist_result.package.files); - // Configure subprojects and create buildspecs for their testing. + for (const bindist_package& d: bindist_result.dependencies) + add_package_names (d.files); + + // Uninstall for the `debian` distribution. // - strings test_specs; - for (const dir_path& d: subprj_dirs) + if (*bindist == step_id::bpkg_bindist_debian) { - // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args> - // <config-args> - // - // b.test-installed.configure + // sudo apt-get remove <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <distribution-package-name>... // - dir_path subprj_src_dir (exists (dist_src) - ? dist_src / d - : build_dir / pkg_dir / d); + step_id b (step_id::bbot_sys_uninstall_apt_get_remove); + step_id s (step_id::bbot_sys_uninstall_apt_get_remove); + step_id ss (step_id::bbot_sys_uninstall_apt_get_remove); - dir_path subprj_out_dir (out_dir / d); - - r.status |= run_b ( - step_id::b_test_installed_configure, - envvars, + r.status |= run_apt_get ( + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "configure('" + - subprj_src_dir.representation () + "'@'" + - subprj_out_dir.representation () + "')", - step_args (env_args, step_id::b_test_installed_configure), - step_args (config_args, step_id::b_test_installed_configure)); + bkp_step, bkp_status, aux_env, last_cmd, + "remove", + "--assume-yes", + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss), + pns); if (!r.status) break; - - test_specs.push_back ( - "test('" + subprj_out_dir.representation () + "')"); } - - if (!r.status) + // + // Fail if the breakpoint refers to the + // bbot.sys-uninstall.apt-get.remove step but the distribution is + // other than `debian`. + // + else if (bkp_step && + *bkp_step == step_id::bbot_sys_uninstall_apt_get_remove) + { + fail_unreached_breakpoint (r); break; + } + // + // Uninstall for the `fedora` distribution. + // + else if (*bindist == step_id::bpkg_bindist_fedora) + { + // sudo dnf remove <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // <distribution-package-name>... + // + step_id b (step_id::bbot_sys_uninstall_dnf_remove); + step_id s (step_id::bbot_sys_uninstall_dnf_remove); + step_id ss (step_id::bbot_sys_uninstall_dnf_remove); + + r.status |= run_dnf ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "remove", + "--assumeyes", + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss), + pns); - // Build/test subprojects. + if (!r.status) + break; + } // - // b test(<subprj-out-dir>)... <env-config-args> <config-args> + // Fail if the breakpoint refers to the bbot.sys-uninstall.dnf.remove + // step but the distribution is other than `fedora`. // - // b.test-installed.test + else if (bkp_step && + *bkp_step == step_id::bbot_sys_uninstall_dnf_remove) + { + fail_unreached_breakpoint (r); + break; + } + else + assert (false); + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to a bbot.sys-uninstall.* step but + // this step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_uninstall_apt_get_remove && + *bkp_step <= step_id::bbot_sys_uninstall_dnf_remove) + { + fail_unreached_breakpoint (add_result ("sys-uninstall")); + break; + } + + // Uninstall, if installed from source. + // + if (install_root) + { + operation_result* pr (&add_result ("uninstall")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + change_wd (trace, &r.log, effective_install_conf); + + // bpkg uninstall <env-config-args> <tgt-config-args> <pkg-config-args> + // <package-name> // - r.status |= run_b ( - step_id::b_test_installed_test, - envvars, + step_id b (step_id::bpkg_uninstall); + step_id s (step_id::bpkg_uninstall); + + r.status |= run_bpkg ( + b, trace, r.log, wre, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, "-v", - test_specs, - step_args (env_args, step_id::b_test_installed_test), - step_args (config_args, step_id::b_test_installed_test)); + "uninstall", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pkg); if (!r.status) break; + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to the bpkg.uninstall step but the + // package was not installed from source. + // + else if (bkp_step && *bkp_step == step_id::bpkg_uninstall) + { + fail_unreached_breakpoint (add_result ("uninstall")); + break; } - // Run external tests. + // Now, after the package is fully tested, let's prepare the build + // artifacts for the upload, using the upload operation log. + // + + // Prepare the bindist artifacts. + // + // Move the binary distribution files generated for the main package and + // bindist-result.json to the upload/bindist/<distribution>/ directory. + // Also serialize the subset of the bindist result as + // bindist-result.manifest. // - if (external_tests) + // Fail if the breakpoint refers to the bbot.bindist.upload step since + // it has no specific command associated. + // + if (bkp_step && *bkp_step == step_id::bbot_bindist_upload) + { + fail_unreached_breakpoint (add_result ("upload")); + break; + } + + if (bindist_upload) { - // Configure. + operation_result* pr (&add_result ("upload")); + operation_result& r (*pr); // @@ TMP: Apple Clang 14.0.3 ICE + + change_wd (trace, &r.log, rwd); + + dir_path d (upload_dir / + dir_path ("bindist") / + dir_path (bindist_result.distribution)); + + log_step_id (trace, r.log, step_id::bbot_bindist_upload); + + mk_p (trace, &r.log, d); + + // Move a file to the upload/bindist/<distribution>/ directory. // - // bpkg create <env-modules> <env-config-args> <config-args> + // On Windows copy the file instead of moving not to fail if it is + // being scanned by the Windows Defender or some such. // - // bpkg.test-installed.create (bpkg.create) + auto mv = [&d, &r, &rwd, &trace] (const path& p) + { +#ifndef _WIN32 + bool mv (true); +#else + bool mv (false); +#endif + // Use relative path, if possible, to keep the operation log tidy + // (won't be the case on Fedora). + // + const path& rp (p.sub (rwd) ? p.leaf (rwd) : p); + + if (mv) + mv_into (trace, &r.log, rp, d); + else + cp_into (trace, &r.log, rp, d); + }; + + // Main package files. // - dir_path config_dir ("build-installed-bpkg"); + for (const bindist_file& f: bindist_result.package.files) + mv (f.path); - r.status |= run_bpkg ( - step_id::bpkg_test_installed_create, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-V", - "create", - "-d", config_dir.string (), - "--wipe", + // Bindist result JSON. + // + mv (bindist_result_file); - step_args (modules, - step_id::bpkg_test_installed_create, - step_id::bpkg_create), + // Bindist result manifest. + // + path mf (d / "bindist-result.manifest"); - step_args (env_args, - step_id::bpkg_test_installed_create, - step_id::bpkg_create), + try + { + ofdstream os (mf); + serializer s (os, mf.string ()); - step_args (config_args, - step_id::bpkg_test_installed_create, - step_id::bpkg_create)); + // Serialize package manifest. + // + s.next ("", "1"); // Start of manifest. - if (!r.status) - break; + s.next ("distribution", bindist_result.distribution); + s.next ("architecture", bindist_result.architecture); - change_wd (trace, &r.log, config_dir); + s.next ("os-release-name-id", bindist_result.os_release.name_id); - // bpkg add <env-config-args> <config-args> <repository-url> - // - // bpkg.test-installed.configure.add (bpkg.configure.add) - // - r.status |= run_bpkg ( - step_id::bpkg_test_installed_configure_add, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "add", + // Should have failed earlier. + // + assert (bindist_result.os_release.version_id); - step_args (env_args, - step_id::bpkg_test_installed_configure_add, - step_id::bpkg_configure_add), + s.next ("os-release-version-id", + *bindist_result.os_release.version_id); - step_args (config_args, - step_id::bpkg_test_installed_configure_add, - step_id::bpkg_configure_add), + s.next ("package-name", bindist_result.package.name); + s.next ("package-version", bindist_result.package.version); - repo); + if (bindist_result.package.system_version) + s.next ("package-system-version", + *bindist_result.package.system_version); - if (!r.status) - break; + s.next ("", ""); // End of manifest. - // bpkg fetch <env-config-args> <config-args> <trust-options> - // - // bpkg.test-installed.configure.fetch (bpkg.configure.fetch) - // - r.status |= run_bpkg ( - step_id::bpkg_test_installed_configure_fetch, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "fetch", + // Serialize package file manifests. + // + for (const bindist_file& f: bindist_result.package.files) + { + s.next ("", "1"); // Start of manifest. - step_args (env_args, - step_id::bpkg_test_installed_configure_fetch, - step_id::bpkg_configure_fetch), + s.next ("package-file-type", f.type); - step_args (config_args, - step_id::bpkg_test_installed_configure_fetch, - step_id::bpkg_configure_fetch), + // Note: the simple path representation is POSIX. + // + s.next ("package-file-path", f.path.leaf ().string ()); - trust_ops); + if (f.system_name) + s.next ("package-file-system-name", *f.system_name); - if (!r.status) - break; + s.next ("", ""); // End of manifest. + } + + s.next ("", ""); // End of stream. + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << mf << "': " << e; + } + catch (const serialization& e) + { + fail << "unable to serialize bindist result: " << e; + } + } + + // Create the archive of the build artifacts for subsequent upload, if + // the upload/ directory is not empty. + // + // Note that the archive will be uploaded later, right before uploading + // the result manifest, unless the task has been aborted by the user or + // the result status is error or worse. + // + if (!empty (rwd / upload_dir) && !step_disabled (step_id::bbot_upload)) + { + // The upload operation log must have been added as part of the + // build artifacts preparation for upload. + // + operation_result& r (rm.results.back ()); - // The test separate installed phase. + // Fail if the breakpoint refers to the bbot.upload step since it has + // no specific command associated. // - if (!test (r, - rwd / dir_path ("dist-installed"), - true /* installed */, - true /* sys_dep */, - nullptr /* import */, - envvars)) + if (bkp_step && *bkp_step == step_id::bbot_upload) + { + fail_unreached_breakpoint (r); break; + } + + change_wd (trace, &r.log, rwd); - // Back to the test installed phase. + // Archive the build artifacts. // - } + { + // tar -cf upload.tar <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // upload/ + // + step_id b (step_id::bbot_upload_tar_create); + step_id s (step_id::bbot_upload_tar_create); + step_id ss (step_id::bbot_upload_tar_create); - rm.status |= r.status; - } + // Make sure the archive is portable. + // + // Note that OpenBSD tar does not support --format but it appear + // ustar is the default (see bpkg/system-package-manager-archive.cxx + // for details). + // + r.status |= run_tar ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + false /* sudo */, +#ifndef __OpenBSD__ + "--format", "ustar", +#endif + "-cf", + upload_archive, + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss), + upload_dir); - // Back to the main phase. - // - // Uninstall. - // - { - operation_result& r (add_result ("uninstall")); + if (!r.status) + break; + } + + // It feels useful to also print the archive content to the log. + // + { + // tar -tf upload.tar <env-config-args> + // <tgt-config-args> + // <pkg-config-args> + // + step_id b (step_id::bbot_upload_tar_list); + step_id s (step_id::bbot_upload_tar_list); + step_id ss (step_id::bbot_upload_tar_list); - change_wd (trace, &r.log, pkg_config); + r.status |= run_tar ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + false /* sudo */, + "-tf", + upload_archive, + step_args (env_args, s, nullopt, nullopt, ss), + step_args (tgt_args, s, nullopt, nullopt, ss), + step_args (pkg_args, s, nullopt, nullopt, ss)); - // bpkg uninstall <env-config-args> <config-args> <package-name> + if (!r.status) + break; + } + + rm.status |= r.status; + } // - // bpkg.uninstall + // Fail if the breakpoint refers to some of the bbot.upload.* steps but + // there is either nothing to upload or the bbot.upload step is + // disabled. // - r.status |= run_bpkg ( - step_id::bpkg_uninstall, - trace, r.log, wre, - bkp_step, bkp_status, last_cmd, - "-v", - "uninstall", - step_args (env_args, step_id::bpkg_uninstall), - step_args (config_args, step_id::bpkg_uninstall), - pkg); + else if (bkp_step && + *bkp_step >= step_id::bbot_upload && + *bkp_step <= step_id::bbot_upload_tar_list) + { + // If the upload operation log have been added as part of the build + // artifacts preparation for upload, then use this log to report the + // error. Otherwise, add the new log for that. + // + // @@ TMP: Apple Clang 14.0.3 ICE + // + operation_result* pr (&rm.results.back ()); - if (!r.status) - break; + if (pr->operation != "upload") + pr = &add_result ("upload"); - rm.status |= r.status; + fail_unreached_breakpoint (*pr); + break; + } + } + // + // Fail if the breakpoint refers to bpkg.update or any dependent step but + // the bpkg.update step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bpkg_update && + *bkp_step <= step_id::bbot_upload) + { + fail_unreached_breakpoint (add_result ("update")); + break; } break; @@ -2048,16 +5883,6 @@ build (size_t argc, const char* argv[]) rm.status |= r.status; // Merge last in case of a break. - // Also merge statuses of the configure and test operations, which logs - // can potentially be shared across multiple steps and which results may - // not be the last in the list. - // - if (configure_result != nullptr) - rm.status |= configure_result->status; - - if (test_result != nullptr) - rm.status |= test_result->status; - // Unless there is an error (or worse) encountered, log the special 'end' // step and, if this step is specified in the interactive manifest value, // ask the user if to continue the task execution. @@ -2067,9 +5892,11 @@ build (size_t argc, const char* argv[]) if (!error) { r.status |= run_cmd (step_id::end, - trace, r.log, regexes (), + trace, r.log, + nullptr /* out_str */, path () /* out_file */, + regexes (), "" /* name */, - bkp_step, bkp_status, last_cmd, + bkp_step, bkp_status, aux_env, last_cmd, process_env ()); rm.status |= r.status; @@ -2116,11 +5943,45 @@ build (size_t argc, const char* argv[]) } } else - assert (rm.status == result_status::abort); + assert (rm.status == result_status::abort || + rm.status == result_status::skip); if (!rwd.empty ()) + { change_wd (trace, nullptr /* log */, rwd); + // Upload the build artifacts archive, if exists. + // + bool error (!rm.status); + if (exists (upload_archive) && !error) + { + const string url ( + "tftp://" + ops.tftp_host () + '/' + upload_archive.string ()); + + try + { + tftp_curl c (trace, + upload_archive, + nullfd, + curl::put, + url, + "--tftp-blksize", tftp_blksize, + "--max-time", tftp_put_timeout); + + if (!c.wait ()) + fail << "curl " << *c.exit; + } + catch (const process_error& e) + { + fail << "unable to execute curl: " << e; + } + catch (const system_error& e) + { + fail << "unable to upload build artifacts to " << url << ": " << e; + } + } + } + // Upload the result. // const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4"); @@ -2145,6 +6006,109 @@ build (size_t argc, const char* argv[]) return 3; } +// Parse the task_manifest::auxiliary_environment value into the list of +// environment variable assignments as expected by the process API. Throw +// invalid_argument if the auxiliary environment is invalid. +// +// If comment is not NULL, then add blank and comment lines prefixed with this +// string (which is normally either '#' or 'rem'). This mode is used to print +// the environment into the build log. +// +static strings +parse_auxiliary_environment (const string& s, const char* comment = nullptr) +{ + strings r; + + // Note: parse observing blanks. + // + for (size_t b (0), e (0), m (0), n (s.size ()); + next_word (s, n, b, e, m, '\n', '\r'), b != n; ) + { + string line (s, b, e - b); + + if (trim (line).empty ()) // Blank. + { + if (comment != nullptr) + r.push_back (comment); + + continue; + } + + if (line.front () == '#') // Comment. + { + if (comment != nullptr) + { + line.erase (0, 1); + line.insert (0, comment); + r.push_back (move (line)); + } + + continue; + } + + size_t p (line.find ('=')); + + if (p == string::npos) + throw invalid_argument ("missing '=' in '" + line + '\''); + + string name (line, 0, p); + + if (trim_right (name).empty () || + name.find_first_not_of ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789") != string::npos) + { + throw invalid_argument ("invalid variable name '" + name + '\''); + } + + // Disallow certain well-known environment variables. + // + if (name == "PATH" +#if defined(_WIN32) +#elif defined(__APPLE__) + || name == "DYLD_LIBRARY_PATH" +#else // Linux, FreeBSD, NetBSD, OpenBSD + || name == "LD_LIBRARY_PATH" +#endif + ) + { + throw invalid_argument ("disallowed variable name '" + name + '\''); + } + + line.erase (0, p + 1); // Value. + + // Note: we allow empty values. + // + if (!trim_left (line).empty ()) + { + // Unquote. + // + char c (line.front ()); + if (c == '"' || c == '\'') + { + if (line.size () == 1 || line.back () != c) + throw invalid_argument ("invalid quoted value '" + line + '\''); + + line.pop_back (); + line.erase (0, 1); + } + } + + // Reassemble. + // + line.insert (0, 1, '='); + line.insert (0, name); + + r.push_back (move (line)); + } + + // Pop the final blank line comment. + // + if (comment != nullptr && r.back () == comment) + r.pop_back (); + + return r; +} + static int startup () { @@ -2154,11 +6118,13 @@ startup () // // 1. Download the task manifest into the build directory (CWD). // - // 2. Parse it and get the target. + // 2. Parse it and get the target, environment name, and auxiliary + // environment. // - // 3. Find the environment setup executable for this target. + // 3. Find the environment setup executable for this name. // - // 4. Execute the environment setup executable. + // 4. Execute the environment setup executable for this target in the + // auxiliary environment. // // 5. If the environment setup executable fails, then upload the (failed) // result ourselves. @@ -2173,6 +6139,33 @@ startup () // task_manifest tm; + auto upload_result = [&trace, &tm] (result_status rs, + operation_results&& ors) + { + const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4"); + + // If we failed before being able to parse the task manifest, use the + // "unknown" values for the package name and version. + // + result_manifest rm { + tm.name.empty () ? bpkg::package_name ("unknown") : tm.name, + tm.version.empty () ? bpkg::version ("0") : tm.version, + rs, + move (ors), + worker_checksum, + nullopt /* dependency_checksum */ + }; + + try + { + upload_manifest (trace, url, rm, "result"); + } + catch (const io_error& e) + { + fail << "unable to upload result manifest to " << url << ": " << e; + } + }; + try { // Download the task. @@ -2254,6 +6247,31 @@ startup () fail << "no default environment setup executable in " << env_dir; } + // Auxiliary environment. + // + strings aux_env; + if (tm.auxiliary_environment) + { + try + { + aux_env = parse_auxiliary_environment (*tm.auxiliary_environment); + } + catch (const invalid_argument& e) + { + // Note: include (unparsed) environment into the log so that we can + // see what we are dealing with. + // + operation_result r { + "configure", + result_status::abort, + *tm.auxiliary_environment + "\n" + + "error: invalid auxiliary environment: " + e.what () + '\n'}; + + upload_result (result_status::abort, {move (r)}); + return 1; + } + } + // Run it. // strings os; @@ -2291,7 +6309,12 @@ startup () // result manifest. There is no reason to retry (most likely there is // nobody listening on the other end anymore). // - switch (run_io_exit (trace, 0, 2, 2, pp, tg, argv0.effect_string (), os)) + switch (run_io_exit (trace, + 0 /* stdin */, 2 /* stdout */, 2 /* stderr */, + process_env (pp, aux_env), + tg, + argv0.effect_string (), + os)) { case 3: case 2: return 1; @@ -2301,27 +6324,7 @@ startup () } catch (const failed&) { - const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4"); - - // If we failed before being able to parse the task manifest, use the - // "unknown" values for the package name and version. - // - result_manifest rm { - tm.name.empty () ? bpkg::package_name ("unknown") : tm.name, - tm.version.empty () ? bpkg::version ("0") : tm.version, - result_status::abnormal, - operation_results {} - }; - - try - { - upload_manifest (trace, url, rm, "result"); - } - catch (const io_error& e) - { - fail << "unable to upload result manifest to " << url << ": " << e; - } - + upload_result (result_status::abnormal, operation_results {}); return 1; } } @@ -2410,7 +6413,7 @@ try << "libbpkg " << LIBBPKG_VERSION_ID << endl << "libbutl " << LIBBUTL_VERSION_ID << endl << "Copyright (c) " << BBOT_COPYRIGHT << "." << endl - << "TBC; All rights reserved" << endl; + << "This is free software released under the MIT license." << endl; return 0; } |