From 9abe5e449cc3dab0715ebc86de2a86e6cb8ecc63 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 6 Apr 2023 22:54:43 +0300 Subject: Add support for bbot.sys-install step in worker --- bbot/worker/worker.cxx | 586 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 551 insertions(+), 35 deletions(-) (limited to 'bbot') diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 98c17d4..e8c6af7 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -225,10 +225,17 @@ enum class step_id bpkg_bindist_archive, // Note that this step is considered disabled unless one of the - // bpkg_bindist_* steps is explicitly enabled. + // bpkg_bindist_* steps is explicitly enabled. Note: not a breakpoint. // bbot_sys_install, + bbot_sys_install_apt_get_update, + bbot_sys_install_apt_get_install, + bbot_sys_install_dnf_install, + bbot_sys_install_tar_extract, + + bbot_sys_install_ldconfig, // Note: disabled by default. + // Note: skipped for modules. // b_test_installed_create, //: b_create @@ -267,7 +274,8 @@ enum class step_id bpkg_uninstall, - bbot_sys_uninstall, + bbot_sys_uninstall_apt_get_remove, + bbot_sys_uninstall_dnf_remove, end }; @@ -300,7 +308,6 @@ static const strings step_id_str { "bpkg.test-separate.test", "bpkg.install", - "bbot.install.ldconfig", "bpkg.bindist.debian", @@ -308,6 +315,11 @@ static const strings step_id_str { "bpkg.bindist.archive", "bbot.sys-install", + "bbot.sys-install.apt-get.update", + "bbot.sys-install.apt-get.install", + "bbot.sys-install.dnf.install", + "bbot.sys-install.tar.extract", + "bbot.sys-install.ldconfig", "b.test-installed.create", "b.test-installed.configure", @@ -332,7 +344,8 @@ static const strings step_id_str { "bpkg.uninstall", - "bbot.sys-uninstall", + "bbot.sys-uninstall.apt-get.remove", + "bbot.sys-uninstall.dnf.remove", "end"}; @@ -641,7 +654,9 @@ run_bpkg (step_id step, { return run_cmd (step, t, - log, nullptr /* out */, path () /* out_file */, warn_detect, + log, + nullptr /* out_str */, path () /* out_file */, + warn_detect, "bpkg " + cmd, bkp_step, bkp_status, last_cmd, process_env ("bpkg", envvars), @@ -683,7 +698,7 @@ run_bpkg (step_id step, { return run_cmd (step, t, - log, nullptr /* out_file */, out, warn_detect, + log, nullptr /* out_str */, out, warn_detect, "bpkg " + cmd, bkp_step, bkp_status, last_cmd, process_env ("bpkg", envvars), @@ -826,6 +841,88 @@ run_ldconfig (step_id step, "ldconfig", forward (a)...); } +template +static result_status +run_apt_get (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional& bkp_step, + const optional& bkp_status, + string& last_cmd, + const string& cmd, A&&... a) +{ + // Note: dumps some of its diagnostics to stdout. + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo apt-get " + cmd, + bkp_step, bkp_status, last_cmd, + process_env ("sudo"), + "apt-get", cmd, forward (a)...); +} + +template +static result_status +run_dnf (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional& bkp_step, + const optional& bkp_status, + string& last_cmd, + const string& cmd, A&&... a) +{ + // Note: dumps some of its diagnostics to stdout. + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo dnf " + cmd, + bkp_step, bkp_status, last_cmd, + process_env ("sudo"), + "dnf", cmd, forward (a)...); +} + +template +static result_status +run_tar (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional& bkp_step, + const optional& bkp_status, + string& last_cmd, + A&&... a) +{ +#ifndef _WIN32 + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "sudo tar", + bkp_step, bkp_status, last_cmd, + process_env ("sudo"), + "tar", forward (a)...); +#else + // Note: using bsdtar which can unpack .zip archives (and also not an MSYS + // executable). + // + return run_cmd (step, + t, + log, + nullptr /* out_str */, path () /* out_file*/, + warn_detect, + "bsdtar", + bkp_step, bkp_status, last_cmd, + process_env ("bsdtar"), + forward (a)...); +#endif +} + // Upload compressed manifest to the specified TFTP URL with curl. Issue // diagnostics and throw failed on invalid manifest or process management // errors and throw io_error for input/output errors or non-zero curl exit. @@ -1766,35 +1863,52 @@ build (size_t argc, const char* argv[]) // optional install_root; + // While building and running tests against the installation created + // either from source or from the archive distribution package we will + // make the bin/ subdirectory of config.install.root, if specified, the + // first entry in the PATH environment variable, except for build system + // modules which supposedly don't install any executables. + // + optional install_bin; + + auto config_install_root = [&step_args, &tgt_args] () -> optional + { + step_id s (step_id::bpkg_target_create); + step_id f1 (step_id::b_create); + step_id f2 (step_id::bpkg_create); + + size_t n (19); + auto space = [] (char c) {return c == ' ' || c == '\t';}; + + for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2))) + { + if (strncmp (a, "config.install.root", n) == 0 && + (a[n] == '=' || space (a[n]))) + { + while (space (a[n])) ++n; // Skip spaces. + if (a[n] == '=') ++n; // Skip the equal sign. + while (space (a[n])) ++n; // Skip spaces. + + // Note that the config.install.root variable value may potentially + // be quoted. + // + return dir_path (unquote (a + n)); + } + } + + return nullopt; + }; + if ((target_pkg || selfhost) && !bindist && !step_disabled (step_id::bpkg_install)) { if (!module_pkg) { - step_id s (step_id::bpkg_target_create); - step_id f1 (step_id::b_create); - step_id f2 (step_id::bpkg_create); - - size_t n (19); - auto space = [] (char c) {return c == ' ' || c == '\t';}; - - for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2))) - { - if (strncmp (a, "config.install.root", n) == 0 && - (a[n] == '=' || space (a[n]))) - { - while (space (a[n])) ++n; // Skip spaces. - if (a[n] == '=') ++n; // Skip the equal sign. - while (space (a[n])) ++n; // Skip spaces. + install_root = config_install_root (); - // Note that the config.install.root variable value may potentially - // be quoted. - // - install_root = dir_path (unquote (a + n)); - break; - } - } + if (install_root) + install_bin = *install_root / dir_path ("bin"); } else install_root = dir_path (); @@ -3353,6 +3467,8 @@ build (size_t argc, const char* argv[]) // if (install_root) { + // Make install_conf refer to the actual configuration directory. + // install_conf = rwd / (create_install ? install_conf : main_pkg_conf); operation_result& r (add_result ("install")); @@ -3479,6 +3595,13 @@ build (size_t argc, const char* argv[]) change_wd (trace, &r.log, rwd); + // Note that for a host or module package we don't need the target + // configuration anymore, if present. So let's free up the space a + // little bit. + // + if (!target_pkg && create_target) + rm_r (trace, &r.log, rwd / target_conf); + const dir_path& bindist_conf ( create_install ? install_conf : main_pkg_conf); @@ -3689,6 +3812,8 @@ build (size_t argc, const char* argv[]) for (const bindist_file& f: bindist_result.package.files) log_line (" " + f.path.string (), r.log); + + rm.status |= r.status; } // // Fail if the breakpoint refers to a bpkg.bindist.* step but this step @@ -3703,6 +3828,260 @@ build (size_t argc, const char* argv[]) break; } + // Install from the binary distribution package generated on a + // bpkg.bindist.* step. + // + if (sys_install) + { + operation_result& r (add_result ("sys-install")); + + // Fail if the breakpoint refers to the bbot.sys-install step since + // it has no specific command associated. + // + if (bkp_step && *bkp_step == step_id::bbot_sys_install) + { + fail_unreached_breakpoint (add_result ("sys-install")); + break; + } + + // Noop, just for the log record. + // + change_wd (trace, &r.log, rwd); + + // Collect the binary package files. + // + // Specifically, for now we consider files with the system name + // specified as package files. + // + cstrings pfs; + + auto add_package_files = [&pfs] (const vector& bfs) + { + for (const bindist_file& f: bfs) + { + if (!f.system_name.empty ()) + pfs.push_back (f.path.string ().c_str ()); + } + }; + + add_package_files (bindist_result.package.files); + + for (const bindist_package& d: bindist_result.dependencies) + add_package_files (d.files); + + // Install for the `debian` distribution. + // + if (*bindist == step_id::bpkg_bindist_debian) + { + // Update package index. + // + { + // sudo apt-get update + // + // + // + step_id b (step_id::bbot_sys_install_apt_get_update); + step_id s (step_id::bbot_sys_install_apt_get_update); + + r.status |= run_apt_get ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "update", + "--assume-yes", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s)); + + if (!r.status) + break; + } + + // Install. + // + { + // sudo apt-get install + // + // + // ... + // + // Note that apt-get install requires a directory separator for an + // argument to be treated as a file rather than name. The paths we + // pass are absolute. + // + step_id b (step_id::bbot_sys_install_apt_get_install); + step_id s (step_id::bbot_sys_install_apt_get_install); + + r.status |= run_apt_get ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "install", + "--assume-yes", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pfs); + + if (!r.status) + break; + } + } + // + // Fail if the breakpoint refers to a bbot.sys-install.apt_get.* step + // but the distribution is other than `debian`. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install_apt_get_update && + *bkp_step <= step_id::bbot_sys_install_apt_get_install) + { + fail_unreached_breakpoint (r); + break; + } + // + // Install for the `fedora` distribution. + // + else if (*bindist == step_id::bpkg_bindist_fedora) + { + // sudo dnf install + // + // + // ... + // + step_id b (step_id::bbot_sys_install_dnf_install); + step_id s (step_id::bbot_sys_install_dnf_install); + + r.status |= run_dnf ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "install", + "--refresh", + "--assumeyes", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pfs); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to a bbot.sys-install.dnf.* step but + // the distribution is other than `fedora`. + // + else if (bkp_step && *bkp_step == step_id::bbot_sys_install_dnf_install) + { + fail_unreached_breakpoint (r); + break; + } + // + // Install for the `archive` distribution. + // + // Since there is no easy way to extract from multiple archives with a + // single command, we run tar in a loop. + // + // Note that it is assumed that the --directory and --strip-components + // options are passed via <*-config-args>. The extracted executables + // can be arranged to be found by setting config.install.root for + // bpkg.target.create, etc (the same way as for installing from + // source). + // + else if (*bindist == step_id::bpkg_bindist_archive) + { + for (const char* f: pfs) + { + // [sudo] tar -xf + // + // + // + step_id b (step_id::bbot_sys_install_tar_extract); + step_id s (step_id::bbot_sys_install_tar_extract); + + r.status |= run_tar ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "-xf", + f, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s)); + + if (!r.status) + break; + } + + if (!r.status) + break; + + // Run ldconfig. + // + if (step_enabled (step_id::bbot_sys_install_ldconfig)) + { + // sudo ldconfig + // + // + // + step_id b (step_id::bbot_sys_install_ldconfig); + step_id s (step_id::bbot_sys_install_ldconfig); + + r.status |= run_ldconfig ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s)); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the bbot.sys-install.ldconfig + // step but this step is disabled. + // + else if (bkp_step && *bkp_step == step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (r); + break; + } + + if (!module_pkg) + { + if (optional ir = config_install_root ()) + install_bin = *ir / dir_path ("bin"); + } + } + // + // Fail if the breakpoint refers to a + // bbot.sys-install.{tar.extract,ldconfig} step but the distribution + // is other than `archive`. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install_tar_extract && + *bkp_step <= step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (r); + break; + } + else + assert (false); + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to a bbot.sys-install.* step but this + // step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_install && + *bkp_step <= step_id::bbot_sys_install_ldconfig) + { + fail_unreached_breakpoint (add_result ("sys-install")); + break; + } + // Now, if the package is installed, either from source or from the // binary distribution package, the overall plan is as follows: // @@ -3714,7 +4093,7 @@ build (size_t argc, const char* argv[]) // and test them in a separate bpkg configuration(s) against the // installed package. // - if (install_root /*|| sys_install */) // @@ TMP + if (install_root || sys_install) { // Run the internal tests if the project contains "testable" // subprojects, but not for a module. @@ -3723,10 +4102,24 @@ build (size_t argc, const char* argv[]) dir_paths subprj_dirs; // "Testable" package subprojects. + // Collect the "testable" subprojects. + // if (!module_pkg) { - // Collect the "testable" subprojects. + // Make install_conf refer to the actual configuration directory, if + // not yet. // + if (sys_install) + install_conf = rwd / (create_install ? install_conf : main_pkg_conf); + + assert (!rm.results.empty ()); + + // Result of the install or sys-install operation. + // + operation_result& r (rm.results.back ()); + + change_wd (trace, &r.log, install_conf); + for (const b_project_info::subproject& sp: prj.subprojects) { // Retrieve the subproject information similar to how we've done it @@ -3757,13 +4150,13 @@ build (size_t argc, const char* argv[]) // small_vector envvars; - if (!module_pkg && install_root) + if (install_bin) { // Note that we add the $config.install.root/bin directory at the // beginning of the PATH environment variable value, so the // installed executables are found first. // - string paths ("PATH=" + (*install_root / "bin").string ()); + string paths ("PATH=" + install_bin->string ()); if (optional s = getenv ("PATH")) { @@ -4419,8 +4812,6 @@ build (size_t argc, const char* argv[]) fail_unreached_breakpoint (r); break; } - - rm.status |= r.status; } // // Fail if the breakpoint refers to some of the @@ -4434,6 +4825,8 @@ build (size_t argc, const char* argv[]) fail_unreached_breakpoint (r); break; } + + rm.status |= r.status; } // // Fail if the breakpoint refers to some of the test installed steps @@ -4456,7 +4849,130 @@ build (size_t argc, const char* argv[]) *bkp_step >= step_id::b_test_installed_create && *bkp_step <= step_id::bpkg_test_separate_installed_test) { - fail_unreached_breakpoint (add_result ("install")); + fail_unreached_breakpoint (add_result ("test-install")); + break; + } + + // Uninstall, if installed from the binary distribution package. + // + // Note: noop for the archive distribution. + // + if (sys_install && + (*bindist == step_id::bpkg_bindist_debian || + *bindist == step_id::bpkg_bindist_fedora)) + { + operation_result& r (add_result ("sys-uninstall")); + + // Noop, just for the log record. + // + change_wd (trace, &r.log, rwd); + + // Collect the binary package system names. + // + cstrings pns; + + auto add_package_names = [&pns] (const vector& bfs) + { + for (const bindist_file& f: bfs) + { + if (!f.system_name.empty ()) + pns.push_back (f.system_name.c_str ()); + } + }; + + add_package_names (bindist_result.package.files); + + for (const bindist_package& d: bindist_result.dependencies) + add_package_names (d.files); + + // Uninstall for the `debian` distribution. + // + if (*bindist == step_id::bpkg_bindist_debian) + { + // sudo apt-get remove + // + // + // ... + // + step_id b (step_id::bbot_sys_uninstall_apt_get_remove); + step_id s (step_id::bbot_sys_uninstall_apt_get_remove); + + r.status |= run_apt_get ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "remove", + "--assume-yes", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pns); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the + // bbot.sys-uninstall.apt-get.remove step but the distribution is + // other than `debian`. + // + else if (bkp_step && + *bkp_step == step_id::bbot_sys_uninstall_apt_get_remove) + { + fail_unreached_breakpoint (r); + break; + } + // + // Uninstall for the `fedora` distribution. + // + else if (*bindist == step_id::bpkg_bindist_fedora) + { + // sudo dnf remove + // + // + // ... + // + step_id b (step_id::bbot_sys_uninstall_dnf_remove); + step_id s (step_id::bbot_sys_uninstall_dnf_remove); + + r.status |= run_dnf ( + b, + trace, r.log, wre, + bkp_step, bkp_status, last_cmd, + "remove", + "--assumeyes", + step_args (env_args, s), + step_args (tgt_args, s), + step_args (pkg_args, s), + pns); + + if (!r.status) + break; + } + // + // Fail if the breakpoint refers to the bbot.sys-uninstall.dnf.remove + // step but the distribution is other than `fedora`. + // + else if (bkp_step && + *bkp_step == step_id::bbot_sys_uninstall_dnf_remove) + { + fail_unreached_breakpoint (r); + break; + } + else + assert (false); + + rm.status |= r.status; + } + // + // Fail if the breakpoint refers to a bbot.sys-uninstall.* step but + // this step is disabled. + // + else if (bkp_step && + *bkp_step >= step_id::bbot_sys_uninstall_apt_get_remove && + *bkp_step <= step_id::bbot_sys_uninstall_dnf_remove) + { + fail_unreached_breakpoint (add_result ("sys-uninstall")); break; } -- cgit v1.1