aboutsummaryrefslogtreecommitdiff
path: root/bbot/worker/worker.cxx
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2020-10-01 22:08:03 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2020-10-06 15:19:27 +0300
commit235771a20da1da0c3ff495d03e6c223880cbcf20 (patch)
tree2e5a24db5500335a58c75fa066089f2bf7b92726 /bbot/worker/worker.cxx
parentdd4393647c51e75abeea8cd3e516ca1ed2bd23f3 (diff)
Add re-distribution phase for checked out packages
Diffstat (limited to 'bbot/worker/worker.cxx')
-rw-r--r--bbot/worker/worker.cxx201
1 files changed, 191 insertions, 10 deletions
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 17c4fa9..28a7a5b 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -58,6 +58,17 @@ namespace bbot
const size_t tftp_get_retries (3); // Task request retries (see startup()).
}
+bool
+exists (const dir_path& d)
+try
+{
+ return dir_exists (d);
+}
+catch (const system_error& e)
+{
+ fail << "unable to stat path " << d << ": " << e << endf;
+}
+
static dir_path
current_directory ()
try
@@ -100,6 +111,42 @@ catch (const system_error& e)
fail << "unable to change current directory to " << d << ": " << e << endf;
}
+static void
+mv (tracer& t, string* log, const dir_path& from, const dir_path& to)
+try
+{
+ if (verb >= 3)
+ t << "mv " << from << ' ' << to;
+
+ if (log != nullptr)
+ *log += "mv " + from.representation () + ' ' + to.representation () +
+ "\n";
+
+ mvdir (from, to);
+}
+catch (const system_error& e)
+{
+ fail << "unable to move directory '" << from << "' to '" << to << "': " << e
+ << endf;
+}
+
+static void
+rm_r (tracer& t, string* log, const dir_path& d)
+try
+{
+ if (verb >= 3)
+ t << "rm -r " << d;
+
+ if (log != nullptr)
+ *log += "rm -r " + d.representation () + '\n';
+
+ rmdir_r (d);
+}
+catch (const system_error& e)
+{
+ fail << "unable to remove directory " << d << ": " << e << endf;
+}
+
using std::regex;
namespace regex_constants = std::regex_constants;
using regexes = vector<regex>;
@@ -621,6 +668,71 @@ build (size_t argc, const char* argv[])
b_project_info prj; // Package project information.
+ rwd = current_directory ();
+
+ // If the package comes from a version control-based repository, then we
+ // will also test its dist meta-operation. Specifically, we will checkout
+ // the package outside the configuration directory passing --checkout-root
+ // to the configure-only pkg-build command, re-distribute the checked out
+ // directory in the load distribution mode, and then use this distribution
+ // as a source to build the package.
+ //
+ dir_path dist_root (rwd / dir_path ("dist"));
+ dir_path dist_src (dist_root / pkg_dir);
+
+ // Redistribute the package source directory (pkg_dir) checked out into
+ // the directory other than the configuration directory (dist_root) and
+ // replace it with the newly created distribution. Assume that the current
+ // directory is the package configuration directory. Optionally pass the
+ // config.import.* variable override and/or set the environment variables
+ // for the build2 process. Return true if the dist meta-operation
+ // succeeds.
+ //
+ auto redist = [&trace, &wre] (operation_result& r,
+ const dir_path& dist_root,
+ const dir_path& pkg_dir, // <name>-<version>
+ const char* import = nullptr,
+ const small_vector<string, 1>& envvars = {})
+ {
+ // Temporarily change the current directory to the distribution root
+ // parent directory from the configuration directory to shorten the
+ // command line paths and try to avoid the '..' path prefix.
+ //
+ dir_path dp (dist_root.directory ());
+ dir_path dn (dist_root.leaf ());
+
+ // Redistribute the package using the configured output directory.
+ //
+ dir_path cnf_dir (change_wd (trace, &r.log, dp));
+ dir_path out_dir (cnf_dir.relative (dp) / pkg_dir);
+ dir_path src_dir (dn / pkg_dir);
+
+ // Create the re-distribution root directory next to the distribution
+ // root.
+ //
+ dir_path redist_root ("re" + dn.string ());
+
+ r.status |= run_b (
+ envvars,
+ trace, r.log, wre,
+ "-v",
+ "config.dist.root=" + redist_root.string (),
+ import,
+ ("dist('" + src_dir.representation () + "'@'" +
+ out_dir.representation () + "')"));
+
+ if (!r.status)
+ return false;
+
+ // Replace the package source directory with the re-distribution result.
+ //
+ rm_r (trace, &r.log, src_dir);
+ mv (trace, &r.log, redist_root / pkg_dir, src_dir);
+
+ change_wd (trace, &r.log, cnf_dir); // Return back to the configuration.
+ return true;
+ };
+
// If this is a build system module, perform a "pre-step" by building it
// in a separate configuration reproducing the one used to build build2
// itself. Note that the configuration and the environment options and
@@ -630,8 +742,6 @@ build (size_t argc, const char* argv[])
bool module (pkg.compare (0, 10, "libbuild2-") == 0);
dir_path module_dir ("build-module");
- rwd = current_directory ();
-
// If this is a build system module that requires bootstrap, then its
// importation into the dependent (test) projects cannot be configured and
// the corresponding config.import.* variable needs to be specified on the
@@ -639,7 +749,7 @@ build (size_t argc, const char* argv[])
//
// Note that such a module must be explicitly marked with `requires:
// bootstrap` in its manifest. This can only be detected after the module
- // is configured and it's manifest available.
+ // is configured and its manifest available.
//
bool bootstrap (false);
@@ -647,7 +757,7 @@ build (size_t argc, const char* argv[])
// configured.
//
package_manifest pm;
- path mf (pkg_dir / "manifest"); // Relative to the configuration directory.
+ path mf ("manifest");
if (module)
{
@@ -732,6 +842,7 @@ build (size_t argc, const char* argv[])
"-v",
"build",
"--configure-only",
+ "--checkout-root", dist_root,
"--yes",
pkg_rev);
@@ -740,11 +851,14 @@ build (size_t argc, const char* argv[])
rm.status |= r.status;
+ bool dist (exists (dist_src));
+ const dir_path& src_dir (dist ? dist_src : pkg_dir);
+
// Note that being unable to parse the package manifest is likely to
// be an infrastructure problem, given that the package has been
// successfully configured.
//
- pm = parse_manifest<package_manifest> (mf, "package");
+ pm = parse_manifest<package_manifest> (src_dir / mf, "package");
bootstrap = find_if (pm.requirements.begin (),
pm.requirements.end (),
@@ -752,6 +866,17 @@ build (size_t argc, const char* argv[])
{
return r.size () == 1 && r[0] == "bootstrap";
}) != pm.requirements.end ();
+
+ if (dist)
+ {
+ // Note that we reuse the configure operation log for the dist
+ // meta-operation.
+ //
+ if (!redist (r, dist_root, pkg_dir))
+ break;
+
+ rm.status |= r.status;
+ }
}
// Update.
@@ -902,6 +1027,7 @@ build (size_t argc, const char* argv[])
"-v",
"build",
"--configure-only",
+ "--checkout-root", dist_root,
"--yes",
step_args (env_args, step_id::bpkg_configure_build),
step_args (config_args, step_id::bpkg_configure_build),
@@ -911,7 +1037,18 @@ build (size_t argc, const char* argv[])
if (!r.status)
break;
- pm = parse_manifest<package_manifest> (mf, "package");
+ bool dist (exists (dist_src));
+ const dir_path& src_dir (dist ? dist_src : pkg_dir);
+
+ pm = parse_manifest<package_manifest> (src_dir / mf, "package");
+
+ if (dist)
+ {
+ if (!redist (r, dist_root, pkg_dir))
+ break;
+
+ rm.status |= r.status;
+ }
}
rm.status |= r.status;
@@ -987,7 +1124,8 @@ build (size_t argc, const char* argv[])
bool external_tests (!pm.tests.empty ());
- // Configure, update, and test packages in the bpkg configuration in the
+ // Configure, re-distribute if comes from a version control-based
+ // repository, update, and test packages in the bpkg configuration in the
// current working directory. Optionally pass the config.import.* variable
// override and/or set the environment variables for bpkg processes.
// Return true if all operations for all packages succeed.
@@ -997,8 +1135,12 @@ build (size_t argc, const char* argv[])
// installed dependents. Note that bpkg configures the dependent package
// as a special dependency for the test package.
//
- auto test = [&pm, &trace, &wre, &step_args, &config_args, &env_args]
+ auto test = [&trace, &wre,
+ &step_args, &config_args, &env_args,
+ &pm,
+ &redist]
(operation_result& r,
+ const dir_path& dist_root,
bool sys_dep,
const char* import = nullptr,
const small_vector<string, 1>& envvars = {})
@@ -1020,6 +1162,7 @@ build (size_t argc, const char* argv[])
"-v",
"build",
"--configure-only",
+ "--checkout-root", dist_root,
"--yes",
step_args (env_args, step_id::bpkg_configure_build),
step_args (config_args, step_id::bpkg_configure_build),
@@ -1031,6 +1174,37 @@ build (size_t argc, const char* argv[])
if (!r.status)
return false;
+ // Note that re-distributing the test package is a bit tricky since we
+ // don't know its version and so cannot deduce its source directory
+ // name easily. We could potentially run the bpkg-status command after
+ // the package is configured and parse the output to obtain the
+ // version. Let's, however, keep it simple and find the source
+ // directory using the package directory name pattern.
+ //
+ if (exists (dist_root))
+ try
+ {
+ dir_path pkg_dir;
+
+ path_search (dir_path (pkg + "-*/"),
+ [&pkg_dir] (path&& pe, const string&, bool interm)
+ {
+ if (!interm)
+ pkg_dir = path_cast<dir_path> (move (pe));
+
+ return interm;
+ },
+ dist_root);
+
+ if (!pkg_dir.empty () &&
+ !redist (r, dist_root, pkg_dir, import, envvars))
+ return false;
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to scan directory " << dist_root << ": " << e;
+ }
+
// Update.
//
// bpkg update <env-config-args> <config-args> <package-name>
@@ -1112,7 +1286,10 @@ build (size_t argc, const char* argv[])
// package (except for the build system module).
//
if (external_tests &&
- !test (r, module, bootstrap ? module_import.c_str () : nullptr))
+ !test (r,
+ dist_root,
+ module,
+ bootstrap ? module_import.c_str () : nullptr))
break;
rm.status |= r.status;
@@ -1368,7 +1545,11 @@ build (size_t argc, const char* argv[])
// Build/test.
//
- if (!test (r, true /* sys_dep */, nullptr /* import */, envvars))
+ if (!test (r,
+ rwd / dir_path ("dist-installed"),
+ true /* sys_dep */,
+ nullptr /* import */,
+ envvars))
break;
}