Add re-distribution phase for checked out packages

3 files changed, 230 insertions, 26 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 17c4fa9..28a7a5b 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -58,6 +58,17 @@ namespace bbot
   const size_t tftp_get_retries (3);    // Task request retries (see startup()).
 }
 
+bool
+exists (const dir_path& d)
+try
+{
+  return dir_exists (d);
+}
+catch (const system_error& e)
+{
+  fail << "unable to stat path " << d << ": " << e << endf;
+}
+
 static dir_path
 current_directory ()
 try
@@ -100,6 +111,42 @@ catch (const system_error& e)
   fail << "unable to change current directory to " << d << ": " << e << endf;
 }
 
+static void
+mv (tracer& t, string* log, const dir_path& from, const dir_path& to)
+try
+{
+  if (verb >= 3)
+    t << "mv " << from << ' ' << to;
+
+  if (log != nullptr)
+    *log += "mv " + from.representation () + ' ' + to.representation () +
+            "\n";
+
+  mvdir (from, to);
+}
+catch (const system_error& e)
+{
+  fail << "unable to move directory '" << from << "' to '" << to << "': " << e
+       << endf;
+}
+
+static void
+rm_r (tracer& t, string* log, const dir_path& d)
+try
+{
+  if (verb >= 3)
+    t << "rm -r " << d;
+
+  if (log != nullptr)
+    *log += "rm -r " + d.representation () + '\n';
+
+  rmdir_r (d);
+}
+catch (const system_error& e)
+{
+  fail << "unable to remove directory " << d << ": " << e << endf;
+}
+
 using std::regex;
 namespace regex_constants = std::regex_constants;
 using regexes = vector<regex>;
@@ -621,6 +668,71 @@ build (size_t argc, const char* argv[])
 
     b_project_info prj; // Package project information.
 
+    rwd = current_directory ();
+
+    // If the package comes from a version control-based repository, then we
+    // will also test its dist meta-operation. Specifically, we will checkout
+    // the package outside the configuration directory passing --checkout-root
+    // to the configure-only pkg-build command, re-distribute the checked out
+    // directory in the load distribution mode, and then use this distribution
+    // as a source to build the package.
+    //
+    dir_path dist_root (rwd / dir_path ("dist"));
+    dir_path dist_src  (dist_root / pkg_dir);
+
+    // Redistribute the package source directory (pkg_dir) checked out into
+    // the directory other than the configuration directory (dist_root) and
+    // replace it with the newly created distribution. Assume that the current
+    // directory is the package configuration directory. Optionally pass the
+    // config.import.* variable override and/or set the environment variables
+    // for the build2 process. Return true if the dist meta-operation
+    // succeeds.
+    //
+    auto redist = [&trace, &wre] (operation_result& r,
+                                  const dir_path& dist_root,
+                                  const dir_path& pkg_dir, // <name>-<version>
+                                  const char* import = nullptr,
+                                  const small_vector<string, 1>& envvars = {})
+    {
+      // Temporarily change the current directory to the distribution root
+      // parent directory from the configuration directory to shorten the
+      // command line paths and try to avoid the '..' path prefix.
+      //
+      dir_path dp (dist_root.directory ());
+      dir_path dn (dist_root.leaf ());
+
+      // Redistribute the package using the configured output directory.
+      //
+      dir_path cnf_dir (change_wd (trace, &r.log, dp));
+      dir_path out_dir (cnf_dir.relative (dp) / pkg_dir);
+      dir_path src_dir (dn / pkg_dir);
+
+      // Create the re-distribution root directory next to the distribution
+      // root.
+      //
+      dir_path redist_root ("re" + dn.string ());
+
+      r.status |= run_b (
+        envvars,
+        trace, r.log, wre,
+        "-v",
+        "config.dist.root=" + redist_root.string (),
+        import,
+        ("dist('" + src_dir.representation () + "'@'" +
+         out_dir.representation () + "')"));
+
+      if (!r.status)
+        return false;
+
+      // Replace the package source directory with the re-distribution result.
+      //
+      rm_r (trace, &r.log, src_dir);
+      mv   (trace, &r.log, redist_root / pkg_dir, src_dir);
+
+      change_wd (trace, &r.log, cnf_dir); // Return back to the configuration.
+      return true;
+    };
+
     // If this is a build system module, perform a "pre-step" by building it
     // in a separate configuration reproducing the one used to build build2
     // itself. Note that the configuration and the environment options and
@@ -630,8 +742,6 @@ build (size_t argc, const char* argv[])
     bool module (pkg.compare (0, 10, "libbuild2-") == 0);
     dir_path module_dir ("build-module");
 
-    rwd = current_directory ();
-
     // If this is a build system module that requires bootstrap, then its
     // importation into the dependent (test) projects cannot be configured and
     // the corresponding config.import.* variable needs to be specified on the
@@ -639,7 +749,7 @@ build (size_t argc, const char* argv[])
     //
     // Note that such a module must be explicitly marked with `requires:
     // bootstrap` in its manifest. This can only be detected after the module
-    // is configured and it's manifest available.
+    // is configured and its manifest available.
     //
     bool bootstrap (false);
 
@@ -647,7 +757,7 @@ build (size_t argc, const char* argv[])
     // configured.
     //
     package_manifest pm;
-    path mf (pkg_dir / "manifest"); // Relative to the configuration directory.
+    path mf ("manifest");
 
     if (module)
     {
@@ -732,6 +842,7 @@ build (size_t argc, const char* argv[])
           "-v",
           "build",
           "--configure-only",
+          "--checkout-root", dist_root,
           "--yes",
           pkg_rev);
 
@@ -740,11 +851,14 @@ build (size_t argc, const char* argv[])
 
         rm.status |= r.status;
 
+        bool dist (exists (dist_src));
+        const dir_path& src_dir (dist ? dist_src : pkg_dir);
+
         // Note that being unable to parse the package manifest is likely to
         // be an infrastructure problem, given that the package has been
         // successfully configured.
         //
-        pm = parse_manifest<package_manifest> (mf, "package");
+        pm = parse_manifest<package_manifest> (src_dir / mf, "package");
 
         bootstrap = find_if (pm.requirements.begin (),
                              pm.requirements.end (),
@@ -752,6 +866,17 @@ build (size_t argc, const char* argv[])
                              {
                                return r.size () == 1 && r[0] == "bootstrap";
                              }) != pm.requirements.end ();
+
+        if (dist)
+        {
+          // Note that we reuse the configure operation log for the dist
+          // meta-operation.
+          //
+          if (!redist (r, dist_root, pkg_dir))
+            break;
+
+          rm.status |= r.status;
+        }
       }
 
       // Update.
@@ -902,6 +1027,7 @@ build (size_t argc, const char* argv[])
           "-v",
           "build",
           "--configure-only",
+          "--checkout-root", dist_root,
           "--yes",
           step_args (env_args,    step_id::bpkg_configure_build),
           step_args (config_args, step_id::bpkg_configure_build),
@@ -911,7 +1037,18 @@ build (size_t argc, const char* argv[])
         if (!r.status)
           break;
 
-        pm = parse_manifest<package_manifest> (mf, "package");
+        bool dist (exists (dist_src));
+        const dir_path& src_dir (dist ? dist_src : pkg_dir);
+
+        pm = parse_manifest<package_manifest> (src_dir / mf, "package");
+
+        if (dist)
+        {
+          if (!redist (r, dist_root, pkg_dir))
+            break;
+
+          rm.status |= r.status;
+        }
       }
 
       rm.status |= r.status;
@@ -987,7 +1124,8 @@ build (size_t argc, const char* argv[])
 
     bool external_tests (!pm.tests.empty ());
 
-    // Configure, update, and test packages in the bpkg configuration in the
+    // Configure, re-distribute if comes from a version control-based
+    // repository, update, and test packages in the bpkg configuration in the
     // current working directory. Optionally pass the config.import.* variable
     // override and/or set the environment variables for bpkg processes.
     // Return true if all operations for all packages succeed.
@@ -997,8 +1135,12 @@ build (size_t argc, const char* argv[])
     // installed dependents. Note that bpkg configures the dependent package
     // as a special dependency for the test package.
     //
-    auto test = [&pm, &trace, &wre, &step_args, &config_args, &env_args]
+    auto test = [&trace, &wre,
+                 &step_args, &config_args, &env_args,
+                 &pm,
+                 &redist]
                 (operation_result& r,
+                 const dir_path& dist_root,
                  bool sys_dep,
                  const char* import = nullptr,
                  const small_vector<string, 1>& envvars = {})
@@ -1020,6 +1162,7 @@ build (size_t argc, const char* argv[])
           "-v",
           "build",
           "--configure-only",
+          "--checkout-root", dist_root,
           "--yes",
           step_args (env_args,    step_id::bpkg_configure_build),
           step_args (config_args, step_id::bpkg_configure_build),
@@ -1031,6 +1174,37 @@ build (size_t argc, const char* argv[])
         if (!r.status)
           return false;
 
+        // Note that re-distributing the test package is a bit tricky since we
+        // don't know its version and so cannot deduce its source directory
+        // name easily. We could potentially run the bpkg-status command after
+        // the package is configured and parse the output to obtain the
+        // version. Let's, however, keep it simple and find the source
+        // directory using the package directory name pattern.
+        //
+        if (exists (dist_root))
+        try
+        {
+          dir_path pkg_dir;
+
+          path_search (dir_path (pkg + "-*/"),
+                       [&pkg_dir] (path&& pe, const string&, bool interm)
+                       {
+                         if (!interm)
+                           pkg_dir = path_cast<dir_path> (move (pe));
+
+                         return interm;
+                       },
+                       dist_root);
+
+          if (!pkg_dir.empty () &&
+              !redist (r, dist_root, pkg_dir, import, envvars))
+            return false;
+        }
+        catch (const system_error& e)
+        {
+          fail << "unable to scan directory " << dist_root << ": " << e;
+        }
+
         // Update.
         //
         // bpkg update <env-config-args> <config-args> <package-name>
@@ -1112,7 +1286,10 @@ build (size_t argc, const char* argv[])
       // package (except for the build system module).
       //
       if (external_tests &&
-          !test (r, module, bootstrap ? module_import.c_str () : nullptr))
+          !test (r,
+                 dist_root,
+                 module,
+                 bootstrap ? module_import.c_str () : nullptr))
         break;
 
       rm.status |= r.status;
@@ -1368,7 +1545,11 @@ build (size_t argc, const char* argv[])
 
         // Build/test.
         //
-        if (!test (r, true /* sys_dep */, nullptr /* import */, envvars))
+        if (!test (r,
+                   rwd / dir_path ("dist-installed"),
+                   true    /* sys_dep */,
+                   nullptr /* import */,
+                   envvars))
           break;
       }
 
diff --git a/doc/manual.cli b/doc/manual.cli
index 85ad89a..c5781ae 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -932,6 +932,11 @@ bpkg -v update <package-name>
 bpkg -v test <package-name>
 \
 
+If a primary or test package comes from a version control-based repository,
+then its \c{dist} meta-operation is also tested as a part of the
+\c{bpkg.configure.build} step by re-distributing the source directory in the
+load distribution mode after configuration.
+
 As an example, the following POSIX shell script can be used to setup the
 environment for building C and C++ packages with GCC 9 on most Linux
 distributions.
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 9833491..fe76c13 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -57,6 +57,14 @@ rep_type = pkg
 rfp = yes
 
 #\
+pkg = hello
+ver = 1.0.0+6
+rep_url = "https://git.build2.org/hello/hello.git"
+rep_type = git
+rfp = yes
+#\
+
+#\
 pkg = libstudxml
 ver = 1.1.0-b.7.20190619110825.0d1dcfe61f15
 rep_url = https://stage.build2.org/1
@@ -65,23 +73,28 @@ rfp = yes
 #\
 
 #\
-pkg = hello
-ver = 0.1.0-a.0.20191025174646.0150b7eb45af
-rep_url = "https://github.com/karen-arutyunov/hello.git#master"
+# To make sure that the test-installed phase succeeds use the build2 driver
+# installed into ~/install/bin.
+#
+pkg = libbuild2-hello
+ver = 0.1.0-a.0.20200810114133.0dba3a3bd624
+rep_url = "https://github.com/build2/libbuild2-hello.git#master"
 rep_type = git
+#rep_url = https://stage.build2.org/1
+#rep_type = pkg
 rfp = yes
 #\
 
 #\
-# To make sure that the test-installed phase succeeds use the build2 driver
-# installed into ~/install/bin.
+# Use the build2 driver installed into ~/install/bin (see above).
 #
-pkg = libbuild2-hello
-ver = 0.1.0-a.0.20200810114133.0dba3a3bd624
-#rep_url = "https://github.com/build2/libbuild2-hello.git#master"
-#rep_type = git
-rep_url = https://stage.build2.org/1
-rep_type = pkg
+pkg = libbuild2-kconfig
+ver = 0.1.0-a.0.20200930082223.f97ea0ff51a6
+rep_url = "https://github.com/build2/libbuild2-kconfig.git#master"
+rep_type = git
+#ver = 0.1.0-a.0.20200910053253.a71aa3f3938b
+#rep_url = https://stage.build2.org/1
+#rep_type = pkg
 rfp = yes
 #\
 
@@ -95,9 +108,11 @@ rfp = yes
 
 #\
 pkg = cli
-ver = 1.2.0-b.7.20200912105851.511d78c03656
-rep_url = https://stage.build2.org/1
-rep_type = pkg
+ver = 1.2.0-b.7.20200803080502.02dac36b07e8
+rep_url = "https://git.codesynthesis.com/cli/cli.git#adhoc-recipe"
+rep_type = git
+#rep_url = https://stage.build2.org/1
+#rep_type = pkg
 rfp = yes
 #\
 
@@ -163,6 +178,9 @@ a = $0
   chmod ugo+x $env;
   sleep $wait;
   $w --verbose 3 --startup --tftp-host $tftp --environments $~ \
-    &?build-module/*** &build/*** &?build-installed/*** \
-    &?build-installed-bpkg/*** &task.manifest 2>|
+    &?build-module/*** &build/*** \
+    &?build-installed/*** &?build-installed-bpkg/*** \
+    &?dist/*** &?redist/***  \
+    &?dist-installed/*** &?redist-installed/*** \
+    &task.manifest 2>|
 }