Use `b info` command to check if package project/subprojects can be tested

3 files changed, 128 insertions, 45 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 6a8eda0..fd6232c 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -10,10 +10,12 @@
 
 #include <map>
 #include <regex>
-#include <cstring>  // strchr()
+#include <cstring>   // strchr()
 #include <sstream>
 #include <iostream>
+#include <algorithm> // find()
 
+#include <libbutl/b.mxx>
 #include <libbutl/pager.mxx>
 #include <libbutl/filesystem.mxx>
 #include <libbutl/string-parser.mxx>
@@ -210,6 +212,29 @@ static result_status
 run_b (tracer& t,
        string& log, const regexes& warn_detect,
        const V& envvars,
+       const strings& buildspecs, A&&... a)
+{
+  string name ("b");
+  for (const string& s: buildspecs)
+  {
+    if (!name.empty ())
+      name += ' ';
+
+    name += s;
+  }
+
+  return run_cmd (t,
+                  log, warn_detect,
+                  name,
+                  process_env ("b", envvars),
+                  "-v", buildspecs, forward<A> (a)...);
+}
+
+template <typename V, typename... A>
+static result_status
+run_b (tracer& t,
+       string& log, const regexes& warn_detect,
+       const V& envvars,
        const string& buildspec, A&&... a)
 {
   return run_cmd (t,
@@ -577,8 +602,33 @@ build (size_t argc, const char* argv[])
       rm.status |= r.status;
     }
 
-    // Test.
+    // Query the package's build system information with `b info`.
+    //
+    dir_path prj_dir (tm.name.string () + '-' + tm.version.string ());
+    b_project_info prj;
+
+    // Note that `b info` diagnostics will not be copied into any of the build
+    // logs. This seems to be fine, as this is likely to be an infrastructural
+    // problem, given that the project distribution has been successfully
+    // created. It's actually not quite clear which log this diagnostics could
+    // go into.
+    //
+    try
+    {
+      prj = b_info (prj_dir, verb, trace);
+    }
+    catch (const b_error& e)
+    {
+      if (e.normal ())
+        throw failed (); // Assume the build2 process issued diagnostics.
+
+      fail << "unable to query project " << prj_dir << " info: " << e;
+    }
+
+    // Test the package if the test operation is supported by the project.
     //
+    if (find (prj.operations.begin (), prj.operations.end (), "test") !=
+        prj.operations.end ())
     {
       operation_result& r (add_result ("test"));
 
@@ -587,8 +637,7 @@ build (size_t argc, const char* argv[])
       // uncommon for them to expect that tests should run in the project root
       // directory.
       //
-      dir_path d (tm.name.string () + '-' + tm.version.string ());
-      dir_path rwd (change_wd (trace, &r.log, d));
+      dir_path rwd (change_wd (trace, &r.log, prj_dir));
 
       // bpkg test <config-args> <env-config-args> <package-name>
       //
@@ -624,9 +673,9 @@ build (size_t argc, const char* argv[])
     //
     // 1. Install the package.
     //
-    // 2. If the package project has the 'tests' subdirectory that is a
-    //    subproject, then configure, build and test it out of the source tree
-    //    against the installed package.
+    // 2. If the package has subprojects that support the test operation, then
+    //    configure, build, and test them out of the source tree against the
+    //    installed package.
     //
     // 3. Uninstall the package.
     //
@@ -654,15 +703,40 @@ build (size_t argc, const char* argv[])
 
     // Test installed.
     //
-    // The package tests subdirectory path (may not exist).
+    // Collect the "testable" subprojects.
     //
-    dir_path tests_dir (tm.name.string () + "-" + tm.version.string ());
-    tests_dir /= "tests";
+    dir_paths subprj_dirs;
+    for (const b_project_info::subproject& sp: prj.subprojects)
+    {
+      // Retrieve the subproject information similar to how we've done it for
+      // the package.
+      //
+      dir_path sd (prj_dir / sp.path);
+
+      // Note that `b info` diagnostics will not be logged (see above for
+      // details).
+      //
+      try
+      {
+        b_project_info si (b_info (sd, verb, trace));
+
+        const strings& ops (si.operations);
+        if (find (ops.begin (), ops.end (), "test") != ops.end ())
+          subprj_dirs.push_back (sp.path);
+      }
+      catch (const b_error& e)
+      {
+        if (e.normal ())
+          throw failed (); // Assume the build2 process issued diagnostics.
 
-    // We will consider the tests subdirectory to be a subproject if it
-    // contains a build2 bootstrap file.
+        fail << "unable to query subproject " << sd << " info: " << e;
+      }
+    }
+
+    // If there are any "testable" subprojects, then configure them
+    // (sequentially) and test/build in parallel afterwards.
     //
-    if (file_exists (tests_dir / path ("build/bootstrap.build")))
+    if (!subprj_dirs.empty ())
     {
       operation_result& r (add_result ("test-installed"));
 
@@ -714,35 +788,50 @@ build (size_t argc, const char* argv[])
 
       small_vector<string, 1> envvars {move (paths)};
 
-      // b configure(<tests-dir>@<tests-out-dir>) <config-args>
-      //                                          <env-config-args>
-      //
-      // b.test-installed.configure
+      // Configure subprojects and create buildspecs for their testing.
       //
-      dir_path tests_out_dir (out_dir / dir_path ("tests"));
+      strings test_specs;
+      for (const dir_path& d: subprj_dirs)
+      {
+        // b configure(<subprj-src-dir>@<subprj-out-dir>) <config-args>
+        //                                                <env-config-args>
+        //
+        // b.test-installed.configure
+        //
+        dir_path subprj_src_dir (build_dir / prj_dir / d);
+        dir_path subprj_out_dir (out_dir / d);
+
+        r.status |= run_b (
+          trace, r.log, wre,
+          envvars,
+          "configure(" +
+          subprj_src_dir.representation ()  + '@' +
+          subprj_out_dir.representation () + ")",
+          step_args (config_args, step_id::b_test_installed_configure),
+          step_args (env_args,    step_id::b_test_installed_configure));
+
+        if (!r.status)
+          break;
 
-      r.status |= run_b (
-        trace, r.log, wre,
-        envvars,
-        "configure(" +
-        (build_dir / tests_dir).representation ()  + '@' +
-        tests_out_dir.representation () + ")",
-        step_args (config_args, step_id::b_test_installed_configure),
-        step_args (env_args,    step_id::b_test_installed_configure));
+        test_specs.push_back (
+          "test(" + subprj_out_dir.representation () + ')');
+      }
 
       if (!r.status)
         break;
 
       rm.status |= r.status;
 
-      // b test(<tests-out-dir>) <config-args> <env-config-args>
+      // Build/test subprojects.
+      //
+      // b test(<subprj-out-dir>)... <config-args> <env-config-args>
       //
       // b.test-installed.test
       //
       r.status |= run_b (
         trace, r.log, wre,
         envvars,
-        "test(" + tests_out_dir.representation () + ')',
+        test_specs,
         step_args (config_args, step_id::b_test_installed_test),
         step_args (env_args,    step_id::b_test_installed_test));
 
diff --git a/doc/manual.cli b/doc/manual.cli
index c0d9b35..6865f1c 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -778,6 +778,8 @@ bpkg -v build --yes --configure-only <package-name>/<package-version>
 #
 bpkg -v update <package-name>
 
+# if the test operation is supported by the package:
+#
 # bpkg.test.test
 #
 bpkg -v test <package-name>
@@ -789,7 +791,7 @@ bpkg -v test <package-name>
   #
   bpkg -v install <package-name>
 
-  # if package contains tests/ subproject:
+  # if the package contains subprojects that support the test operation:
   #
   {
     # b.test-installed.create
@@ -950,10 +952,10 @@ linux*-gcc_8 linux-gcc_8 x86_64-linux-gnu \"all gcc-8+:gcc-7+\"
 
 If the \c{<config-arg>} list contains the \c{config.install.root} variable
 that applies to the \c{bpkg.configure.create} step, then in addition to
-building and running tests, the \c{bbot} worker will also test installing and
-uninstalling each package. Furthermore, if the package contains the \c{tests}
-subdirectory that is a subproject, then the worker will additionally build and
-run tests against the installation.
+building and possibly running tests, the \c{bbot} worker will also test
+installing and uninstalling each package. Furthermore, if the package contains
+subprojects that support the test operation, then the worker will additionally
+build such subprojects against the installation and run their tests.
 
 Two types of installations can be tested: \i{system} and \i{private}. A system
 installation uses a well-known location, such as \c{/usr} or \c{/usr/local},
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 7d7d7d9..3827318 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -49,7 +49,7 @@ wait=1s
 controller = --fake-request ../task --dump-result
 
 pkg = libhello
-ver = 1.0.0
+ver = 1.0.0+1
 #rep_url = https://build2.org/pkg/1/hello/stable
 #rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
 rep_url = https://stage.build2.org/1
@@ -57,19 +57,11 @@ rep_type = pkg
 rfp = yes
 
 #\
-pkg = bpkg
-ver = 0.6.0-a.0.1499870184.53fff46c5e5fd4df
+pkg = libstudxml
+ver = 1.1.0-b.4.20181203094958.ed267da35ccd
 rep_url = https://stage.build2.org/1
 rep_type = pkg
-rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65
-#\
-
-#\
-pkg = build2
-ver = 0.6.0-a.0.1500478402.30578be48463b93f
-rep_url = https://stage.build2.org/1
-rep_type = pkg
-rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65
+rfp = yes
 #\
 
 # Note that we also need to make sure that the installed package libraries are