Add support for tests, examples, and benchmarks package manifest values

1 files changed, 466 insertions, 136 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index d263a5c..d97dff0 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -118,7 +118,7 @@ run_cmd (tracer& t,
 
     process pr (
       process_start_callback (cmdc,
-                              fdnull (), // Never reads from stdout.
+                              fdnull (), // Never reads from stdin.
                               2,         // 1>&2
                               pipe,
                               pe,
@@ -195,9 +195,10 @@ run_cmd (tracer& t,
   }
 }
 
-template <typename... A>
+template <typename V, typename... A>
 static result_status
-run_bpkg (tracer& t,
+run_bpkg (const V& envvars,
+          tracer& t,
           string& log, const regexes& warn_detect,
           const char* verbosity,
           const string& cmd, A&&... a)
@@ -205,14 +206,30 @@ run_bpkg (tracer& t,
   return run_cmd (t,
                   log, warn_detect,
                   "bpkg " + cmd,
-                  "bpkg", verbosity, cmd, forward<A> (a)...);
+                  process_env ("bpkg", envvars),
+                  verbosity, cmd, forward<A> (a)...);
+}
+
+template <typename... A>
+static result_status
+run_bpkg (tracer& t,
+          string& log, const regexes& warn_detect,
+          const char* verbosity,
+          const string& cmd, A&&... a)
+{
+  const char* const* envvars (nullptr);
+
+  return run_bpkg (envvars,
+                   t,
+                   log, warn_detect,
+                   verbosity, cmd, forward<A> (a)...);
 }
 
 template <typename V, typename... A>
 static result_status
-run_b (tracer& t,
+run_b (const V& envvars,
+       tracer& t,
        string& log, const regexes& warn_detect,
-       const V& envvars,
        const char* verbosity,
        const strings& buildspecs, A&&... a)
 {
@@ -234,9 +251,9 @@ run_b (tracer& t,
 
 template <typename V, typename... A>
 static result_status
-run_b (tracer& t,
+run_b (const V& envvars,
+       tracer& t,
        string& log, const regexes& warn_detect,
-       const V& envvars,
        const char* verbosity,
        const string& buildspec, A&&... a)
 {
@@ -255,16 +272,77 @@ run_b (tracer& t,
        const string& buildspec, A&&... a)
 {
   const char* const* envvars (nullptr);
-  return run_b (t,
+  return run_b (envvars,
+                t,
                 log, warn_detect,
-                envvars,
-                verbosity,
-                buildspec, forward<A> (a)...);
+                verbosity, buildspec, forward<A> (a)...);
+}
+
+// Run a program and return its output as a string if it prints exactly one
+// line and exits with zero code and fail otherwise.
+//
+template <typename... A>
+static string
+cmd_line (tracer& t, const string& name, const process_env& pe, A&&... a)
+{
+  try
+  {
+    fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.
+
+    process pr (
+      process_start_callback (t,
+                              fdnull () /* stdin */,
+                              pipe,     /* stdout */
+                              2         /* stderr */,
+                              pe,
+                              forward<A> (a)...));
+
+    pipe.out.close ();
+
+    ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+    optional<string> r;
+    if (is.peek () != ifdstream::traits_type::eof ())
+    {
+      string s;
+      getline (is, s);
+
+      if (!is.eof () && is.peek () == ifdstream::traits_type::eof ())
+      {
+        r = move (s);
+
+        if (verb >= 3)
+          t << *r;
+      }
+    }
+
+    is.close ();
+
+    if (pr.wait ())
+    {
+      if (r)
+        return move (*r);
+
+      fail << "invalid " << name << " output";
+    }
+
+    fail << name << ' ' << *pr.exit << endf;
+  }
+  catch (const process_error& e)
+  {
+    fail << "unable to execute " << name << ": " << e << endf;
+  }
+  catch (const io_error& e)
+  {
+    fail << "unable to read " << name << " stdout: " << e << endf;
+  }
 }
 
 static int bbot::
 build (size_t argc, const char* argv[])
 {
+  using namespace bpkg;
+
   using string_parser::unquote;
 
   tracer trace ("build");
@@ -337,6 +415,7 @@ build (size_t argc, const char* argv[])
       b_test_installed_create,
       b_test_installed_configure,
       b_test_installed_test,
+      bpkg_test_installed_create,
       bpkg_uninstall_uninstall
     };
 
@@ -351,6 +430,7 @@ build (size_t argc, const char* argv[])
       "b.test-installed.create",
       "b.test-installed.configure",
       "b.test-installed.test",
+      "bpkg.test-installed.create",
       "bpkg.uninstall.uninstall"};
 
     // Split the argument into prefix (empty if not present) and unquoted
@@ -385,7 +465,8 @@ build (size_t argc, const char* argv[])
       else
       {
         args.emplace ("bpkg.configure.create", a.second);
-        args.emplace ("b.test-installed.create", move (a.second));
+        args.emplace ("b.test-installed.create", a.second);
+        args.emplace ("bpkg.test-installed.create", move (a.second));
       }
     };
 
@@ -476,6 +557,26 @@ build (size_t argc, const char* argv[])
     //
     dir_path install_root;
 
+    // bpkg-rep-fetch trust options.
+    //
+    cstrings trust_ops;
+    {
+      const char* t ("--trust-no");
+
+      for (const string& fp: tm.trust)
+      {
+        if (fp == "yes")
+          t = "--trust-yes";
+        else
+        {
+          trust_ops.push_back ("--trust");
+          trust_ops.push_back (fp.c_str ());
+        }
+      }
+
+      trust_ops.push_back (t);
+    }
+
     // Configuration directory name.
     //
     dir_path build_dir ("build");
@@ -554,28 +655,13 @@ build (size_t argc, const char* argv[])
       //
       // bpkg.configure.fetch
       //
-      string t ("--trust-no");
-
-      cstrings ts;
-      for (const string& fp: tm.trust)
-      {
-        if (fp == "yes")
-          t = "--trust-yes";
-        else
-        {
-          ts.push_back ("--trust");
-          ts.push_back (fp.c_str ());
-        }
-      }
-
       r.status |= run_bpkg (
         trace, r.log, wre,
         "-v",
         "fetch",
         step_args (config_args, step_id::bpkg_configure_fetch),
         step_args (env_args,    step_id::bpkg_configure_fetch),
-        ts,
-        t);
+        trust_ops);
 
       if (!r.status)
         break;
@@ -588,11 +674,11 @@ build (size_t argc, const char* argv[])
       // Specify the revision explicitly not to end up with a race condition
       // building the latest revision rather than the zero revision.
       //
-      bpkg::version v (tm.version.epoch,
-                       tm.version.upstream,
-                       tm.version.release,
-                       tm.version.effective_revision (),
-                       tm.version.iteration);
+      version v (tm.version.epoch,
+                 tm.version.upstream,
+                 tm.version.release,
+                 tm.version.effective_revision (),
+                 tm.version.iteration);
 
       r.status |= run_bpkg (
         trace, r.log, wre,
@@ -657,39 +743,206 @@ build (size_t argc, const char* argv[])
       fail << "unable to query project " << prj_dir << " info: " << e;
     }
 
-    // Test the package if the test operation is supported by the project.
+    // Run the package internal tests if the test operation is supported by
+    // the project.
+    //
+    bool internal_tests (find (prj.operations.begin (),
+                               prj.operations.end (),
+                               "test") != prj.operations.end ());
+
+    // Parse the package manifest.
+    //
+    // Note that being unable to parse the package manifest is likely to be an
+    // infrastructural problem, given that the package has been successfully
+    // configured.
+    //
+    path mf (prj_dir / "manifest");
+    package_manifest pm (parse_manifest<package_manifest> (mf, "package"));
+
+    // Run the package external tests if any of the tests, examples, or
+    // benchmarks manifest values are specified.
+    //
+    // Note that we assume that these packages belong to the dependent
+    // package's repository or its complement repositories, recursively. Thus,
+    // we test them in the configuration used to build the dependent package.
+    //
+    bool external_tests (!pm.tests.empty ()    ||
+                         !pm.examples.empty () ||
+                         !pm.benchmarks.empty ());
+
+    // Configure, update, and test packages in the bpkg configuration in the
+    // current working directory. Optionally set the environment variables for
+    // bpkg processes. Return true if all operations for all packages succeed.
     //
-    if (find (prj.operations.begin (), prj.operations.end (), "test") !=
-        prj.operations.end ())
+    auto test = [&trace, &wre, &step_args, &config_args, &env_args]
+                (const small_vector<dependency, 1>& pkgs,
+                 operation_result& r,
+                 const small_vector<string, 1>& envvars = {})
+    {
+      for (const dependency& d: pkgs)
+      {
+        const string& pkg (d.name.string ());
+
+        // Configure.
+        //
+        // bpkg build --configure-only <config-args> <env-config-args>
+        //            '<package-name>[ <version-constraint>]'
+        //
+        // bpkg.configure.build
+        //
+        r.status |= run_bpkg (
+          envvars,
+          trace, r.log, wre,
+          "-v",
+          "build",
+          "--configure-only",
+          "--yes",
+          step_args (config_args, step_id::bpkg_configure_build),
+          step_args (env_args,    step_id::bpkg_configure_build),
+          "--",
+          d.string ());
+
+        if (!r.status)
+          return false;
+
+        // Update.
+        //
+        // bpkg update <config-args> <env-config-args> <package-name>
+        //
+        // bpkg.update.update
+        //
+        r.status |= run_bpkg (
+          envvars,
+          trace, r.log, wre,
+          "-v",
+          "update",
+          step_args (config_args, step_id::bpkg_update_update),
+          step_args (env_args,    step_id::bpkg_update_update),
+          pkg);
+
+        if (!r.status)
+          return false;
+
+        // Test.
+        //
+        // Note that we assume that the package supports the test operation
+        // since this is its main purpose.
+        //
+        // Lets change the working directory to the package directory to
+        // help ported to build2 third-party packages a bit. Note that this
+        // is not uncommon for them to expect that tests should run in the
+        // project root directory.
+        //
+        // To determine the package directory name we need to obtain the
+        // configured package version. Parsing the bpkg-pkg-status output
+        // seems to be the easiest way to accomplish this.
+        //
+        version ver;
+        {
+          string status (cmd_line (trace,
+                                   "bpkg-pkg-status",
+                                   "bpkg", "status", "--no-hold", pkg));
+
+          // Get the version offset in the status line.
+          //
+          string prefix (pkg + " configured ");
+          size_t p (prefix.size ());
+
+          // Make sure the status line prefix matches our expectations.
+          //
+          if (status.compare (0, p, prefix) != 0)
+            fail << "unexpected " << pkg << " status: " << status;
+
+          // Extract the package version from the status line.
+          //
+          size_t n (status.find (' ', p));
+          string v (status, p, n != string::npos ? n - p : n);
+
+          try
+          {
+            ver = version (v);
+          }
+          catch (const invalid_argument& e)
+          {
+            fail << "invalid " << pkg << " version '" << v << "' in "
+                 << "package status: " << e <<
+              info << "status: " << status;
+          }
+        }
+
+        // Finally, change the working directory to the package directory
+        // and run the tests.
+        //
+        dir_path prj_dir (pkg + '-' + ver.string ());
+        dir_path owd (change_wd (trace, &r.log, prj_dir));
+
+        // bpkg test <config-args> <env-config-args> <package-name>
+        //
+        // bpkg.test.test
+        //
+        r.status |= run_bpkg (
+          envvars,
+          trace, r.log, wre,
+          "-v",
+          "test",
+          "-d", "..",
+          step_args (config_args, step_id::bpkg_test_test),
+          step_args (env_args,    step_id::bpkg_test_test),
+          pkg);
+
+        if (!r.status)
+          return false;
+
+        change_wd (trace, &r.log, owd);
+      }
+
+      return true;
+    };
+
+    if (internal_tests || external_tests)
     {
       operation_result& r (add_result ("test"));
 
-      // Lets change the working directory to the package directory to help
-      // ported to build2 third-party packages a bit. Note that this is not
-      // uncommon for them to expect that tests should run in the project root
-      // directory.
+      // Run internal tests.
       //
-      dir_path rwd (change_wd (trace, &r.log, prj_dir));
+      if (internal_tests)
+      {
+        // Lets change the working directory to the package directory to help
+        // ported to build2 third-party packages a bit (see above for
+        // details).
+        //
+        dir_path owd (change_wd (trace, &r.log, prj_dir));
 
-      // bpkg test <config-args> <env-config-args> <package-name>
-      //
-      // bpkg.test.test
-      //
-      r.status |= run_bpkg (
-        trace, r.log, wre,
-        "-v",
-        "test",
-        "-d", "..",
-        step_args (config_args, step_id::bpkg_test_test),
-        step_args (env_args,    step_id::bpkg_test_test),
-        tm.name.string ());
+        // bpkg test <config-args> <env-config-args> <package-name>
+        //
+        // bpkg.test.test
+        //
+        r.status |= run_bpkg (
+          trace, r.log, wre,
+          "-v",
+          "test",
+          "-d", "..",
+          step_args (config_args, step_id::bpkg_test_test),
+          step_args (env_args,    step_id::bpkg_test_test),
+          tm.name.string ());
 
-      if (!r.status)
-        break;
+        if (!r.status)
+          break;
 
-      rm.status |= r.status;
+        change_wd (trace, &r.log, owd);
+      }
 
-      change_wd (trace, &r.log, rwd);
+      // Run external tests.
+      //
+      if (external_tests)
+      {
+        if (!test (pm.tests, r)    ||
+            !test (pm.examples, r) ||
+            !test (pm.benchmarks, r))
+          break;
+      }
+
+      rm.status |= r.status;
     }
 
     // Install the package, optionally test the installation and uninstall
@@ -710,7 +963,11 @@ build (size_t argc, const char* argv[])
     //    configure, build, and test them out of the source tree against the
     //    installed package.
     //
-    // 3. Uninstall the package.
+    // 3. If any of the tests, examples, or benchmarks packages are specified,
+    //    then configure, build, and test them in a separate bpkg
+    //    configuration against the installed package.
+    //
+    // 4. Uninstall the package.
     //
     // Install.
     //
@@ -737,6 +994,26 @@ build (size_t argc, const char* argv[])
 
     // Test installed.
     //
+    // Make sure that the installed package executables are properly imported
+    // when configuring and running tests.
+    //
+    small_vector<string, 1> envvars;
+    {
+      // Note that we add the $config.install.root/bin directory at the
+      // beginning of the PATH environment variable value, so the installed
+      // executables are found first.
+      //
+      string paths ("PATH=" + (install_root / "bin").string ());
+
+      if (optional<string> s = getenv ("PATH"))
+      {
+        paths += path::traits_type::path_separator;
+        paths += *s;
+      }
+
+      envvars.push_back (move (paths));
+    }
+
     // Collect the "testable" subprojects.
     //
     dir_paths subprj_dirs;
@@ -770,114 +1047,167 @@ build (size_t argc, const char* argv[])
     // If there are any "testable" subprojects, then configure them
     // (sequentially) and test/build in parallel afterwards.
     //
-    if (!subprj_dirs.empty ())
+    internal_tests = !subprj_dirs.empty ();
+
+    if (internal_tests || external_tests)
     {
       operation_result& r (add_result ("test-installed"));
 
       change_wd (trace, &r.log, rwd);
 
-      string mods;  // build2 create meta-operation parameters.
-
-      for (const string& m:
-             step_args (modules, step_id::b_test_installed_create))
+      // Run internal tests.
+      //
+      if (internal_tests)
       {
-        mods += mods.empty () ? ", " : " ";
-        mods += m;
-      }
+        string mods; // build2 create meta-operation parameters.
 
-      // b create(<dir>, <env-modules>) <config-args> <env-config-args>
-      //
-      // b.test-installed.create
-      //
-      // Amalgamation directory that will contain configuration subdirectory
-      // for package tests out of source tree build.
-      //
-      dir_path out_dir ("build-installed");
+        for (const string& m:
+               step_args (modules, step_id::b_test_installed_create))
+        {
+          mods += mods.empty () ? ", " : " ";
+          mods += m;
+        }
 
-      r.status |= run_b (
-        trace, r.log, wre,
-        "-V",
-        "create('" + out_dir.representation () + "'" + mods + ")",
-        step_args (config_args, step_id::b_test_installed_create),
-        step_args (env_args,    step_id::b_test_installed_create));
+        // b create(<dir>, <env-modules>) <config-args> <env-config-args>
+        //
+        // b.test-installed.create
+        //
+        // Amalgamation directory that will contain configuration subdirectory
+        // for package tests out of source tree build.
+        //
+        dir_path out_dir ("build-installed");
 
-      if (!r.status)
-        break;
+        r.status |= run_b (
+          trace, r.log, wre,
+          "-V",
+          "create('" + out_dir.representation () + "'" + mods + ")",
+          step_args (config_args, step_id::b_test_installed_create),
+          step_args (env_args,    step_id::b_test_installed_create));
 
-      rm.status |= r.status;
+        if (!r.status)
+          break;
 
-      // Make sure that the installed package executables are properly imported
-      // when configuring and running tests.
-      //
-      // Note that we add the $config.install.root/bin directory at the
-      // beginning of the PATH environment variable value, so the installed
-      // executables are found first.
-      //
-      string paths ("PATH=" + (install_root / "bin").string ());
+        // Configure subprojects and create buildspecs for their testing.
+        //
+        strings test_specs;
+        for (const dir_path& d: subprj_dirs)
+        {
+          // b configure(<subprj-src-dir>@<subprj-out-dir>) <config-args>
+          //                                                <env-config-args>
+          //
+          // b.test-installed.configure
+          //
+          dir_path subprj_src_dir (build_dir / prj_dir / d);
+          dir_path subprj_out_dir (out_dir / d);
+
+          r.status |= run_b (
+            envvars,
+            trace, r.log, wre,
+            "-v",
+            "configure('" +
+            subprj_src_dir.representation () + "'@'" +
+            subprj_out_dir.representation () + "')",
+            step_args (config_args, step_id::b_test_installed_configure),
+            step_args (env_args,    step_id::b_test_installed_configure));
+
+          if (!r.status)
+            break;
 
-      if (optional<string> s = getenv ("PATH"))
-      {
-        paths += path::traits_type::path_separator;
-        paths += *s;
-      }
+          test_specs.push_back (
+            "test('" + subprj_out_dir.representation () + "')");
+        }
 
-      small_vector<string, 1> envvars {move (paths)};
+        if (!r.status)
+          break;
+
+        // Build/test subprojects.
+        //
+        // b test(<subprj-out-dir>)... <config-args> <env-config-args>
+        //
+        // b.test-installed.test
+        //
+        r.status |= run_b (
+          envvars,
+          trace, r.log, wre,
+          "-v",
+          test_specs,
+          step_args (config_args, step_id::b_test_installed_test),
+          step_args (env_args,    step_id::b_test_installed_test));
+
+        if (!r.status)
+          break;
+      }
 
-      // Configure subprojects and create buildspecs for their testing.
+      // Run external tests.
       //
-      strings test_specs;
-      for (const dir_path& d: subprj_dirs)
+      if (external_tests)
       {
-        // b configure(<subprj-src-dir>@<subprj-out-dir>) <config-args>
-        //                                                <env-config-args>
+        // Configure.
         //
-        // b.test-installed.configure
+        // bpkg create <env-modules> <config-args> <env-config-args>
         //
-        dir_path subprj_src_dir (build_dir / prj_dir / d);
-        dir_path subprj_out_dir (out_dir / d);
+        // bpkg.test-installed.create
+        //
+        dir_path config_dir ("build-installed-bpkg");
 
-        r.status |= run_b (
+        r.status |= run_bpkg (
           trace, r.log, wre,
-          envvars,
-          "-v",
-          "configure('" +
-          subprj_src_dir.representation () + "'@'" +
-          subprj_out_dir.representation () + "')",
-          step_args (config_args, step_id::b_test_installed_configure),
-          step_args (env_args,    step_id::b_test_installed_configure));
+          "-V",
+          "create",
+          "-d", config_dir.string (),
+          "--wipe",
+          step_args (modules,     step_id::bpkg_test_installed_create),
+          step_args (config_args, step_id::bpkg_test_installed_create),
+          step_args (env_args,    step_id::bpkg_test_installed_create));
 
         if (!r.status)
           break;
 
-        test_specs.push_back (
-          "test('" + subprj_out_dir.representation () + "')");
-      }
+        dir_path owd (change_wd (trace, &r.log, config_dir));
 
-      if (!r.status)
-        break;
+        // bpkg add <config-args> <env-config-args> <repository-url>
+        //
+        // bpkg.configure.add
+        //
+        r.status |= run_bpkg (
+          trace, r.log, wre,
+          "-v",
+          "add",
+          step_args (config_args, step_id::bpkg_configure_add),
+          step_args (env_args,    step_id::bpkg_configure_add),
+          tm.repository.string ());
 
-      rm.status |= r.status;
+        if (!r.status)
+          break;
 
-      // Build/test subprojects.
-      //
-      // b test(<subprj-out-dir>)... <config-args> <env-config-args>
-      //
-      // b.test-installed.test
-      //
-      r.status |= run_b (
-        trace, r.log, wre,
-        envvars,
-        "-v",
-        test_specs,
-        step_args (config_args, step_id::b_test_installed_test),
-        step_args (env_args,    step_id::b_test_installed_test));
+        // bpkg fetch <config-args> <env-config-args> <trust-options>
+        //
+        // bpkg.configure.fetch
+        //
+        r.status |= run_bpkg (
+          trace, r.log, wre,
+          "-v",
+          "fetch",
+          step_args (config_args, step_id::bpkg_configure_fetch),
+          step_args (env_args,    step_id::bpkg_configure_fetch),
+          trust_ops);
 
-      if (!r.status)
-        break;
+        if (!r.status)
+          break;
 
-      rm.status |= r.status;
+        // Build/test.
+        //
+        if (!test (pm.tests, r, envvars)    ||
+            !test (pm.examples, r, envvars) ||
+            !test (pm.benchmarks, r, envvars))
+          break;
+
+        change_wd (trace, &r.log, owd);
+      }
 
       change_wd (trace, &r.log, build_dir);
+
+      rm.status |= r.status;
     }
 
     // Uninstall.