Add more step ids to worker script

Also rename bpkg.test-installed.{configure.build,update,test} to
bpkg.test-separate-installed.*.

3 files changed, 157 insertions, 90 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index ef86dd4..bd880ae 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -511,12 +511,21 @@ build (size_t argc, const char* argv[])
     //
     enum class step_id
     {
+      bpkg_module_create,
+      bpkg_module_configure_add,
+      bpkg_module_configure_fetch,
+      bpkg_module_configure_build,
+      bpkg_module_update,
+      bpkg_module_test,
       bpkg_create,
       bpkg_configure_add,
       bpkg_configure_fetch,
       bpkg_configure_build,
       bpkg_update,
       bpkg_test,
+      bpkg_test_separate_configure_build,
+      bpkg_test_separate_update,
+      bpkg_test_separate_test,
       bpkg_install,
       b_test_installed_create,
       b_test_installed_configure,
@@ -524,19 +533,28 @@ build (size_t argc, const char* argv[])
       bpkg_test_installed_create,
       bpkg_test_installed_configure_add,
       bpkg_test_installed_configure_fetch,
-      bpkg_test_installed_configure_build,
-      bpkg_test_installed_update,
-      bpkg_test_installed_test,
+      bpkg_test_separate_installed_configure_build,
+      bpkg_test_separate_installed_update,
+      bpkg_test_separate_installed_test,
       bpkg_uninstall
     };
 
     const strings step_id_str {
+      "bpkg.module.create",
+      "bpkg.module.configure.add",
+      "bpkg.module.configure.fetch",
+      "bpkg.module.configure.build",
+      "bpkg.module.update",
+      "bpkg.module.test",
       "bpkg.create",
       "bpkg.configure.add",
       "bpkg.configure.fetch",
       "bpkg.configure.build",
       "bpkg.update",
       "bpkg.test",
+      "bpkg.test-separate.configure.build",
+      "bpkg.test-separate.update",
+      "bpkg.test-separate.test",
       "bpkg.install",
       "b.test-installed.create",
       "b.test-installed.configure",
@@ -544,9 +562,9 @@ build (size_t argc, const char* argv[])
       "bpkg.test-installed.create",
       "bpkg.test-installed.configure.add",
       "bpkg.test-installed.configure.fetch",
-      "bpkg.test-installed.configure.build",
-      "bpkg.test-installed.update",
-      "bpkg.test-installed.test",
+      "bpkg.test-separate-installed.configure.build",
+      "bpkg.test-separate-installed.update",
+      "bpkg.test-separate-installed.test",
       "bpkg.uninstall"};
 
     // Split the argument into prefix (empty if not present) and unquoted
@@ -834,8 +852,9 @@ build (size_t argc, const char* argv[])
       return true;
     };
 
-    // The main phase.
+    // The module phase.
     //
+
     // If this is a build system module, perform a "pre-step" by building it
     // in a separate configuration reproducing the one used to build build2
     // itself. Note that the configuration and the environment options and
@@ -876,6 +895,8 @@ build (size_t argc, const char* argv[])
 
         // b create(<dir>) config.config.load=~build2
         //
+        // [bpkg.module.create]
+        //
         // Note also that we suppress warnings about unused config.* values,
         // such CLI configuration.
         //
@@ -910,14 +931,21 @@ build (size_t argc, const char* argv[])
 
         // bpkg add <env-config-args> <config-args> <repository-url>
         //
-        // bpkg.configure.add
+        // bpkg.module.configure.add (bpkg.configure.add)
         //
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-v",
           "add",
-          step_args (env_args,    step_id::bpkg_configure_add),
-          step_args (config_args, step_id::bpkg_configure_add),
+
+          step_args (env_args,
+                     step_id::bpkg_module_configure_add,
+                     step_id::bpkg_configure_add),
+
+          step_args (config_args,
+                     step_id::bpkg_module_configure_add,
+                     step_id::bpkg_configure_add),
+
           repo);
 
         if (!r.status)
@@ -925,14 +953,21 @@ build (size_t argc, const char* argv[])
 
         // bpkg fetch <env-config-args> <config-args> <trust-options>
         //
-        // bpkg.configure.fetch
+        // bpkg.module.configure.fetch (bpkg.configure.fetch)
         //
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-v",
           "fetch",
-          step_args (env_args,    step_id::bpkg_configure_fetch),
-          step_args (config_args, step_id::bpkg_configure_fetch),
+
+          step_args (env_args,
+                     step_id::bpkg_module_configure_fetch,
+                     step_id::bpkg_configure_fetch),
+
+          step_args (config_args,
+                     step_id::bpkg_module_configure_fetch,
+                     step_id::bpkg_configure_fetch),
+
           trust_ops);
 
         if (!r.status)
@@ -940,6 +975,8 @@ build (size_t argc, const char* argv[])
 
         // bpkg build --configure-only <package-name>/<package-version>
         //
+        // [bpkg.module.configure.build]
+        //
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-v",
@@ -995,6 +1032,8 @@ build (size_t argc, const char* argv[])
 
         // bpkg update <package-name>
         //
+        // [bpkg.module.update]
+        //
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-v",
@@ -1027,6 +1066,8 @@ build (size_t argc, const char* argv[])
         //
         // bpkg test <package-name>
         //
+        // [bpkg.module.test]
+        //
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-v",
@@ -1041,7 +1082,7 @@ build (size_t argc, const char* argv[])
       }
     }
 
-    // The "main" step.
+    // The main phase.
     //
 
     // Use the global override for modules that require bootstrap.
@@ -1188,7 +1229,9 @@ build (size_t argc, const char* argv[])
     bool internal_tests;
 
     if (module)
+    {
       internal_tests = false;
+    }
     else
     {
       prj = prj_info (pkg_dir, true /* ext_mods */, "project");
@@ -1233,10 +1276,11 @@ build (size_t argc, const char* argv[])
     // override and/or set the environment variables for bpkg processes.
     // Return true if all operations for all packages succeed.
     //
-    // Pass true as the installed argument to use the test installed phase
-    // step ids (bpkg.test-installed.*), falling back to the main phase step
-    // ids (bpkg.*) when no environment/configuration arguments are specified
-    // for them.
+    // Pass true as the installed argument to use the test separate installed
+    // phase step ids (bpkg.test-separate-installed.*) and the test separate
+    // phase step ids (bpkg.test-separate.*) otherwise. In both cases fall
+    // back to the main phase step ids (bpkg.*) when no environment/
+    // configuration arguments are specified for them.
     //
     // Pass true as the sys_dep argument to configure the dependent package as
     // a system dependency, which is normally required for testing modules and
@@ -1256,12 +1300,13 @@ build (size_t argc, const char* argv[])
     {
       auto args = [installed, &step_args] (
         const std::multimap<string, string>& args,
-        step_id main_step,
-        step_id test_installed_step)
+        step_id test_separate_installed_step,
+        step_id test_separate_step,
+        step_id main_step)
       {
         return installed
-               ? step_args (args, test_installed_step, main_step)
-               : step_args (args, main_step);
+               ? step_args (args, test_separate_installed_step, main_step)
+               : step_args (args, test_separate_step, main_step);
 
       };
 
@@ -1274,7 +1319,7 @@ build (size_t argc, const char* argv[])
         // bpkg build --configure-only <env-config-args> <config-args>
         //            '<package-name>[ <version-constraint>]'
         //
-        // bpkg.configure.build
+        // bpkg.test-separate[-installed].configure.build (bpkg.configure.build)
         //
         r.status |= run_bpkg (
           envvars,
@@ -1286,12 +1331,14 @@ build (size_t argc, const char* argv[])
           "--yes",
 
           args (env_args,
-                step_id::bpkg_configure_build,
-                step_id::bpkg_test_installed_configure_build),
+                step_id::bpkg_test_separate_installed_configure_build,
+                step_id::bpkg_test_separate_configure_build,
+                step_id::bpkg_configure_build),
 
           args (config_args,
-                step_id::bpkg_configure_build,
-                step_id::bpkg_test_installed_configure_build),
+                step_id::bpkg_test_separate_installed_configure_build,
+                step_id::bpkg_test_separate_configure_build,
+                step_id::bpkg_configure_build),
 
           import,
           "--",
@@ -1336,7 +1383,7 @@ build (size_t argc, const char* argv[])
         //
         // bpkg update <env-config-args> <config-args> <package-name>
         //
-        // bpkg.update.update
+        // bpkg.test-separate[-installed].update (bpkg.update)
         //
         r.status |= run_bpkg (
           envvars,
@@ -1345,12 +1392,14 @@ build (size_t argc, const char* argv[])
           "update",
 
           args (env_args,
-                step_id::bpkg_update,
-                step_id::bpkg_test_installed_update),
+                step_id::bpkg_test_separate_installed_update,
+                step_id::bpkg_test_separate_update,
+                step_id::bpkg_update),
 
           args (config_args,
-                step_id::bpkg_update,
-                step_id::bpkg_test_installed_update),
+                step_id::bpkg_test_separate_installed_update,
+                step_id::bpkg_test_separate_update,
+                step_id::bpkg_update),
 
           import,
           pkg);
@@ -1365,7 +1414,7 @@ build (size_t argc, const char* argv[])
         //
         // bpkg test <env-config-args> <config-args> <package-name>
         //
-        // bpkg.test.test
+        // bpkg.test-separate[-installed].test (bpkg.test)
         //
         r.status |= run_bpkg (
           envvars,
@@ -1375,12 +1424,14 @@ build (size_t argc, const char* argv[])
           "--package-cwd", // See above for details.
 
           args (env_args,
-                step_id::bpkg_test,
-                step_id::bpkg_test_installed_test),
+                step_id::bpkg_test_separate_installed_test,
+                step_id::bpkg_test_separate_test,
+                step_id::bpkg_test),
 
           args (config_args,
-                step_id::bpkg_test,
-                step_id::bpkg_test_installed_test),
+                step_id::bpkg_test_separate_installed_test,
+                step_id::bpkg_test_separate_test,
+                step_id::bpkg_test),
 
           import,
           pkg);
@@ -1407,7 +1458,7 @@ build (size_t argc, const char* argv[])
 
       // Run internal tests.
       //
-      if (internal_tests)
+      if (internal_tests) // Note: false for modules (see above).
       {
         // bpkg test <env-config-args> <config-args> <package-name>
         //
@@ -1433,13 +1484,20 @@ build (size_t argc, const char* argv[])
       // Thus, we test them in the configuration used to build the dependent
       // package (except for the build system module).
       //
-      if (external_tests &&
-          !test (r,
-                 dist_root,
-                 false /* installed */,
-                 module,
-                 bootstrap ? module_import.c_str () : nullptr))
-        break;
+      if (external_tests)
+      {
+        // The test separate phase.
+        //
+        if (!test (r,
+                   dist_root,
+                   false /* installed */,
+                   module,
+                   bootstrap ? module_import.c_str () : nullptr))
+          break;
+
+        // Back to the main phase.
+        //
+      }
 
       rm.status |= r.status;
     }
@@ -1495,6 +1553,7 @@ build (size_t argc, const char* argv[])
 
     // The test installed phase.
     //
+
     // Make sure that the installed package executables are properly imported
     // when configuring and running tests, unless we are testing the build
     // system module (that supposedly doesn't install any executables).
@@ -1646,7 +1705,7 @@ build (size_t argc, const char* argv[])
         //
         // bpkg create <env-modules> <env-config-args> <config-args>
         //
-        // bpkg.test-installed.create (fallback to bpkg.create)
+        // bpkg.test-installed.create (bpkg.create)
         //
         dir_path config_dir ("build-installed-bpkg");
 
@@ -1676,7 +1735,7 @@ build (size_t argc, const char* argv[])
 
         // bpkg add <env-config-args> <config-args> <repository-url>
         //
-        // bpkg.test-installed.configure.add (fallback to bpkg.configure.add)
+        // bpkg.test-installed.configure.add (bpkg.configure.add)
         //
         r.status |= run_bpkg (
           trace, r.log, wre,
@@ -1698,8 +1757,7 @@ build (size_t argc, const char* argv[])
 
         // bpkg fetch <env-config-args> <config-args> <trust-options>
         //
-        // bpkg.test-installed.configure.fetch (fallback to
-        // bpkg.configure.fetch)
+        // bpkg.test-installed.configure.fetch (bpkg.configure.fetch)
         //
         r.status |= run_bpkg (
           trace, r.log, wre,
@@ -1719,7 +1777,7 @@ build (size_t argc, const char* argv[])
         if (!r.status)
           break;
 
-        // Build/test.
+        // The test separate installed phase.
         //
         if (!test (r,
                    rwd / dir_path ("dist-installed"),
@@ -1728,6 +1786,9 @@ build (size_t argc, const char* argv[])
                    nullptr /* import */,
                    envvars))
           break;
+
+        // Back to the test installed phase.
+        //
       }
 
       rm.status |= r.status;
diff --git a/doc/manual.cli b/doc/manual.cli
index 4337627..8285366 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -779,9 +779,11 @@ modules (\c{<env-modules>}) and the list of configuration options and variables
 The re-executed \c{bbot} worker then proceeds to test the package from the
 repository by executing the following commands, collectively called a
 \i{worker script}. Each command has a unique \i{step id} that can be used as a
-prefix in the \c{<config-args>}, \c{<env-config-args>}, and \c{<env-modules>}
-values as discussed in \l{#arch-controller Controller Logic}. The
-\c{<>}-values are from the task manifest and the environment:
+breakpoint as well as a prefix in the \c{<config-args>},
+\c{<env-config-args>}, and \c{<env-modules>} values as discussed in
+\l{#arch-controller Controller Logic}. Some step ids have fallback step ids
+(listed in parenthesis) which are used in the absence of the primary step id
+values. The \c{<>}-values are from the task manifest and the environment.
 
 \
 # bpkg.create
@@ -804,39 +806,41 @@ bpkg -v build --yes --configure-only <package-name>/<package-version>
 #
 bpkg -v update <package-name>
 
-# if the test operation is supported by the package:
+# If the test operation is supported by the package:
 #
-# bpkg.test
-#
-bpkg -v test <package-name>
+{
+  # bpkg.test
+  #
+  bpkg -v test <package-name>
+}
 
-# for each package referred to by the tests, examples, or benchmarks
+# For each package referred to by the tests, examples, or benchmarks
 # package manifest values and not excluded by the test-exclude task
 # manifest values:
 #
 {
-  # bpkg.configure.build
+  # bpkg.test-separate.configure.build (bpkg.configure.build)
   #
   bpkg -v build --yes --configure-only \\
        '<package-name> [<version-constraint>]'
 
-  # bpkg.update
+  # bpkg.test-separate.update (bpkg.update)
   #
   bpkg -v update <package-name>
 
-  # bpkg.test
+  # bpkg.test-separate.test (bpkg.test)
   #
   bpkg -v test <package-name>
 }
 
-# if config.install.root is specified:
+# If config.install.root is specified:
 #
 {
   # bpkg.install
   #
   bpkg -v install <package-name>
 
-  # if the package contains subprojects that support the test
+  # If the package contains subprojects that support the test
   # operation:
   #
   {
@@ -853,42 +857,39 @@ bpkg -v test <package-name>
     b -v test
   }
 
-  # if any of the tests, examples, or benchmarks package manifest
+  # If any of the tests, examples, or benchmarks package manifest
   # values are specified and are not all excluded by the test-exclude
   # task manifest values:
   #
   {
-    # bpkg.test-installed.create (fallback to bpkg.create if absent)
+    # bpkg.test-installed.create (bpkg.create)
     #
     bpkg -V create <env-modules> <env-config-args> <config-args>
 
-    # bpkg.test-installed.configure.add (fallback to
-    # bpkg.configure.add if absent)
+    # bpkg.test-installed.configure.add (bpkg.configure.add)
     #
     bpkg -v add <repository-url>
 
-    # bpkg.test-installed.configure.fetch (fallback to
-    # bpkg.configure.fetch if absent)
+    # bpkg.test-installed.configure.fetch (bpkg.configure.fetch)
     #
     bpkg -v fetch --trust <repository-fp>
 
-    # for each package referred to by the tests, examples, or
+    # For each package referred to by the tests, examples, or
     # benchmarks package manifest values and not excluded by the
     # test-exclude task manifest values:
     #
     {
-      # bpkg.test-installed.configure.build (fallback to
-      # bpkg.configure.build if absent)
+      # bpkg.test-separate-installed.configure.build (
+      #   bpkg.configure.build)
       #
       bpkg -v build --yes --configure-only \\
            '<package-name> [<version-constraint>]'
 
-      # bpkg.test-installed.update (fallback to bpkg.update if
-      # absent)
+      # bpkg.test-separate-installed.update (bpkg.update)
       #
       bpkg -v update <package-name>
 
-      # bpkg.test-installed.test (fallback to bpkg.test if absent)
+      # bpkg.test-separate-installed.test (bpkg.test)
       #
       bpkg -v test <package-name>
     }
@@ -907,39 +908,44 @@ If the package is a build system module, then it is built and tested (using
 the bundled tests) in a separate configuration that mimics the one used to
 build \c{build2} itself. Note that the configuration and environment options
 and variables are not passed to commands that may affect this configuration.
-Such commands, therefore, have no associated \i{step id}:
+Such commands, therefore, have associated step ids that can only be used
+as breakpoints (listed in square brackets):
 
 \
-#
+# [bpkg.module.create]
 #
 b -V create config.config.load=~build2
 bpkg -v create --existing
 
-# bpkg.configure.add
+# bpkg.module.configure.add (bpkg.configure.add)
 #
 bpkg -v add <repository-url>
 
-# bpkg.configure.fetch
+# bpkg.module.configure.fetch (bpkg.configure.fetch)
 #
 bpkg -v fetch --trust <repository-fp>
 
-#
+# [bpkg.module.configure.build]
 #
 bpkg -v build --yes --configure-only <package-name>/<package-version>
 
-#
+# [bpkg.module.update]
 #
 bpkg -v update <package-name>
 
-# if the test operation is supported by the package:
+# If the test operation is supported by the package:
 #
-bpkg -v test <package-name>
+{
+  # [bpkg.module.test]
+  #
+  bpkg -v test <package-name>
+}
 \
 
 If a primary or test package comes from a version control-based repository,
 then its \c{dist} meta-operation is also tested as a part of the
-\c{bpkg.configure.build} step by re-distributing the source directory in the
-load distribution mode after configuration.
+\c{bpkg[.*].configure.build} steps by re-distributing the source directory in
+the load distribution mode after configuration.
 
 As an example, the following POSIX shell script can be used to setup the
 environment for building C and C++ packages with GCC 9 on most Linux
@@ -985,7 +991,7 @@ are ignored. All other lines in this file have the following format:
 <machine-pattern> <config> <target>[/<environment>] <classes> [<config-arg>]* [<warning-regex>]*
 
 <config-arg> = [<prefix>:](<variable>|<option>)
-<prefix> = <tool>[.phase][.<operation>[.<command>]]
+<prefix> = <tool>[.<phase>][.<operation>[.<command>]]
 \
 
 Where \c{<machine-pattern>} is filesystem wildcard pattern that is matched
@@ -1093,7 +1099,7 @@ that support the test operation and/or refers to other packages via the
 \c{tests}, \c{examples}, or \c{benchmarks} manifest values which are not
 excluded by the \c{test-exclude} task manifest values, then the worker will
 additionally build such subprojects/packages against the installation and run
-their tests (test installed phase).
+their tests (test installed and test separate installed phases).
 
 Two types of installations can be tested: \i{system} and \i{private}. A system
 installation uses a well-known location, such as \c{/usr} or \c{/usr/local},
diff --git a/tests/integration/testscript b/tests/integration/testscript
index fe1b578..e9b0db1 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -108,7 +108,7 @@ rfp = yes
 
 #\
 pkg = cli
-ver = 1.2.0-b.7.20200803080502.02dac36b07e8
+ver = 1.2.0-b.7.20210311174126.7aba3e27228e
 rep_url = "https://git.codesynthesis.com/cli/cli.git#adhoc-recipe"
 rep_type = git
 #rep_url = https://stage.build2.org/1
@@ -118,8 +118,8 @@ rfp = yes
 
 #\
 pkg = libxsd
-ver = 4.2.0-b.1.20210215122732.91a2243739fd
-rep_url = "https://git.codesynthesis.com/xsd/xsd.git#build2"
+ver = 4.2.0-b.1.20210302135218.6a71bc57f6eb
+rep_url = "https://git.codesynthesis.com/xsd/xsd.git#master"
 rep_type = git
 #rep_url = https://stage.build2.org/1
 #rep_type = pkg