Add support for modules that require bootstrap

2 files changed, 78 insertions, 37 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index dd941ad..ade9060 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -630,6 +630,23 @@ build (size_t argc, const char* argv[])
 
     rwd = current_directory ();
 
+    // If this is a build system module that requires bootstrap, then its
+    // importation into the dependent (test) projects cannot be configured and
+    // the corresponding config.import.* variable needs to be specified on the
+    // bpkg/build2 command line as a global override, whenever required.
+    //
+    // Note that such a module must be explicitly marked with `requires:
+    // bootstrap` in its manifest. This can only be detected after the module
+    // is configured and it's manifest available.
+    //
+    bool bootstrap (false);
+
+    // Note that we will parse the package manifest right after the package is
+    // configured.
+    //
+    package_manifest pm;
+    path mf (pkg_dir / "manifest"); // Relative to the configuration directory.
+
     if (module)
     {
       // Configure.
@@ -720,6 +737,19 @@ build (size_t argc, const char* argv[])
           break;
 
         rm.status |= r.status;
+
+        // Note that being unable to parse the package manifest is likely to
+        // be an infrastructure problem, given that the package has been
+        // successfully configured.
+        //
+        pm = parse_manifest<package_manifest> (mf, "package");
+
+        bootstrap = find_if (pm.requirements.begin (),
+                             pm.requirements.end (),
+                             [] (const requirement_alternatives& r)
+                             {
+                               return r.size () == 1 && r[0] == "bootstrap";
+                             }) != pm.requirements.end ();
       }
 
       // Update.
@@ -761,6 +791,10 @@ build (size_t argc, const char* argv[])
         // Use --package-cwd to help ported to build2 third-party packages a
         // bit (see bpkg-pkg-test(1) for details).
         //
+        // Note that internal tests that load the module itself don't make
+        // much sense, thus we don't pass the config.import.*  variable on
+        // the command line for modules that require bootstrap.
+        //
         // bpkg test <package-name>
         //
         r.status |= run_bpkg (
@@ -780,6 +814,14 @@ build (size_t argc, const char* argv[])
     // The "main" step.
     //
 
+    // Use the global override for modules that require bootstrap.
+    //
+    string module_import (
+      module
+      ? ((bootstrap ? "!config.import." : "config.import.") +
+         tm.name.variable () + "=" + (rwd / module_dir).string ())
+      : "");
+
     // Configure.
     //
     dir_path build_dir ("build"); // Configuration directory name.
@@ -797,13 +839,8 @@ build (size_t argc, const char* argv[])
       //
       {
         // If the package is a build system module, then make sure it is
-        // importable in this configuration.
+        // importable in this configuration (see above about bootstrap).
         //
-        string import (module
-                       ? ("config.import." + tm.name.variable () + "=" +
-                          module_dir.string ())
-                       : "");
-
         r.status |= run_bpkg (
           trace, r.log, wre,
           "-V",
@@ -813,7 +850,7 @@ build (size_t argc, const char* argv[])
           step_args (modules,  step_id::bpkg_configure_create),
           step_args (env_args, step_id::bpkg_configure_create),
           cargs,
-          import.empty () ? nullptr : import.c_str ());
+          module && !bootstrap ? module_import.c_str () : nullptr);
 
         if (!r.status)
           break;
@@ -871,6 +908,8 @@ build (size_t argc, const char* argv[])
 
         if (!r.status)
           break;
+
+        pm = parse_manifest<package_manifest> (mf, "package");
       }
 
       rm.status |= r.status;
@@ -917,26 +956,6 @@ build (size_t argc, const char* argv[])
                              "test") != prj.operations.end ();
     }
 
-    // Parse the package manifest.
-    //
-    // Note that being unable to parse the package manifest is likely to be an
-    // infrastructure problem, given that the package has been successfully
-    // configured.
-    //
-    path mf (pkg_config / pkg_dir / "manifest");
-    package_manifest pm (parse_manifest<package_manifest> (mf, "package"));
-
-    // A build system module that requires bootstrap must be explicitly marked
-    // with `requires: bootstrap`.
-    //
-    bool bootstrap (module &&
-                    find_if (pm.requirements.begin (),
-                             pm.requirements.end (),
-                             [] (const requirement_alternatives& r)
-                             {
-                               return r.size () == 1 && r[0] == "bootstrap";
-                             }) != pm.requirements.end ());
-
     // Run the package external tests, if specified. But first filter them
     // against the test-exclude task manifest values using the package names.
     //
@@ -967,8 +986,9 @@ build (size_t argc, const char* argv[])
     bool external_tests (!pm.tests.empty ());
 
     // Configure, update, and test packages in the bpkg configuration in the
-    // current working directory. Optionally set the environment variables for
-    // bpkg processes. Return true if all operations for all packages succeed.
+    // current working directory. Optionally pass the config.import.* variable
+    // override and/or set the environment variables for bpkg processes.
+    // Return true if all operations for all packages succeed.
     //
     // Note that we assume that these packages belong to the dependent
     // package's repository or its complement repositories, recursively. Thus,
@@ -977,6 +997,7 @@ build (size_t argc, const char* argv[])
     //
     auto test = [&pm, &trace, &wre, &step_args, &config_args, &env_args]
                 (operation_result& r,
+                 const char* import = nullptr,
                  const small_vector<string, 1>& envvars = {})
     {
       for (const test_dependency& td: pm.tests)
@@ -990,6 +1011,17 @@ build (size_t argc, const char* argv[])
         //
         // bpkg.configure.build
         //
+        // @@ Note that this currently fails for packages that use modules
+        //    requiring bootstrap that come from git repositories (unless
+        //    special measures are taken to omit the module for info and dist
+        //    meta-oprations) since the package checkout involves build2 dist
+        //    meta-operation to which the import variable override is not
+        //    passed (it is supposed to be passed later, when the checked out
+        //    package is being configured). This issue should be gone when
+        //    bpkg is fixed to run the build2 dist meta-operation in some
+        //    special (not yet implemented) "bootstrap" mode which doesn't
+        //    load the project.
+        //
         r.status |= run_bpkg (
           envvars,
           trace, r.log, wre,
@@ -999,6 +1031,7 @@ build (size_t argc, const char* argv[])
           "--yes",
           step_args (env_args,    step_id::bpkg_configure_build),
           step_args (config_args, step_id::bpkg_configure_build),
+          import,
           "--",
           td.string ());
 
@@ -1018,6 +1051,7 @@ build (size_t argc, const char* argv[])
           "update",
           step_args (env_args,    step_id::bpkg_update_update),
           step_args (config_args, step_id::bpkg_update_update),
+          import,
           pkg);
 
         if (!r.status)
@@ -1040,6 +1074,7 @@ build (size_t argc, const char* argv[])
           "--package-cwd", // See above for details.
           step_args (env_args,    step_id::bpkg_test_test),
           step_args (config_args, step_id::bpkg_test_test),
+          import,
           pkg);
 
         if (!r.status)
@@ -1049,7 +1084,7 @@ build (size_t argc, const char* argv[])
       return true;
     };
 
-    if ((internal_tests || external_tests) && !bootstrap) //@@ TMP
+    if (internal_tests || external_tests)
     {
       operation_result& r (test_result != nullptr
                            ? *test_result
@@ -1078,7 +1113,8 @@ build (size_t argc, const char* argv[])
 
       // Run external tests.
       //
-      if (external_tests && !test (r))
+      if (external_tests &&
+          !test (r, bootstrap ? module_import.c_str () : nullptr))
         break;
 
       rm.status |= r.status;
@@ -1332,7 +1368,7 @@ build (size_t argc, const char* argv[])
 
         // Build/test.
         //
-        if (!test (r, envvars))
+        if (!test (r, nullptr /* import */, envvars))
           break;
       }
 
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 1d7a013..9833491 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -49,7 +49,7 @@ wait = 1 # Seconds.
 controller = --fake-request ../task --dump-result
 
 pkg = libhello
-ver = 1.0.0+6
+ver = 1.0.0+7
 #rep_url = https://build2.org/pkg/1/hello/stable
 #rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
 rep_url = https://stage.build2.org/1
@@ -73,10 +73,15 @@ rfp = yes
 #\
 
 #\
+# To make sure that the test-installed phase succeeds use the build2 driver
+# installed into ~/install/bin.
+#
 pkg = libbuild2-hello
-ver = 0.1.0-a.0.20200402131859.22b478618203
-rep_url = "https://github.com/build2/libbuild2-hello.git#master"
-rep_type = git
+ver = 0.1.0-a.0.20200810114133.0dba3a3bd624
+#rep_url = "https://github.com/build2/libbuild2-hello.git#master"
+#rep_type = git
+rep_url = https://stage.build2.org/1
+rep_type = pkg
 rfp = yes
 #\
 
@@ -90,7 +95,7 @@ rfp = yes
 
 #\
 pkg = cli
-ver = 1.2.0-b.6.20200619094719.2d23842ebf91
+ver = 1.2.0-b.7.20200912105851.511d78c03656
 rep_url = https://stage.build2.org/1
 rep_type = pkg
 rfp = yes