Configure system dependencies from build package configuration globally in worker

3 files changed, 89 insertions, 30 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index f1e5e8c..3367aaa 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -1109,20 +1109,23 @@ build (size_t argc, const char* argv[])
     // If the package configuration is specified, then parse it into the
     // prefixed global options and configuration variables map, unprefixed
     // global options list, the main package-specific configuration variables
-    // list, and the dependency packages list, potentially with their own
-    // configuration variables (but not options). The prefixed arguments are
-    // added to the command lines at the corresponding steps after potential
-    // environment and target configuration arguments. Unprefixed arguments
-    // are added to the bpkg-pkg-build command line at the
-    // bpkg.configure.build step. Specifically, the unprefixed global options
-    // are specified after all the prefixed global options and the unprefixed
-    // variables are specified for the main package only, wherever it is
-    // configured.
+    // list, the main package-specific dependency packages list (only
+    // configured where the main package is configured), potentially with
+    // their own configuration variables (but not options), and the global
+    // system dependency packages list (configured in all configurations). The
+    // prefixed arguments are added to the command lines at the corresponding
+    // steps after potential environment and target configuration arguments.
+    // Unprefixed arguments are added to the bpkg-pkg-build command line at
+    // the bpkg.configure.build step. Specifically, the unprefixed global
+    // options are specified after all the prefixed global options and the
+    // unprefixed variables are specified for the main package only, wherever
+    // it is configured.
     //
     std::multimap<string, string> pkg_args;
     strings                       pkg_config_opts;
     strings                       pkg_config_vars;
-    vector<pair<string, strings>> pkg_config_deps;
+    vector<pair<string, strings>> pkg_config_main_deps; // ?<pkg>, sys:<pkg>
+    vector<pair<string, strings>> pkg_config_glob_deps; // ?sys:<pkg>
 
     if (!tm.package_config.empty ())
     {
@@ -1251,7 +1254,35 @@ build (size_t argc, const char* argv[])
                 vars.push_back (move (da));
               }
 
-              pkg_config_deps.push_back (make_pair (move (a), move (vars)));
+              // Add the system dependency packages (prefixed with `?sys:`) to
+              // a separate list, to specify them globally on the
+              // bpkg-pkg-build command line for configuring them in all the
+              // (being) created configurations.
+              //
+              // Note, though, that we will handle the build-to-hold system
+              // packages (prefixed with `sys:`) in the same way as non system
+              // dependencies, since such an auto-configuration is only
+              // supported by bpkg-pkg-build for system dependencies. In the
+              // future, we may support that on the bbot worker level by, for
+              // example, specifying all the configurations manually for the
+              // build-to-hold system packages and also specifying them as a
+              // system dependencies globally. We need to be careful to make
+              // sure that these dependencies are also auto-configured for the
+              // private configurations potentially created by bpkg-pkg-build.
+              //
+              // Also note that in the future we may allow dependency-specific
+              // --config-uuid options to only configure such dependencies in
+              // the specified configurations. We may also invent the special
+              // 00000000-0000-0000-0000-000000000005 configuration id to, for
+              // example, only configure them at the
+              // bpkg.test-separate-installed.configure.build step.
+              //
+              if (a.compare (0, 5, "?sys:") != 0)
+                pkg_config_main_deps.push_back (make_pair (move (a),
+                                                           move (vars)));
+              else
+                pkg_config_glob_deps.push_back (make_pair (move (a),
+                                                           move (vars)));
             }
           }
           else
@@ -2276,8 +2307,9 @@ build (size_t argc, const char* argv[])
         //      --
         //      { <pkg-config-vars>|config.<pkg-name>.develop=false }+ <pkg>
         //      { config.<runtime-test-name>.develop=false          }+ <runtime-test>...
-        //      { <dep-config-vars>                                 }+ <dep>...
-        //                                                             <dep>...
+        //      { <dep-config-vars>                                 }+ <main-dep>...
+        //                                                             <main-dep>...
+        //                                                             <glob-dep>...
         //
         step_id s  (step_id::bpkg_target_configure_build);
         step_id f1 (step_id::b_configure);
@@ -2335,7 +2367,7 @@ build (size_t argc, const char* argv[])
 
         // Add the main package dependencies.
         //
-        for (const pair<string, strings>& d: pkg_config_deps)
+        for (const pair<string, strings>& d: pkg_config_main_deps)
         {
           if (!d.second.empty ())
           {
@@ -2376,8 +2408,9 @@ build (size_t argc, const char* argv[])
         //                                        <pkg-config-args>
         //                                        config.<buildtime-test-name>.develop=false        }+   <buildtime-test>...
         //
-        //      { <build-config> <install-config> <dep-config-vars>                                 }+   <dep>...
-        //      { <build-config> <install-config>                                                   }+ { <dep>... }
+        //      { <build-config> <install-config> <dep-config-vars>                                 }+   <main-dep>...
+        //      { <build-config> <install-config>                                                   }+ { <main-dep>... }
+        //                                                                                               <glob-dep>...
         //
 
         // Main package configuration name.
@@ -2615,7 +2648,7 @@ build (size_t argc, const char* argv[])
           // specified and count the number of others.
           //
           size_t no_vars (0);
-          for (const pair<string, strings>& d: pkg_config_deps)
+          for (const pair<string, strings>& d: pkg_config_main_deps)
           {
             if (!d.second.empty ())
             {
@@ -2660,7 +2693,7 @@ build (size_t argc, const char* argv[])
             if (no_vars != 1)
               pkgs.push_back ("{");
 
-            for (const pair<string, strings>& d: pkg_config_deps)
+            for (const pair<string, strings>& d: pkg_config_main_deps)
             {
               if (d.second.empty ())
                 pkgs.push_back (d.first);
@@ -2672,6 +2705,11 @@ build (size_t argc, const char* argv[])
         }
       }
 
+      // Add the global system dependencies.
+      //
+      for (const pair<string, strings>& d: pkg_config_glob_deps)
+        pkgs.push_back (d.first);
+
       // Finally, configure all the packages.
       //
       {
@@ -3729,6 +3767,7 @@ build (size_t argc, const char* argv[])
             //      { <config> }+ { <runtime-test>... }
             //                      <buildtime-test>...
             //                      ?sys:<pkg>
+            //                      <glob-dep>...
             //
             strings pkgs;
 
@@ -3776,6 +3815,11 @@ build (size_t argc, const char* argv[])
 
             pkgs.push_back ("?sys:" + pkg_rev);
 
+            // Add the global system dependencies.
+            //
+            for (const pair<string, strings>& d: pkg_config_glob_deps)
+              pkgs.push_back (d.first);
+
             // Finally, configure all the test packages.
             //
             {
diff --git a/doc/manual.cli b/doc/manual.cli
index b50ad77..8b64687 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -1031,8 +1031,9 @@ bpkg -v build --configure-only \\
      [{ <pkg-config-vars> }+] <package-name>/<package-version> \\
      [<test-package-name>[ <test-version-constraint>]...] \\
      [([{ <dep-config-vars> }+] \\
-       (?[sys:]|sys:)<dependency-name> \\
-       [<dependency-version-constraint>])...]
+       (?|sys:)<dependency-name> \\
+       [<dependency-version-constraint>])...] \\
+     [?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
 # bpkg.update
 #
@@ -1117,7 +1118,8 @@ bpkg -v update <package-name>
     bpkg -v build --configure-only \\
          <env-config-args> <tgt-config-args> <pkg-config-args> \\
          <test-package-name>[ <test-version-constraint>]... \\
-         ?sys:<package-name>/<package-version>
+         ?sys:<package-name>/<package-version> \\
+         [?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
     # For each (runtime) tests, examples, or benchmarks package
     # referred to by the task manifest:
@@ -1271,7 +1273,9 @@ bpkg -v build --configure-only \\
 \\
 ({ --config-uuid <host-uuid> [--config-uuid <install-uuid>] \\
    [<dep-config-vars>] }+ \\
- (?[sys:]|sys:)<dependency-name>[ <dependency-version-constraint>])...
+ (?|sys:)<dependency-name>[ <dependency-version-constraint>])... \\
+\\
+[?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
 # bpkg.update
 #
@@ -1416,7 +1420,9 @@ bpkg -v update -d <host-conf> <package-name>
     { --config-name <target-conf> }+ \\
     <buildtime-test-package-name>[ <test-version-constraint>]... \\
     \\
-    ?sys:<package-name>/<package-version>
+    ?sys:<package-name>/<package-version> \\
+    \\
+    [?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
     # For each tests, examples, or benchmarks package referred
     # to by the task manifest:
@@ -1556,7 +1562,9 @@ bpkg -v build --configure-only \\
 \\
 ({ --config-uuid <host-uuid> [--config-uuid <install-uuid>] \\
    [<dep-config-vars>] }+ \\
- (?[sys:]|sys:)<dependency-name>[ <dependency-version-constraint>])...
+ (?|sys:)<dependency-name>[ <dependency-version-constraint>])... \\
+\\
+[?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
 # bpkg.update
 #
@@ -1640,7 +1648,9 @@ bpkg -v update -d <module-conf> <package-name>
     { --config-name <target-conf> }+ \\
     <buildtime-test-package-name>[ <test-version-constraint>]... \\
     \\
-    ?sys:<package-name>/<package-version>
+    ?sys:<package-name>/<package-version> \\
+    \\
+    [?sys:<dependency-name>[ <dependency-version-constraint>]...]
 
     # For each (build-time) tests, examples, or benchmarks package
     # referred to by the task manifest:
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 1e2ebb7..5c14d73 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -182,7 +182,7 @@ tests="tests: libxsd-tests == $ver"
 
 #\
 pkg = xsd
-ver = 4.2.0-b.4.20221031121010.4fdbd8298c4d
+ver = 4.2.0-b.4.20230320140030.aafc60b2e901
 rep_url = "https://git.codesynthesis.com/xsd/xsd.git#master"
 rep_type = git
 #rep_url = https://queue.stage.build2.org/1
@@ -194,12 +194,17 @@ examples: * xsd-examples == $ver"
 host='host: true'
 package_config = 'package-config:
 \
-?libxerces-c +{ config.libxerces_c.network=true }
-?libcurl/7.76.0
-?sys:libz/*
+?sys:libxerces-c --sys-install --sys-yes
 \'
-#dependency_checksum = 'dependency-checksum: 40a0ad4546d836a3afc83a9e7da22f2b5d224af4e62996d88f7103eaee23e9e1'
 #\
+#package_config = 'package-config:
+# \
+#?libxerces-c +{ config.libxerces_c.network=true }
+#?libcurl/7.76.0
+#?sys:libz/*
+# \'
+
+#dependency_checksum = 'dependency-checksum: 40a0ad4546d836a3afc83a9e7da22f2b5d224af4e62996d88f7103eaee23e9e1'
 
 #\
 pkg = libxerces-c