For rep-create, pkg-verify, and rep-info don't verify build-time toolchain dependency constraints if --ignore-unknown option is specified

14 files changed, 172 insertions, 60 deletions

diff --git a/bpkg/package.cxx b/bpkg/package.cxx
index a7282c5..44eff01 100644
--- a/bpkg/package.cxx
+++ b/bpkg/package.cxx
@@ -571,11 +571,13 @@ namespace bpkg
       ? pkg_verify (options,
                     a->absolute () ? *a : db.config_orig / *a,
                     true /* ignore_unknown */,
+                    false /* ignore_toolchain */,
                     false /* expand_values */,
                     true /* load_buildfiles */)
       : pkg_verify (options,
                     sp->effective_src_root (db.config_orig),
                     true /* ignore_unknown */,
+                    false /* ignore_toolchain */,
                     true /* load_buildfiles */,
                     // Copy potentially fixed up version from selected package.
                     [&sp] (version& v) {v = sp->version;}));
@@ -827,7 +829,7 @@ namespace bpkg
 
   bool
   toolchain_buildtime_dependency (const common_options& o,
-                                  const dependency_alternatives_ex& das,
+                                  const dependency_alternatives& das,
                                   const package_name* pkg)
   {
     if (das.buildtime)
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index 694d068..aecb4e2 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -589,7 +589,7 @@ namespace bpkg
 
   bool
   toolchain_buildtime_dependency (const common_options&,
-                                  const dependency_alternatives_ex&,
+                                  const dependency_alternatives&,
                                   const package_name*);
 
   // Return true if any dependency other than toolchain build-time
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index 2b1c4f3..4217b76 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -10197,6 +10197,7 @@ namespace bpkg
                 pkg_verify (o,
                             a,
                             true /* ignore_unknown */,
+                            false /* ignore_toolchain */,
                             false /* expand_values */,
                             true /* load_buildfiles */,
                             true /* complete_depends */,
@@ -10263,6 +10264,7 @@ namespace bpkg
                     o,
                     d,
                     true /* ignore_unknown */,
+                    false /* ignore_toolchain */,
                     true /* load_buildfiles */,
                     [&o, &d, &pvi] (version& v)
                     {
diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx
index d065ff1..50f0937 100644
--- a/bpkg/pkg-fetch.cxx
+++ b/bpkg/pkg-fetch.cxx
@@ -163,6 +163,7 @@ namespace bpkg
     package_manifest m (pkg_verify (co,
                                     a,
                                     true /* ignore_unknown */,
+                                    false /* ignore_toolchain */,
                                     false /* expand_values */,
                                     false /* load_buildfiles */));
 
diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx
index 99bd161..48a64ea 100644
--- a/bpkg/pkg-unpack.cxx
+++ b/bpkg/pkg-unpack.cxx
@@ -217,6 +217,7 @@ namespace bpkg
       pkg_verify (o,
                   d,
                   true /* ignore_unknown */,
+                  false /* ignore_toolchain */,
                   false /* load_buildfiles */,
                   [&o, &d, &pvi] (version& v)
                   {
@@ -433,6 +434,7 @@ namespace bpkg
           pkg_verify (co,
                       d,
                       true  /* ignore_unknown */,
+                      false /* ignore_toolchain */,
                       false /* load_buildfiles */,
                       function<package_manifest::translate_function> ()));
 
diff --git a/bpkg/pkg-verify.cli b/bpkg/pkg-verify.cli
index bd6609f..9d45cad 100644
--- a/bpkg/pkg-verify.cli
+++ b/bpkg/pkg-verify.cli
@@ -47,8 +47,9 @@ namespace bpkg
     bool --ignore-unknown
     {
       "Ignore unknown manifest entries. By default, \cb{bpkg} will refuse to
-       declare such a package valid since it cannot be sure the unknown entries
-       are valid."
+       declare such a package valid since it cannot be sure the unknown
+       entries are valid. Note that this option also ignores the version
+       constraints in the special toolchain build-time dependencies."
     }
 
     bool --manifest
diff --git a/bpkg/pkg-verify.cxx b/bpkg/pkg-verify.cxx
index 26393d6..6d5a2fe 100644
--- a/bpkg/pkg-verify.cxx
+++ b/bpkg/pkg-verify.cxx
@@ -18,9 +18,10 @@ using namespace butl;
 
 namespace bpkg
 {
-  vector<manifest_name_value>
+  pkg_verify_result
   pkg_verify (const common_options& co,
               manifest_parser& p,
+              bool it,
               const path& what,
               int diag_level)
   {
@@ -36,10 +37,10 @@ namespace bpkg
       throw manifest_parsing (p.name (), nv.value_line, nv.value_column,
                               "unsupported format version");
 
-    vector<manifest_name_value> r;
+    pkg_verify_result r;
 
     // For the depends name, parse the value and if it contains the build2 or
-    // bpkg constraints, verify that they are satisfied.
+    // bpkg constraints, verify that they are satisfied, if requested.
     //
     // Note that if the semantics of the depends value changes we may be
     // unable to parse some of them before we get to build2 or bpkg and issue
@@ -66,15 +67,19 @@ namespace bpkg
 
         if (das.buildtime)
         {
-          for (const dependency_alternative& da: das)
+          for (dependency_alternative& da: das)
           {
-            for (const dependency& d: da)
+            for (dependency& d: da)
             {
               const package_name& dn (d.name);
 
               if (dn != "build2" && dn != "bpkg")
                 continue;
 
+              // Even if the toolchain build-time dependencies are requested
+              // to be ignored let's make sure they are well-formed, i.e. they
+              // are the only dependencies in the respective depends values.
+              //
               if (da.size () != 1)
               {
                 if (diag_level != 0)
@@ -95,7 +100,7 @@ namespace bpkg
 
               if (dn == "build2")
               {
-                if (d.constraint && !satisfy_build2 (co, d))
+                if (!it && d.constraint && !satisfy_build2 (co, d))
                 {
                   if (diag_level != 0)
                   {
@@ -111,10 +116,12 @@ namespace bpkg
 
                   throw failed ();
                 }
+
+                r.build2_dependency = move (d);
               }
               else
               {
-                if (d.constraint && !satisfy_bpkg (co, d))
+                if (!it && d.constraint && !satisfy_bpkg (co, d))
                 {
                   if (diag_level != 0)
                   {
@@ -124,11 +131,14 @@ namespace bpkg
                     if (!what.empty ())
                       dr << " for package " << what;
 
-                    dr << "available bpkg version is " << bpkg_version;
+                    dr << info << "available bpkg version is "
+                       << bpkg_version;
                   }
 
                   throw failed ();
                 }
+
+                r.bpkg_dependency = move (d);
               }
             }
           }
@@ -153,6 +163,7 @@ namespace bpkg
   pkg_verify (const common_options& co,
               const path& af,
               bool iu,
+              bool it,
               bool ev,
               bool lb,
               bool cd,
@@ -181,9 +192,10 @@ namespace bpkg
       manifest_parser mp (is, mf.string ());
 
       package_manifest m (mp.name (),
-                          pkg_verify (co, mp, af, diag_level),
+                          pkg_verify (co, mp, it, af, diag_level),
                           iu,
                           cd);
+
       is.close ();
 
       if (wait ())
@@ -415,6 +427,7 @@ namespace bpkg
   pkg_verify (const common_options& co,
               const dir_path& d,
               bool iu,
+              bool it,
               bool lb,
               const function<package_manifest::translate_function>& tf,
               int diag_level)
@@ -437,7 +450,7 @@ namespace bpkg
       manifest_parser mp (ifs, mf.string ());
 
       package_manifest m (mp.name (),
-                          pkg_verify (co, mp, d, diag_level),
+                          pkg_verify (co, mp, it, d, diag_level),
                           tf,
                           iu);
 
@@ -546,13 +559,15 @@ namespace bpkg
     //
     try
     {
-      package_manifest m (pkg_verify (o,
-                                      a,
-                                      o.ignore_unknown (),
-                                      o.deep () /* expand_values */,
-                                      o.deep () /* load_buildfiles */,
-                                      o.deep () /* complete_depends */,
-                                      o.silent () ? 0 : 2));
+      package_manifest m (
+        pkg_verify (o,
+                    a,
+                    o.ignore_unknown (),
+                    o.ignore_unknown () /* ignore_toolchain */,
+                    o.deep () /* expand_values */,
+                    o.deep () /* load_buildfiles */,
+                    o.deep () /* complete_depends */,
+                    o.silent () ? 0 : 2));
 
       if (o.manifest ())
       {
diff --git a/bpkg/pkg-verify.hxx b/bpkg/pkg-verify.hxx
index b4b536b..631f0be 100644
--- a/bpkg/pkg-verify.hxx
+++ b/bpkg/pkg-verify.hxx
@@ -19,9 +19,11 @@ namespace bpkg
   pkg_verify (const pkg_verify_options&, cli::scanner& args);
 
   // Verify archive is a valid package and return its manifest. If requested,
-  // expand the file-referencing manifest values (description, changes, etc),
-  // setting them to the contents of files they refer to, set the potentially
-  // absent description-type value to the effective description type (see
+  // verify that all manifest entries are recognized and the package is
+  // compatible with the current toolchain. Also, if requested, expand the
+  // file-referencing manifest values (description, changes, etc), setting
+  // them to the contents of files they refer to, set the potentially absent
+  // description-type value to the effective description type (see
   // libbpkg/manifest.hxx), load the bootstrap, root, and config/*.build
   // buildfiles into the respective *-build values, and complete the
   // dependency constraints.
@@ -42,6 +44,7 @@ namespace bpkg
   pkg_verify (const common_options&,
               const path& archive,
               bool ignore_unknown,
+              bool ignore_toolchain,
               bool expand_values,
               bool load_buildfiles,
               bool complete_depends = true,
@@ -57,24 +60,34 @@ namespace bpkg
   pkg_verify (const common_options&,
               const dir_path& source,
               bool ignore_unknown,
+              bool ignore_toolchain,
               bool load_buildfiles,
               const function<package_manifest::translate_function>&,
               int diag_level = 2);
 
   // Pre-parse the package manifest and return the name value pairs list,
-  // stripping the format version and the end-of-manifest/stream pairs. Also
-  // verify that the package is compatible with the current toolchain and
-  // issue diagnostics and throw failed if it is not. Pass through the
-  // manifest_parsing and io_error exceptions, so that the caller can decide
-  // how to handle them (for example, ignore them if the manifest-printing
-  // process has failed, etc).
+  // stripping the format version and the end-of-manifest/stream pairs,
+  // together with the build2/bpkg build-time dependencies, if present. If
+  // requested, verify that the package is compatible with the current
+  // toolchain and issue diagnostics and throw failed if it is not.
+  //
+  // Pass through the manifest_parsing and io_error exceptions, so that the
+  // caller can decide how to handle them (for example, ignore them if the
+  // manifest-printing process has failed, etc).
   //
   // To omit the package location from the diagnostics, pass an empty path as
   // the what argument.
   //
-  vector<butl::manifest_name_value>
+  struct pkg_verify_result: vector<butl::manifest_name_value>
+  {
+    optional<dependency> build2_dependency;
+    optional<dependency> bpkg_dependency;
+  };
+
+  pkg_verify_result
   pkg_verify (const common_options&,
               butl::manifest_parser&,
+              bool ignore_toolchain,
               const path& what,
               int diag_level = 2);
 }
diff --git a/bpkg/rep-create.cli b/bpkg/rep-create.cli
index 9d062ec..0e44b68 100644
--- a/bpkg/rep-create.cli
+++ b/bpkg/rep-create.cli
@@ -33,7 +33,8 @@ namespace bpkg
 
     bool --ignore-unknown
     {
-      "Ignore unknown manifest entries."
+      "Ignore unknown manifest entries. Note that this option also ignores the
+       version constraints in the special toolchain build-time dependencies."
     }
 
     butl::standard_version --min-bpkg-version
diff --git a/bpkg/rep-create.cxx b/bpkg/rep-create.cxx
index c9ee924..6c95679 100644
--- a/bpkg/rep-create.cxx
+++ b/bpkg/rep-create.cxx
@@ -101,6 +101,7 @@ namespace bpkg
         pkg_verify (o,
                     a,
                     o.ignore_unknown (),
+                    o.ignore_unknown () /* ignore_toolchain */,
                     true /* expand_values */,
                     true /* load_buildfiles */));
 
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 4fceff7..e00c7c5 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -17,6 +17,7 @@
 #include <bpkg/rep-remove.hxx>
 #include <bpkg/pkg-verify.hxx>
 #include <bpkg/diagnostics.hxx>
+#include <bpkg/satisfaction.hxx>
 #include <bpkg/manifest-utility.hxx>
 
 using namespace std;
@@ -53,7 +54,8 @@ namespace bpkg
                  database* db,
                  const repository_location& rl,
                  const optional<string>& dependent_trust,
-                 bool ignore_unknown)
+                 bool ignore_unknown,
+                 bool ignore_toolchain)
   {
     // First fetch the repositories list and authenticate the base's
     // certificate.
@@ -115,6 +117,18 @@ namespace bpkg
       authenticate_repository (co, conf, cert_pem, *cert, sm, rl);
     }
 
+    // If requested, verify that the packages are compatible with the current
+    // toolchain.
+    //
+    if (!ignore_toolchain)
+    {
+      for (const package_manifest& m: fr.packages)
+      {
+        for (const dependency_alternatives& das: m.dependencies)
+          toolchain_buildtime_dependency (co, das, &m.name);
+      }
+    }
+
     return rep_fetch_data {{move (fr)}, move (cert_pem), move (cert)};
   }
 
@@ -216,6 +230,7 @@ namespace bpkg
                            const dir_path& repo_dir,
                            vector<package_manifest>&& pms,
                            bool iu,
+                           bool it,
                            const repository_location& rl,
                            const optional<string>& fragment) // For diagnostics.
   {
@@ -229,8 +244,10 @@ namespace bpkg
     };
 
     // Verify that all the package directories contain the package manifest
-    // files and retrieve the package versions via the single `b info` call.
-    // While at it cache the manifest paths for the future use.
+    // files and retrieve the package versions via the single `b info` call,
+    // but only if the current build2 version is satisfactory for all the
+    // repository packages. While at it cache the manifest paths for the
+    // future use.
     //
     // Note that if the package is not compatible with the toolchain, not to
     // end up with an unfriendly build2 error message (referring a line in the
@@ -240,16 +257,21 @@ namespace bpkg
     // since we need the package versions for that. Thus, we cache the
     // respective name value lists instead.
     //
-    package_version_infos               pvs;
+    optional<package_version_infos>     pvs;
     paths                               mfs;
-    dir_paths                           pds;
     vector<vector<manifest_name_value>> nvs;
     {
       mfs.reserve (pms.size ());
       nvs.reserve (pms.size ());
 
+      dir_paths pds;
       pds.reserve (pms.size ());
 
+      // If true, then build2 version is satisfactory for all the repository
+      // packages.
+      //
+      bool bs (true);
+
       for (const package_manifest& pm: pms)
       {
         assert (pm.location);
@@ -286,7 +308,14 @@ namespace bpkg
           // (e.g., .bpkg/tmp/6f746365314d/) and it's probably better to omit
           // it entirely (the above exception guard will print all we've got).
           //
-          nvs.push_back (pkg_verify (co, mp, dir_path ()));
+          pkg_verify_result r (pkg_verify (co, mp, it, dir_path ()));
+
+          if (bs                  &&
+              r.build2_dependency &&
+              !satisfy_build2 (co, *r.build2_dependency))
+            bs = false;
+
+          nvs.push_back (move (r));
         }
         catch (const manifest_parsing& e)
         {
@@ -301,14 +330,17 @@ namespace bpkg
         pds.push_back (move (d));
       }
 
-      pvs = package_versions (co, pds);
+      if (bs)
+        pvs = package_versions (co, pds);
     }
 
     // Parse package manifests, fixing up their versions.
     //
     pair<vector<package_manifest>, vector<package_info>> r;
-    r.first.reserve  (pms.size ());
-    r.second.reserve (pms.size ());
+    r.first.reserve (pms.size ());
+
+    if (pvs)
+      r.second.reserve (pms.size ());
 
     for (size_t i (0); i != pms.size (); ++i)
     {
@@ -318,15 +350,18 @@ namespace bpkg
 
       try
       {
-        optional<version>& pv (pvs[i].version);
-
         package_manifest m (
           mfs[i].string (),
           move (nvs[i]),
-          [&pv] (version& v)
+          [&pvs, i] (version& v)
           {
-            if (pv)
-              v = move (*pv);
+            if (pvs)
+            {
+              optional<version>& pv ((*pvs)[i].version);
+
+              if (pv)
+                v = move (*pv);
+            }
           },
           iu);
 
@@ -344,7 +379,9 @@ namespace bpkg
       }
 
       r.first.push_back  (move (pm));
-      r.second.push_back (move (pvs[i].info));
+
+      if (pvs)
+        r.second.push_back (move ((*pvs)[i].info));
     }
 
     return r;
@@ -390,6 +427,7 @@ namespace bpkg
   rep_fetch_dir (const common_options& co,
                  const repository_location& rl,
                  bool iu,
+                 bool it,
                  bool ev,
                  bool lb)
   {
@@ -417,6 +455,7 @@ namespace bpkg
                                rd,
                                move (pms),
                                iu,
+                               it,
                                rl,
                                empty_string /* fragment */));
 
@@ -483,6 +522,7 @@ namespace bpkg
                  const dir_path* conf,
                  const repository_location& rl,
                  bool iu,
+                 bool it,
                  bool ev,
                  bool lb)
   {
@@ -612,6 +652,7 @@ namespace bpkg
                                  td,
                                  move (pms),
                                  iu,
+                                 it,
                                  rl,
                                  fr.friendly_name));
 
@@ -714,14 +755,24 @@ namespace bpkg
              const repository_location& rl,
              const optional<string>& dt,
              bool iu,
+             bool it,
              bool ev,
              bool lb)
   {
     switch (rl.type ())
     {
-    case repository_type::pkg: return rep_fetch_pkg (co, conf, db, rl, dt, iu);
-    case repository_type::dir: return rep_fetch_dir (co, rl, iu, ev, lb);
-    case repository_type::git: return rep_fetch_git (co, conf, rl, iu, ev, lb);
+    case repository_type::pkg:
+      {
+        return rep_fetch_pkg (co, conf, db, rl, dt, iu, it);
+      }
+    case repository_type::dir:
+      {
+        return rep_fetch_dir (co, rl, iu, it, ev, lb);
+      }
+    case repository_type::git:
+      {
+        return rep_fetch_git (co, conf, rl, iu, it, ev, lb);
+      }
     }
 
     assert (false); // Can't be here.
@@ -733,6 +784,7 @@ namespace bpkg
              const dir_path* conf,
              const repository_location& rl,
              bool iu,
+             bool it,
              bool ev,
              bool lb)
   {
@@ -742,6 +794,7 @@ namespace bpkg
                       rl,
                       nullopt /* dependent_trust */,
                       iu,
+                      it,
                       ev,
                       lb);
   }
@@ -1028,6 +1081,10 @@ namespace bpkg
       //
       if (rl.directory_based ())
       {
+        // Wouldn't be here otherwise. Note that rep_fetch_data is retrieved
+        // by rep_fetch() with false passed as the ignore_toolchain argument
+        // (see rep_fetch() for more details).
+        //
         assert (!pis.empty ());
 
         // Note that we can't check if the external package of this upstream
@@ -1235,6 +1292,7 @@ namespace bpkg
                  rl,
                  dependent_trust,
                  true /* ignore_unknow */,
+                 false /* ignore_toolchain */,
                  false /* expand_values */,
                  true /* load_buildfiles */));
 
diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx
index 7e03999..a26eba3 100644
--- a/bpkg/rep-fetch.hxx
+++ b/bpkg/rep-fetch.hxx
@@ -37,7 +37,8 @@ namespace bpkg
 
       // Empty if the build2 project info is not available for the packages.
       // Currently we only retrieve it for the directory and version control
-      // based repositories.
+      // based repositories, but only if the current build2 version is
+      // satisfactory for all the repository packages.
       //
       vector<package_info>        package_infos;
     };
@@ -50,19 +51,22 @@ namespace bpkg
     shared_ptr<const bpkg::certificate> certificate; // Authenticated.
   };
 
-  // If requested, expand the file-referencing package manifest values
-  // (description, changes, etc), setting them to the contents of files they
-  // refer to and set the potentially absent description-type value to the
-  // effective description type (see libbpkg/manifest.hxx) and load the
-  // bootstrap, root, and config/*.build buildfiles into the respective *-build
-  // values. Note that for pkg repositories such values are expanded/loaded at
-  // the repository creation time.
+  // If requested, verify that all manifest entries are recognized and the
+  // packages are compatible with the current toolchain. Also, if requested,
+  // expand the file-referencing package manifest values (description,
+  // changes, etc), setting them to the contents of files they refer to and
+  // set the potentially absent description-type value to the effective
+  // description type (see libbpkg/manifest.hxx) and load the bootstrap, root,
+  // and config/*.build buildfiles into the respective *-build values. Note
+  // that for pkg repositories such values are expanded/loaded at the
+  // repository creation time.
   //
   rep_fetch_data
   rep_fetch (const common_options&,
              const dir_path* configuration,
              const repository_location&,
              bool ignore_unknown,
+             bool ignore_toolchain,
              bool expand_values,
              bool load_buildfiles);
 
diff --git a/bpkg/rep-info.cli b/bpkg/rep-info.cli
index 441227e..c18831f 100644
--- a/bpkg/rep-info.cli
+++ b/bpkg/rep-info.cli
@@ -93,7 +93,16 @@ namespace bpkg
        in the resulting packages manifest the \cb{*-file} values are replaced
        with the contents of the referenced files and the \c{*-build} values
        are automatically added (unless the corresponding files are
-       absent). See also \cb{--repositories-file} and \cb{--packages-file}."
+       absent). See also \cb{--ignore-unknown}, \cb{--repositories-file}, and
+       \cb{--packages-file}."
+    }
+
+    bool --ignore-unknown
+    {
+      "Ignore unknown manifest entries. Note that this option also ignores
+       the version constraints in the special toolchain build-time
+       dependencies. This option is implied if \cb{--manifest} is not
+       specified."
     }
 
     bool --deep
diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx
index 7d78173..190a210 100644
--- a/bpkg/rep-info.cxx
+++ b/bpkg/rep-info.cxx
@@ -65,11 +65,14 @@ namespace bpkg
 
     init_tmp (conf != nullptr ? *conf : empty_dir_path);
 
+    bool ignore_unknown (!o.manifest () || o.ignore_unknown ());
+
     rep_fetch_data rfd (
       rep_fetch (o,
                  conf,
                  rl,
-                 !o.manifest () /* ignore_unknow */,
+                 ignore_unknown,
+                 ignore_unknown /* ignore_toolchain */,
                  o.deep () /* expand_values */,
                  o.deep () /* load_buildfiles */));