Optimize rep-fetch for dir and git repos quering projects info in single call

4 files changed, 110 insertions, 63 deletions

diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 6ca9e18..a4cee94 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -272,33 +272,39 @@ namespace bpkg
     }
   }
 
-  optional<version>
-  package_version (const common_options& o, const dir_path& d)
+  vector<optional<version>>
+  package_versions (const common_options& o, const dir_paths& ds)
   {
     path b (name_b (o));
 
+    vector<b_project_info> pis;
     try
     {
-      b_project_info pi (
-        b_info (d,
-                false /* ext_mods */,
-                verb,
-                [] (const char* const args[], size_t n)
-                {
-                  if (verb >= 2)
-                    print_process (args, n);
-                },
-                b,
-                exec_dir,
-                o.build_option ()));
-
-      optional<version> r;
-
-      // An empty version indicates that the version module is not enabled for
-      // the project.
-      //
-      if (!pi.version.empty ())
-        r = version (pi.version.string ());
+      b_info (pis,
+              ds,
+              false /* ext_mods */,
+              verb,
+              [] (const char* const args[], size_t n)
+              {
+                if (verb >= 2)
+                  print_process (args, n);
+              },
+              b,
+              exec_dir,
+              o.build_option ());
+
+      vector<optional<version>> r;
+      r.reserve (pis.size ());
+
+      for (const b_project_info& pi: pis)
+      {
+        // An empty version indicates that the version module is not enabled
+        // for the project.
+        //
+        r.push_back (!pi.version.empty ()
+                     ? version (pi.version.string ())
+                     : optional<version> ());
+      }
 
       return r;
     }
@@ -307,7 +313,10 @@ namespace bpkg
       if (e.normal ())
         throw failed (); // Assume the build2 process issued diagnostics.
 
-      fail << "unable to parse project " << d << " info: " << e <<
+      diag_record dr (fail);
+      dr << "unable to parse project ";
+      if (pis.size () < ds.size ()) dr << ds[pis.size ()] << ' ';
+      dr << "info: " << e <<
         info << "produced by '" << b << "'; use --build to override" << endf;
     }
   }
diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx
index 29d548d..b7d9b07 100644
--- a/bpkg/manifest-utility.hxx
+++ b/bpkg/manifest-utility.hxx
@@ -95,20 +95,29 @@ namespace bpkg
   bool
   repository_name (const string&);
 
-  // Return the version of a package as provided by the build2 version module.
-  // Return nullopt if the version module is disabled for the package (or the
-  // build2 project directory doesn't contain the manifest file). Fail if the
-  // directory is not a build2 project.
+  // Return the versions of packages as provided by the build2 version module.
+  // Return nullopt for a package if the version module is disabled for it (or
+  // the build2 project directory doesn't contain the manifest file). Fail if
+  // any of the specified directories is not a build2 project.
   //
-  // Note that if the package directory is under the version control, then the
+  // Note that if a package directory is under the version control, then the
   // resulting version may be populated with the snapshot information (see
   // libbutl/standard-version.mxx for more details). Thus, this function can
-  // be used for fixing up the package manifest version.
+  // be used for fixing up the package manifest versions.
   //
   class common_options;
 
-  optional<version>
-  package_version (const common_options&, const dir_path&);
+  vector<optional<version>>
+  package_versions (const common_options&, const dir_paths&);
+
+  // As above but return the version of a single package.
+  //
+  inline optional<version>
+  package_version (const common_options& o, const dir_path& d)
+  {
+    vector<optional<version>> r (package_versions (o, dir_paths ({d})));
+    return move (r[0]);
+  }
 }
 
 #endif // BPKG_MANIFEST_UTILITY_HXX
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index f6165b7..80598c5 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -195,76 +195,105 @@ namespace bpkg
   static vector<package_manifest>
   parse_package_manifests (const common_options& co,
                            const dir_path& repo_dir,
-                           vector<package_manifest>&& sms,
+                           vector<package_manifest>&& pms,
                            bool iu,
                            const repository_location& rl,
                            const optional<string>& fragment) // For diagnostics.
   {
-    vector<package_manifest> r;
-    r.reserve (sms.size ());
+    auto package_info = [&rl, &fragment] (const package_manifest& pm,
+                                          diag_record& dr)
+    {
+      dr << "package ";
+
+      if (!pm.location->current ())
+        dr << "'" << pm.location->string () << "' "; // Strip trailing '/'.
+
+      dr << "in repository " << rl;
+
+      if (fragment)
+        dr << ' ' << *fragment;
+    };
 
-    for (package_manifest& sm: sms)
+    // Verify that all the package directories contain the package manifest
+    // files and retrieve the package versions via the single `b info` call.
+    // While at it cache the manifest paths for the future use.
+    //
+    paths                     mfs;
+    vector<optional<version>> pvs;
     {
-      assert (sm.location);
+      mfs.reserve (pms.size ());
 
-      auto package_info = [&sm, &rl, &fragment] (diag_record& dr)
+      dir_paths pds;
+      pds.reserve (pms.size ());
+
+      for (const package_manifest& pm: pms)
       {
-        dr << "package ";
+        assert (pm.location);
 
-        if (!sm.location->current ())
-          dr << "'" << sm.location->string () << "' "; // Strip trailing '/'.
+        dir_path d (repo_dir / path_cast<dir_path> (*pm.location));
+        d.normalize (); // In case location is './'.
 
-        dr << "in repository " << rl;
+        path f (d / manifest_file);
+        if (!exists (f))
+        {
+          diag_record dr (fail);
+          dr << "no manifest file for ";
+          package_info (pm, dr);
+        }
 
-        if (fragment)
-          dr << ' ' << *fragment;
-      };
+        mfs.push_back (move (f));
+        pds.push_back (move (d));
+      }
 
-      auto failure = [&package_info] (const char* desc)
-      {
-        diag_record dr (fail);
-        dr << desc << " for ";
-        package_info (dr);
-      };
+      pvs = package_versions (co, pds);
+    }
 
-      dir_path d (repo_dir / path_cast<dir_path> (*sm.location));
-      d.normalize (); // In case location is './'.
+    // Parse package manifests, fixing up their versions.
+    //
+    vector<package_manifest> r;
+    r.reserve (pms.size ());
+
+    for (size_t i (0); i != pms.size (); ++i)
+    {
+      package_manifest& pm (pms[i]);
 
-      path f (d / manifest_file);
-      if (!exists (f))
-        failure ("no manifest file");
+      assert (pm.location);
+
+      const path& f (mfs[i]);
 
       try
       {
         ifdstream ifs (f);
         manifest_parser mp (ifs, f.string ());
 
+        optional<version>& pv (pvs[i]);
+
         package_manifest m (
           mp,
-          [&co, &d] (version& v)
+          [&pv] (version& v)
           {
-            if (optional<version> pv = package_version (co, d))
+            if (pv)
               v = move (*pv);
           },
           iu);
 
         // Save the package manifest, preserving its location.
         //
-        m.location = move (*sm.location);
-        sm = move (m);
+        m.location = move (*pm.location);
+        pm = move (m);
       }
       catch (const manifest_parsing& e)
       {
         diag_record dr (fail (e.name, e.line, e.column));
         dr << e.description << info;
-        package_info (dr);
+        package_info (pm, dr);
       }
       catch (const io_error& e)
       {
         fail << "unable to read from " << f << ": " << e;
       }
 
-      r.emplace_back (move (sm));
+      r.emplace_back (move (pm));
     }
 
     return r;
diff --git a/bpkg/types.hxx b/bpkg/types.hxx
index 1c6e89b..8e00666 100644
--- a/bpkg/types.hxx
+++ b/bpkg/types.hxx
@@ -95,8 +95,8 @@ namespace bpkg
 
   using butl::path_cast;
 
-  using paths = std::vector<path>;
-  using dir_paths = std::vector<dir_path>;
+  using paths = vector<path>;
+  using dir_paths = vector<dir_path>;
 
   // <libbutl/url.mxx>
   //