From cd315fecaf4c7473950c1610e06417634b416954 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 16 Aug 2021 18:54:15 +0300 Subject: Optimize rep-fetch for dir and git repos quering projects info in single call --- bpkg/manifest-utility.cxx | 55 +++++++++++++++++------------ bpkg/manifest-utility.hxx | 25 ++++++++----- bpkg/rep-fetch.cxx | 89 +++++++++++++++++++++++++++++++---------------- bpkg/types.hxx | 4 +-- 4 files changed, 110 insertions(+), 63 deletions(-) diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 6ca9e18..a4cee94 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -272,33 +272,39 @@ namespace bpkg } } - optional - package_version (const common_options& o, const dir_path& d) + vector> + package_versions (const common_options& o, const dir_paths& ds) { path b (name_b (o)); + vector pis; try { - b_project_info pi ( - b_info (d, - false /* ext_mods */, - verb, - [] (const char* const args[], size_t n) - { - if (verb >= 2) - print_process (args, n); - }, - b, - exec_dir, - o.build_option ())); - - optional r; - - // An empty version indicates that the version module is not enabled for - // the project. - // - if (!pi.version.empty ()) - r = version (pi.version.string ()); + b_info (pis, + ds, + false /* ext_mods */, + verb, + [] (const char* const args[], size_t n) + { + if (verb >= 2) + print_process (args, n); + }, + b, + exec_dir, + o.build_option ()); + + vector> r; + r.reserve (pis.size ()); + + for (const b_project_info& pi: pis) + { + // An empty version indicates that the version module is not enabled + // for the project. + // + r.push_back (!pi.version.empty () + ? version (pi.version.string ()) + : optional ()); + } return r; } @@ -307,7 +313,10 @@ namespace bpkg if (e.normal ()) throw failed (); // Assume the build2 process issued diagnostics. - fail << "unable to parse project " << d << " info: " << e << + diag_record dr (fail); + dr << "unable to parse project "; + if (pis.size () < ds.size ()) dr << ds[pis.size ()] << ' '; + dr << "info: " << e << info << "produced by '" << b << "'; use --build to override" << endf; } } diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index 29d548d..b7d9b07 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -95,20 +95,29 @@ namespace bpkg bool repository_name (const string&); - // Return the version of a package as provided by the build2 version module. - // Return nullopt if the version module is disabled for the package (or the - // build2 project directory doesn't contain the manifest file). Fail if the - // directory is not a build2 project. + // Return the versions of packages as provided by the build2 version module. + // Return nullopt for a package if the version module is disabled for it (or + // the build2 project directory doesn't contain the manifest file). Fail if + // any of the specified directories is not a build2 project. // - // Note that if the package directory is under the version control, then the + // Note that if a package directory is under the version control, then the // resulting version may be populated with the snapshot information (see // libbutl/standard-version.mxx for more details). Thus, this function can - // be used for fixing up the package manifest version. + // be used for fixing up the package manifest versions. // class common_options; - optional - package_version (const common_options&, const dir_path&); + vector> + package_versions (const common_options&, const dir_paths&); + + // As above but return the version of a single package. + // + inline optional + package_version (const common_options& o, const dir_path& d) + { + vector> r (package_versions (o, dir_paths ({d}))); + return move (r[0]); + } } #endif // BPKG_MANIFEST_UTILITY_HXX diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index f6165b7..80598c5 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -195,76 +195,105 @@ namespace bpkg static vector parse_package_manifests (const common_options& co, const dir_path& repo_dir, - vector&& sms, + vector&& pms, bool iu, const repository_location& rl, const optional& fragment) // For diagnostics. { - vector r; - r.reserve (sms.size ()); + auto package_info = [&rl, &fragment] (const package_manifest& pm, + diag_record& dr) + { + dr << "package "; + + if (!pm.location->current ()) + dr << "'" << pm.location->string () << "' "; // Strip trailing '/'. + + dr << "in repository " << rl; + + if (fragment) + dr << ' ' << *fragment; + }; - for (package_manifest& sm: sms) + // Verify that all the package directories contain the package manifest + // files and retrieve the package versions via the single `b info` call. + // While at it cache the manifest paths for the future use. + // + paths mfs; + vector> pvs; { - assert (sm.location); + mfs.reserve (pms.size ()); - auto package_info = [&sm, &rl, &fragment] (diag_record& dr) + dir_paths pds; + pds.reserve (pms.size ()); + + for (const package_manifest& pm: pms) { - dr << "package "; + assert (pm.location); - if (!sm.location->current ()) - dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. + dir_path d (repo_dir / path_cast (*pm.location)); + d.normalize (); // In case location is './'. - dr << "in repository " << rl; + path f (d / manifest_file); + if (!exists (f)) + { + diag_record dr (fail); + dr << "no manifest file for "; + package_info (pm, dr); + } - if (fragment) - dr << ' ' << *fragment; - }; + mfs.push_back (move (f)); + pds.push_back (move (d)); + } - auto failure = [&package_info] (const char* desc) - { - diag_record dr (fail); - dr << desc << " for "; - package_info (dr); - }; + pvs = package_versions (co, pds); + } - dir_path d (repo_dir / path_cast (*sm.location)); - d.normalize (); // In case location is './'. + // Parse package manifests, fixing up their versions. + // + vector r; + r.reserve (pms.size ()); + + for (size_t i (0); i != pms.size (); ++i) + { + package_manifest& pm (pms[i]); - path f (d / manifest_file); - if (!exists (f)) - failure ("no manifest file"); + assert (pm.location); + + const path& f (mfs[i]); try { ifdstream ifs (f); manifest_parser mp (ifs, f.string ()); + optional& pv (pvs[i]); + package_manifest m ( mp, - [&co, &d] (version& v) + [&pv] (version& v) { - if (optional pv = package_version (co, d)) + if (pv) v = move (*pv); }, iu); // Save the package manifest, preserving its location. // - m.location = move (*sm.location); - sm = move (m); + m.location = move (*pm.location); + pm = move (m); } catch (const manifest_parsing& e) { diag_record dr (fail (e.name, e.line, e.column)); dr << e.description << info; - package_info (dr); + package_info (pm, dr); } catch (const io_error& e) { fail << "unable to read from " << f << ": " << e; } - r.emplace_back (move (sm)); + r.emplace_back (move (pm)); } return r; diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 1c6e89b..8e00666 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -95,8 +95,8 @@ namespace bpkg using butl::path_cast; - using paths = std::vector; - using dir_paths = std::vector; + using paths = vector; + using dir_paths = vector; // // -- cgit v1.1