From ee16762d8435ec1a6dfe4a82655cde5624bb2c38 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 12 Mar 2018 19:21:01 +0300 Subject: Add support for shallow fetch --- bpkg/package.hxx | 24 ++++++-- bpkg/pkg-build.cli | 6 ++ bpkg/pkg-build.cxx | 2 +- bpkg/pkg-configure.cxx | 13 +--- bpkg/rep-fetch.cli | 7 +++ bpkg/rep-fetch.cxx | 159 ++++++++++++++++++++++++++++++++----------------- bpkg/rep-fetch.hxx | 9 ++- 7 files changed, 148 insertions(+), 72 deletions(-) (limited to 'bpkg') diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 3567b10..5905dd2 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -597,11 +597,12 @@ namespace bpkg bool purge_src; // Path to the output directory of this package, if any. It is - // always relative to the configuration directory and currently - // is always -. It is only set once the package - // is configured and its main purse is to keep track of what - // needs to be cleaned by the user before a broken package can - // be purged. Note that it could be the same as out_root. + // always relative to the configuration directory, and is + // for external packages and - for others. It is + // only set once the package is configured and its main purse is + // to keep track of what needs to be cleaned by the user before + // a broken package can be purged. Note that it could be the + // same as src_root. // optional out_root; @@ -618,6 +619,19 @@ namespace bpkg return substate == package_substate::system; } + bool + external () const + { + return + // pkg-unpack / + // + (!repository.empty () && repository.directory_based ()) || + + // pkg-unpack --existing + // + (repository.empty () && !archive); + } + // Represent the wildcard version with the "*" string. Represent naturally // all other versions. // diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli index 3c5568f..a4d49ef 100644 --- a/bpkg/pkg-build.cli +++ b/bpkg/pkg-build.cli @@ -141,5 +141,11 @@ namespace bpkg "Print to \cb{STDOUT} what would be done without actually doing anything." } + + bool --fetch-shallow + { + "Do not re-fetch complement and prerequisite repositories. Refer to + the \cb{--shallow} option in \l{bpkg-rep-fetch(1)} for details." + } }; } diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index cc14f1e..8f80ff5 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -1178,7 +1178,7 @@ namespace bpkg } if (!locations.empty ()) - rep_fetch (o, c, db, locations); + rep_fetch (o, c, db, locations, o.fetch_shallow ()); // Expand @ arguments. // diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx index c839d42..aaa426f 100644 --- a/bpkg/pkg-configure.cxx +++ b/bpkg/pkg-configure.cxx @@ -127,16 +127,9 @@ namespace bpkg ? *p->src_root : c / *p->src_root); - const repository_location& rl (p->repository); - - // For external packages call the output directory , rather than - // -. - // - dir_path out_root ( - (!rl.empty () && rl.directory_based ()) || // pkg-unpack / - ( rl.empty () && !p->archive) // pkg-unpack --existing - ? c / dir_path (p->name) - : c / dir_path (p->name + "-" + p->version.string ())); + dir_path out_root (p->external () + ? c / dir_path (p->name) + : c / dir_path (p->name + "-" + p->version.string ())); l4 ([&]{trace << "src_root: " << src_root << ", " << "out_root: " << out_root;}); diff --git a/bpkg/rep-fetch.cli b/bpkg/rep-fetch.cli index 719232d..a39aafb 100644 --- a/bpkg/rep-fetch.cli +++ b/bpkg/rep-fetch.cli @@ -39,5 +39,12 @@ namespace bpkg class rep_fetch_options: configuration_options { "\h|REP-FETCH OPTIONS|" + + bool --shallow + { + "Do not re-fetch complement and prerequisite repositories of the + specified repositories unless the set of complements and/or + prerequisites has changed." + } }; } diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 8f1528c..1f0f893 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -5,6 +5,7 @@ #include #include +#include // equal() #include @@ -431,6 +432,7 @@ namespace bpkg const shared_ptr& r, repositories& fetched, repositories& removed, + bool shallow, const string& reason = string ()) { tracer trace ("rep_fetch(rep)"); @@ -471,42 +473,19 @@ namespace bpkg dr << " (" << reason << ")"; } - // Register complements and prerequisites for potential removal unless - // they are fetched. Clear repository dependency sets afterwards. - // - auto remove = [&fetched, &removed] (const lazy_shared_ptr& rp) - { - shared_ptr r (rp.load ()); - if (fetched.find (r) == fetched.end ()) - removed.insert (move (r)); - }; - - for (const lazy_shared_ptr& cr: r->complements) - { - // Remove the complement unless it is the root repository (see - // rep_fetch() for details). - // - if (cr.object_id () != "") - remove (cr); - } - - for (const lazy_weak_ptr& pr: r->prerequisites) - remove (lazy_shared_ptr (pr)); - - r->complements.clear (); - r->prerequisites.clear (); - - // Remove this repository from locations of the available packages it - // contains. - // - rep_remove_package_locations (t, r->name); - // Load the repository and package manifests and use them to populate the // prerequisite and complement repository sets as well as available // packages. // rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + // Create the new prerequisite and complement repository sets. While doing + // this we may also reset the shallow flag if discover that any of these + // sets have changed. + // + repository::complements_type complements; + repository::prerequisites_type prerequisites; + for (repository_manifest& rm: rfd.repositories) { repository_role rr (rm.effective_role ()); @@ -542,25 +521,16 @@ namespace bpkg { pr = make_shared (move (l)); db.persist (pr); // Enter into session, important if recursive. + + shallow = false; } else if (pr->location.url () != l.url ()) { pr->location = move (l); db.update (r); - } - // Load the prerequisite repository. - // - string reason; - switch (rr) - { - case repository_role::complement: reason = "complements "; break; - case repository_role::prerequisite: reason = "prerequisite of "; break; - case repository_role::base: assert (false); + shallow = false; } - reason += r->name; - - rep_fetch (co, conf, t, pr, fetched, removed, reason); // @@ What if we have duplicated? Ideally, we would like to check // this once and as early as possible. The original idea was to @@ -582,13 +552,13 @@ namespace bpkg case repository_role::complement: { l4 ([&]{trace << pr->name << " complement of " << r->name;}); - r->complements.insert (lazy_shared_ptr (db, pr)); + complements.insert (lazy_shared_ptr (db, pr)); break; } case repository_role::prerequisite: { l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); - r->prerequisites.insert (lazy_weak_ptr (db, pr)); + prerequisites.insert (lazy_weak_ptr (db, pr)); break; } case repository_role::base: @@ -610,8 +580,8 @@ namespace bpkg case repository_type::git: case repository_type::dir: { - if (r->complements.empty () && r->prerequisites.empty ()) - r->complements.insert (lazy_shared_ptr (db, string ())); + if (complements.empty () && prerequisites.empty ()) + complements.insert (lazy_shared_ptr (db, string ())); break; } @@ -624,9 +594,83 @@ namespace bpkg } } - // Save the changes to the repository object. + // Reset the shallow flag if the set of complements and/or prerequisites + // has changed. // - db.update (r); + // Note that weak pointers are generally incomparable (as can point to + // expired objects), and thus we can't compare the prerequisite sets + // directly. + // + if (shallow) + shallow = r->complements == complements && + equal (r->prerequisites.begin (), r->prerequisites.end (), + prerequisites.begin (), prerequisites.end (), + [] (const lazy_weak_ptr& x, + const lazy_weak_ptr& y) + { + return x.object_id () == y.object_id (); + }); + + // Fetch prerequisites and complements, unless this is a shallow fetch. + // + if (!shallow) + { + // Register complements and prerequisites for potential removal unless + // they are fetched. Clear repository dependency sets afterwards. + // + auto rm = [&fetched, &removed] (const lazy_shared_ptr& rp) + { + shared_ptr r (rp.load ()); + if (fetched.find (r) == fetched.end ()) + removed.insert (move (r)); + }; + + for (const lazy_shared_ptr& cr: r->complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + rm (cr); + } + + for (const lazy_weak_ptr& pr: r->prerequisites) + rm (lazy_shared_ptr (pr)); + + r->complements = move (complements); + r->prerequisites = move (prerequisites); + + // Fetch complements. + // + for (const auto& cr: r->complements) + { + if (cr.object_id () != "") + rep_fetch (co, + conf, + t, + cr.load (), + fetched, + removed, + false /* shallow */, + "complements " + r->name); + } + + // Fetch prerequisites. + // + for (const auto& pr: r->prerequisites) + rep_fetch (co, + conf, + t, + pr.load (), + fetched, + removed, + false /* shallow */, + "prerequisite of " + r->name); + + // Save the changes to the repository object. + // + db.update (r); + } // "Suspend" session while persisting packages to reduce memory // consumption. @@ -634,6 +678,11 @@ namespace bpkg session& s (session::current ()); session::reset_current (); + // Remove this repository from locations of the available packages it + // contains. + // + rep_remove_package_locations (t, r->name); + for (rep_fetch_data::package& fp: rfd.packages) { package_manifest& pm (fp.manifest); @@ -702,7 +751,8 @@ namespace bpkg rep_fetch (const common_options& o, const dir_path& conf, transaction& t, - const vector>& repos) + const vector>& repos, + bool shallow) { // As a fist step we fetch repositories recursively building the list of // the former prerequisites and complements to be considered for removal. @@ -722,7 +772,7 @@ namespace bpkg repositories removed; for (const lazy_shared_ptr& r: repos) - rep_fetch (o, conf, t, r.load (), fetched, removed); + rep_fetch (o, conf, t, r.load (), fetched, removed, shallow); // Finally, remove dangling repositories. // @@ -752,8 +802,11 @@ namespace bpkg rep_fetch (const common_options& o, const dir_path& conf, database& db, - const vector& rls) + const vector& rls, + bool shallow) { + assert (session::has_current ()); + vector> repos; repos.reserve (rls.size ()); @@ -775,7 +828,7 @@ namespace bpkg repos.emplace_back (r); } - rep_fetch (o, conf, t, repos); + rep_fetch (o, conf, t, repos, shallow); t.commit (); } @@ -846,7 +899,7 @@ namespace bpkg } } - rep_fetch (o, c, t, repos); + rep_fetch (o, c, t, repos, o.shallow ()); size_t rcount (0), pcount (0); if (verb) diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 030b192..18a50e9 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -51,14 +51,17 @@ namespace bpkg bool ignore_unknown); // Add (or update) repository locations to the configuration and fetch - // them. On failure clean up the configuration (see rep_remove_clean() for - // details). Note that it should be called in session. + // them. If shallow is true, then don't fetch their prerequisite and/or + // complements unless the respective sets have changed. On failure clean up + // the configuration (see rep_remove_clean() for details). Note that it + // should be called in session. // void rep_fetch (const common_options&, const dir_path& conf, database&, - const vector&); + const vector&, + bool shallow); } #endif // BPKG_REP_FETCH_HXX -- cgit v1.1