diff options
| -rw-r--r-- | bpkg/package.hxx | 24 | ||||
| -rw-r--r-- | bpkg/pkg-build.cli | 6 | ||||
| -rw-r--r-- | bpkg/pkg-build.cxx | 2 | ||||
| -rw-r--r-- | bpkg/pkg-configure.cxx | 13 | ||||
| -rw-r--r-- | bpkg/rep-fetch.cli | 7 | ||||
| -rw-r--r-- | bpkg/rep-fetch.cxx | 159 | ||||
| -rw-r--r-- | bpkg/rep-fetch.hxx | 9 | ||||
| -rw-r--r-- | tests/rep-fetch.test | 64 |
8 files changed, 200 insertions, 84 deletions
diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 3567b10..5905dd2 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -597,11 +597,12 @@ namespace bpkg bool purge_src; // Path to the output directory of this package, if any. It is - // always relative to the configuration directory and currently - // is always <name>-<version>. It is only set once the package - // is configured and its main purse is to keep track of what - // needs to be cleaned by the user before a broken package can - // be purged. Note that it could be the same as out_root. + // always relative to the configuration directory, and is <name> + // for external packages and <name>-<version> for others. It is + // only set once the package is configured and its main purse is + // to keep track of what needs to be cleaned by the user before + // a broken package can be purged. Note that it could be the + // same as src_root. // optional<dir_path> out_root; @@ -618,6 +619,19 @@ namespace bpkg return substate == package_substate::system; } + bool + external () const + { + return + // pkg-unpack <name>/<version> + // + (!repository.empty () && repository.directory_based ()) || + + // pkg-unpack --existing <dir> + // + (repository.empty () && !archive); + } + // Represent the wildcard version with the "*" string. Represent naturally // all other versions. // diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli index 3c5568f..a4d49ef 100644 --- a/bpkg/pkg-build.cli +++ b/bpkg/pkg-build.cli @@ -141,5 +141,11 @@ namespace bpkg "Print to \cb{STDOUT} what would be done without actually doing anything." } + + bool --fetch-shallow + { + "Do not re-fetch complement and prerequisite repositories. Refer to + the \cb{--shallow} option in \l{bpkg-rep-fetch(1)} for details." + } }; } diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index cc14f1e..8f80ff5 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -1178,7 +1178,7 @@ namespace bpkg } if (!locations.empty ()) - rep_fetch (o, c, db, locations); + rep_fetch (o, c, db, locations, o.fetch_shallow ()); // Expand <packages>@<location> arguments. // diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx index c839d42..aaa426f 100644 --- a/bpkg/pkg-configure.cxx +++ b/bpkg/pkg-configure.cxx @@ -127,16 +127,9 @@ namespace bpkg ? *p->src_root : c / *p->src_root); - const repository_location& rl (p->repository); - - // For external packages call the output directory <pkg>, rather than - // <pkg>-<ver>. - // - dir_path out_root ( - (!rl.empty () && rl.directory_based ()) || // pkg-unpack <name>/<version> - ( rl.empty () && !p->archive) // pkg-unpack --existing <dir> - ? c / dir_path (p->name) - : c / dir_path (p->name + "-" + p->version.string ())); + dir_path out_root (p->external () + ? c / dir_path (p->name) + : c / dir_path (p->name + "-" + p->version.string ())); l4 ([&]{trace << "src_root: " << src_root << ", " << "out_root: " << out_root;}); diff --git a/bpkg/rep-fetch.cli b/bpkg/rep-fetch.cli index 719232d..a39aafb 100644 --- a/bpkg/rep-fetch.cli +++ b/bpkg/rep-fetch.cli @@ -39,5 +39,12 @@ namespace bpkg class rep_fetch_options: configuration_options { "\h|REP-FETCH OPTIONS|" + + bool --shallow + { + "Do not re-fetch complement and prerequisite repositories of the + specified repositories unless the set of complements and/or + prerequisites has changed." + } }; } diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 8f1528c..1f0f893 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -5,6 +5,7 @@ #include <bpkg/rep-fetch.hxx> #include <set> +#include <algorithm> // equal() #include <libbutl/manifest-parser.mxx> @@ -431,6 +432,7 @@ namespace bpkg const shared_ptr<repository>& r, repositories& fetched, repositories& removed, + bool shallow, const string& reason = string ()) { tracer trace ("rep_fetch(rep)"); @@ -471,42 +473,19 @@ namespace bpkg dr << " (" << reason << ")"; } - // Register complements and prerequisites for potential removal unless - // they are fetched. Clear repository dependency sets afterwards. - // - auto remove = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp) - { - shared_ptr<repository> r (rp.load ()); - if (fetched.find (r) == fetched.end ()) - removed.insert (move (r)); - }; - - for (const lazy_shared_ptr<repository>& cr: r->complements) - { - // Remove the complement unless it is the root repository (see - // rep_fetch() for details). - // - if (cr.object_id () != "") - remove (cr); - } - - for (const lazy_weak_ptr<repository>& pr: r->prerequisites) - remove (lazy_shared_ptr<repository> (pr)); - - r->complements.clear (); - r->prerequisites.clear (); - - // Remove this repository from locations of the available packages it - // contains. - // - rep_remove_package_locations (t, r->name); - // Load the repository and package manifests and use them to populate the // prerequisite and complement repository sets as well as available // packages. // rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + // Create the new prerequisite and complement repository sets. While doing + // this we may also reset the shallow flag if discover that any of these + // sets have changed. + // + repository::complements_type complements; + repository::prerequisites_type prerequisites; + for (repository_manifest& rm: rfd.repositories) { repository_role rr (rm.effective_role ()); @@ -542,25 +521,16 @@ namespace bpkg { pr = make_shared<repository> (move (l)); db.persist (pr); // Enter into session, important if recursive. + + shallow = false; } else if (pr->location.url () != l.url ()) { pr->location = move (l); db.update (r); - } - // Load the prerequisite repository. - // - string reason; - switch (rr) - { - case repository_role::complement: reason = "complements "; break; - case repository_role::prerequisite: reason = "prerequisite of "; break; - case repository_role::base: assert (false); + shallow = false; } - reason += r->name; - - rep_fetch (co, conf, t, pr, fetched, removed, reason); // @@ What if we have duplicated? Ideally, we would like to check // this once and as early as possible. The original idea was to @@ -582,13 +552,13 @@ namespace bpkg case repository_role::complement: { l4 ([&]{trace << pr->name << " complement of " << r->name;}); - r->complements.insert (lazy_shared_ptr<repository> (db, pr)); + complements.insert (lazy_shared_ptr<repository> (db, pr)); break; } case repository_role::prerequisite: { l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); - r->prerequisites.insert (lazy_weak_ptr<repository> (db, pr)); + prerequisites.insert (lazy_weak_ptr<repository> (db, pr)); break; } case repository_role::base: @@ -610,8 +580,8 @@ namespace bpkg case repository_type::git: case repository_type::dir: { - if (r->complements.empty () && r->prerequisites.empty ()) - r->complements.insert (lazy_shared_ptr<repository> (db, string ())); + if (complements.empty () && prerequisites.empty ()) + complements.insert (lazy_shared_ptr<repository> (db, string ())); break; } @@ -624,9 +594,83 @@ namespace bpkg } } - // Save the changes to the repository object. + // Reset the shallow flag if the set of complements and/or prerequisites + // has changed. // - db.update (r); + // Note that weak pointers are generally incomparable (as can point to + // expired objects), and thus we can't compare the prerequisite sets + // directly. + // + if (shallow) + shallow = r->complements == complements && + equal (r->prerequisites.begin (), r->prerequisites.end (), + prerequisites.begin (), prerequisites.end (), + [] (const lazy_weak_ptr<repository>& x, + const lazy_weak_ptr<repository>& y) + { + return x.object_id () == y.object_id (); + }); + + // Fetch prerequisites and complements, unless this is a shallow fetch. + // + if (!shallow) + { + // Register complements and prerequisites for potential removal unless + // they are fetched. Clear repository dependency sets afterwards. + // + auto rm = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp) + { + shared_ptr<repository> r (rp.load ()); + if (fetched.find (r) == fetched.end ()) + removed.insert (move (r)); + }; + + for (const lazy_shared_ptr<repository>& cr: r->complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + rm (cr); + } + + for (const lazy_weak_ptr<repository>& pr: r->prerequisites) + rm (lazy_shared_ptr<repository> (pr)); + + r->complements = move (complements); + r->prerequisites = move (prerequisites); + + // Fetch complements. + // + for (const auto& cr: r->complements) + { + if (cr.object_id () != "") + rep_fetch (co, + conf, + t, + cr.load (), + fetched, + removed, + false /* shallow */, + "complements " + r->name); + } + + // Fetch prerequisites. + // + for (const auto& pr: r->prerequisites) + rep_fetch (co, + conf, + t, + pr.load (), + fetched, + removed, + false /* shallow */, + "prerequisite of " + r->name); + + // Save the changes to the repository object. + // + db.update (r); + } // "Suspend" session while persisting packages to reduce memory // consumption. @@ -634,6 +678,11 @@ namespace bpkg session& s (session::current ()); session::reset_current (); + // Remove this repository from locations of the available packages it + // contains. + // + rep_remove_package_locations (t, r->name); + for (rep_fetch_data::package& fp: rfd.packages) { package_manifest& pm (fp.manifest); @@ -702,7 +751,8 @@ namespace bpkg rep_fetch (const common_options& o, const dir_path& conf, transaction& t, - const vector<lazy_shared_ptr<repository>>& repos) + const vector<lazy_shared_ptr<repository>>& repos, + bool shallow) { // As a fist step we fetch repositories recursively building the list of // the former prerequisites and complements to be considered for removal. @@ -722,7 +772,7 @@ namespace bpkg repositories removed; for (const lazy_shared_ptr<repository>& r: repos) - rep_fetch (o, conf, t, r.load (), fetched, removed); + rep_fetch (o, conf, t, r.load (), fetched, removed, shallow); // Finally, remove dangling repositories. // @@ -752,8 +802,11 @@ namespace bpkg rep_fetch (const common_options& o, const dir_path& conf, database& db, - const vector<repository_location>& rls) + const vector<repository_location>& rls, + bool shallow) { + assert (session::has_current ()); + vector<lazy_shared_ptr<repository>> repos; repos.reserve (rls.size ()); @@ -775,7 +828,7 @@ namespace bpkg repos.emplace_back (r); } - rep_fetch (o, conf, t, repos); + rep_fetch (o, conf, t, repos, shallow); t.commit (); } @@ -846,7 +899,7 @@ namespace bpkg } } - rep_fetch (o, c, t, repos); + rep_fetch (o, c, t, repos, o.shallow ()); size_t rcount (0), pcount (0); if (verb) diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 030b192..18a50e9 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -51,14 +51,17 @@ namespace bpkg bool ignore_unknown); // Add (or update) repository locations to the configuration and fetch - // them. On failure clean up the configuration (see rep_remove_clean() for - // details). Note that it should be called in session. + // them. If shallow is true, then don't fetch their prerequisite and/or + // complements unless the respective sets have changed. On failure clean up + // the configuration (see rep_remove_clean() for details). Note that it + // should be called in session. // void rep_fetch (const common_options&, const dir_path& conf, database&, - const vector<repository_location>&); + const vector<repository_location>&, + bool shallow); } #endif // BPKG_REP_FETCH_HXX diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test index f0c0e2d..2339b91 100644 --- a/tests/rep-fetch.test +++ b/tests/rep-fetch.test @@ -142,19 +142,19 @@ $* 2>>/EOE != 0 $* --trust-yes 2>>EOE; fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) 5 package(s) in 5 repository(s) EOE $* 2>>EOE fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) 5 package(s) in 5 repository(s) EOE } @@ -166,20 +166,20 @@ $* 2>>/EOE != 0 $* --trust-yes 2>>EOE &cfg/.bpkg/certs/**; fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) fetching pkg:build2.org/rep-fetch/hello 6 package(s) in 6 repository(s) EOE $* 2>>EOE fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) fetching pkg:build2.org/rep-fetch/hello 6 package(s) in 6 repository(s) EOE @@ -193,19 +193,19 @@ $* 2>>/EOE != 0 $* --trust-yes $rep/bar/unstable 2>>EOE; added pkg:build2.org/rep-fetch/bar/unstable fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) 5 package(s) in 5 repository(s) EOE $* 'pkg:build2.org/rep-fetch/bar/unstable' 2>>EOE; fetching pkg:build2.org/rep-fetch/bar/unstable - fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable) - fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing) fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable) fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing) + fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable) + fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing) 5 package(s) in 5 repository(s) EOE @@ -369,6 +369,46 @@ if ($remote != true) prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git) EOO } + + : shallow + : + { + +$clone_cfg + + : respected + : + { + $clone_cfg; + $* 2>!; + + $* --shallow "dir:($rep/libbar.git)" 2>>"EOE"; + fetching dir:($rep/libbar.git) + 3 package\(s\) in 2 repository\(s\) + EOE + + $rep_list >>"EOO" + dir:($rep/libbar.git) ($rep/libbar.git) + prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git) + EOO + } + + : ignored + : + { + $clone_cfg; + + $* --shallow 2>>"EOE"; + fetching dir:($rep/libbar.git) + fetching dir:($rep/style-basic.git) \(prerequisite of dir:($rep/libbar.git)\) + 3 package\(s\) in 2 repository\(s\) + EOE + + $rep_list >>"EOO" + dir:($rep/libbar.git) ($rep/libbar.git) + prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git) + EOO + } + } } : git-rep |