From a2b084651909929d58f6b4bc0f3c742d87ee31f6 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Fri, 27 Apr 2018 15:53:00 +0300 Subject: Add support for repository fragments --- bpkg/cfg-create.cxx | 25 +- bpkg/fetch-git.cxx | 164 ++++++++---- bpkg/fetch.hxx | 23 +- bpkg/forward.hxx | 1 + bpkg/manifest-utility.cxx | 16 +- bpkg/manifest-utility.hxx | 3 +- bpkg/package.cxx | 143 ++++++---- bpkg/package.hxx | 287 +++++++++++++------- bpkg/package.xml | 73 ++++-- bpkg/pkg-build.cxx | 292 ++++++++++++--------- bpkg/pkg-checkout.cxx | 33 ++- bpkg/pkg-configure.cxx | 6 +- bpkg/pkg-fetch.cxx | 22 +- bpkg/pkg-status.cxx | 7 +- bpkg/pkg-unpack.cxx | 16 +- bpkg/rep-add.cxx | 2 +- bpkg/rep-fetch.cxx | 655 +++++++++++++++++++++++++++++----------------- bpkg/rep-fetch.hxx | 21 +- bpkg/rep-info.cxx | 126 ++++++++- bpkg/rep-list.cxx | 52 ++-- bpkg/rep-remove.cxx | 177 ++++++++++--- bpkg/rep-remove.hxx | 17 +- 22 files changed, 1423 insertions(+), 738 deletions(-) (limited to 'bpkg') diff --git a/bpkg/cfg-create.cxx b/bpkg/cfg-create.cxx index b62163e..04d97f4 100644 --- a/bpkg/cfg-create.cxx +++ b/bpkg/cfg-create.cxx @@ -96,10 +96,31 @@ namespace bpkg // database db (open (c, trace, true)); - // Add the special, root repository object with empty location. + // Add the special, root repository object with empty location and + // containing a single repository fragment having an empty location as + // well. + // + // Note that the root repository serves as a complement for dir and git + // repositories that have neither prerequisites nor complements. The + // root repository fragment is used for transient available package + // locations and as a search starting point for held packages (see + // pkg-build for details). // transaction t (db); - db.persist (repository (repository_location ())); + + shared_ptr fr ( + make_shared (repository_location ())); + + db.persist (fr); + + shared_ptr r ( + make_shared (repository_location ())); + + r->fragments.push_back ( + repository::fragment_type {string () /* friendly_name */, move (fr)}); + + db.persist (r); + t.commit (); if (verb && !o.no_result ()) diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index e6836a6..9991075 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -5,7 +5,7 @@ #include #include -#include // find(), find_if(), replace() +#include // find(), find_if(), replace(), sort() #include // digit(), xdigit() #include @@ -864,18 +864,21 @@ namespace bpkg // Add a fragment to the resulting list, suppressing duplicates. // - auto add_frag = [&r] (const git_fragment& f) + // Note that the timestamp is set to zero. It is properly filled by the + // sort() lambda (see below). + // + auto add_frag = [&r] (const string& c, string n = string ()) { auto i (find_if (r.begin (), r.end (), - [&f] (const git_fragment& i) + [&c] (const git_fragment& i) { - return i.commit == f.commit; + return i.commit == c; })); if (i == r.end ()) - r.push_back (f); - else if (i->name.empty ()) - i->name = f.name; + r.push_back (git_fragment {c, 0 /* timestamp */, move (n)}); + else if (i->friendly_name.empty ()) + i->friendly_name = move (n); }; // Add a commit to the list for subsequent fetching. @@ -890,11 +893,9 @@ namespace bpkg // auto friendly_name = [] (const string& n) -> string { - return n.compare (0, 11, "refs/heads/") == 0 - ? "branch " + string (n, 11) - : n.compare (0, 10, "refs/tags/") == 0 - ? "tag " + string (n, 10) - : "reference " + n; + // Strip 'refs/' prefix if present. + // + return n.compare (0, 5, "refs/") == 0 ? string (n, 5) : n; }; if (rfs.empty ()) @@ -912,7 +913,7 @@ namespace bpkg // Add the commit to the resulting list. // - add_frag (git_fragment {rf.commit, friendly_name (rf.name)}); + add_frag (rf.commit, friendly_name (rf.name)); // Skip the commit if it is already fetched. // @@ -939,7 +940,7 @@ namespace bpkg // if (rf.commit && commit_fetched (co, dir, *rf.commit)) { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); continue; } @@ -951,7 +952,7 @@ namespace bpkg if (commit_fetched (co, dir, rfc.commit)) { - add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); + add_frag (rfc.commit, friendly_name (rfc.name)); continue; } } @@ -982,7 +983,7 @@ namespace bpkg const ref& rfc (reference (*rf.name, true /* abbr_commit */)); - add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); + add_frag (rfc.commit, friendly_name (rfc.name)); add_commit (rfc.commit, fetch_list); } // If commit is specified and the shallow fetch is possible, then we @@ -990,7 +991,7 @@ namespace bpkg // else if (shallow) { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (*rf.commit, fetch_list); } // If commit is specified and the shallow fetch is not possible, but @@ -1001,7 +1002,7 @@ namespace bpkg { // Note that commits we return and fetch are likely to differ. // - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (reference (*rf.name, false /* abbr_commit */).commit, fetch_list); @@ -1011,7 +1012,7 @@ namespace bpkg // else { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (*rf.commit, fetch_list); if (!commit_advertized (co, url, *rf.commit)) @@ -1020,10 +1021,50 @@ namespace bpkg } } + // Set timestamps for commits and sort them in the timestamp ascending + // order. + // + auto sort = [&co, &dir] (git_fragments&& frs) -> git_fragments + { + for (git_fragment& fr: frs) + { + // Add '^{commit}' suffix to strip some unwanted output that appears + // for tags. + // + string s (git_string (co, "commit timestamp", + co.git_option (), + "-C", dir, + "show", + "-s", + "--format=%ct", + fr.commit + "^{commit}")); + try + { + fr.timestamp = static_cast (stoull (s)); + } + // Catches both std::invalid_argument and std::out_of_range that + // inherit from std::logic_error. + // + catch (const logic_error&) + { + fail << "'" << s << "' doesn't appear to contain a git commit " + "timestamp" << endg; + } + } + + std::sort (frs.begin (), frs.end (), + [] (const git_fragment& x, const git_fragment& y) + { + return x.timestamp < y.timestamp; + }); + + return frs; + }; + // Bail out if all commits are already fetched. // if (scs.empty () && dcs.empty ()) - return r; + return sort (move (r)); auto fetch = [&co, &url, &dir] (const strings& refspecs, bool shallow) { @@ -1125,11 +1166,11 @@ namespace bpkg // for (auto& fr: r) { - if (fr.name.empty ()) - fr.name = "commit " + fr.commit.substr (0, 12); + if (fr.friendly_name.empty ()) + fr.friendly_name = fr.commit.substr (0, 12); } - return r; + return sort (move (r)); } // Checkout the repository submodules (see git_checkout_submodules() @@ -1377,50 +1418,62 @@ namespace bpkg init (co, dir, url); } - vector - git_fetch (const common_options& co, - const repository_location& rl, - const dir_path& dir) + // Update the repository remote origin URL, if changed. + // + static void + sync_origin_url (const common_options& co, + const repository_location& rl, + const dir_path& dir) { - git_ref_filters rfs; repository_url url (rl.url ()); + url.fragment = nullopt; - if (url.fragment) - try - { - rfs = parse_git_ref_filters (*url.fragment); - url.fragment = nullopt; - } - catch (const invalid_argument& e) - { - fail << "unable to fetch " << url << ": " << e; - } - - // Update the repository URL, if changed. - // repository_url u (origin_url (co, dir)); if (url != u) { - // Note that the repository canonical name can not change under the - // legal scenarios that lead to the location change. Changed canonical - // name means that the repository was manually amended. We could fix-up - // such repositories as well but want to leave the backdoor for tests. + // Note that the repository canonical name with the fragment part + // stripped can not change under the legal scenarios that lead to the + // location change. Changed canonical name means that the repository was + // manually amended. We could fix-up such repositories as well but want + // to leave the backdoor for tests. // - u.fragment = rl.url ().fragment; // Restore the fragment. - repository_location l (u, rl.type ()); - - if (rl.canonical_name () == l.canonical_name ()) + if (repository_location (url, rl.type ()).canonical_name () == + repository_location (u, rl.type ()).canonical_name ()) { if (verb) + { + u.fragment = rl.url ().fragment; // Restore the fragment. + info << "location changed for " << rl.canonical_name () << info << "new location " << rl << - info << "old location " << l; + info << "old location " << repository_location (u, rl.type ()); + } origin_url (co, dir, url); } } + } + vector + git_fetch (const common_options& co, + const repository_location& rl, + const dir_path& dir) + { + git_ref_filters rfs; + const repository_url& url (rl.url ()); + + if (url.fragment) + try + { + rfs = parse_git_ref_filters (*url.fragment); + } + catch (const invalid_argument& e) + { + fail << "unable to fetch " << url << ": " << e; + } + + sync_origin_url (co, rl, dir); return fetch (co, dir, dir_path () /* submodule */, rfs); } @@ -1462,8 +1515,17 @@ namespace bpkg } void - git_checkout_submodules (const common_options& co, const dir_path& dir) + git_checkout_submodules (const common_options& co, + const repository_location& rl, + const dir_path& dir) { + // Note that commits could come from different repository URLs that may + // contain different sets of commits. Thus, we need to switch to the URL + // the checked out commit came from to properly complete submodule + // relative URLs. + // + sync_origin_url (co, rl, dir); + checkout_submodules (co, dir, dir / dir_path (".git"), diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx index d05b8da..9a8efd0 100644 --- a/bpkg/fetch.hxx +++ b/bpkg/fetch.hxx @@ -5,6 +5,8 @@ #ifndef BPKG_FETCH_HXX #define BPKG_FETCH_HXX +#include // time_t + #include #include @@ -60,15 +62,21 @@ namespace bpkg // Fetch a git repository in the specifid directory (previously created by // git_init() for the references obtained with the repository URL fragment - // filters, returning commit ids these references resolve to. After fetching - // the repository working tree state is unspecified (see git_checkout ()). + // filters, returning commit ids these references resolve to in the earliest + // to latest order. Update the remote repository URL, if changed. After + // fetching the repository working tree state is unspecified (see + // git_checkout()). // // Note that submodules are not fetched. // struct git_fragment { - string commit; - string name; // User-friendly name (like 'branch foo', 'tag bar', ...). + // User-friendly fragment name is either a ref (tags/v1.2.3, heads/master, + // HEAD) or an abbreviated commit id (0123456789ab). + // + string commit; + std::time_t timestamp; + string friendly_name; }; vector @@ -86,10 +94,13 @@ namespace bpkg const string& commit); // Fetch (if necessary) and checkout submodules, recursively, in a working - // tree previously checked out by git_checkout(). + // tree previously checked out by git_checkout(). Update the remote + // repository URL, if changed. // void - git_checkout_submodules (const common_options&, const dir_path&); + git_checkout_submodules (const common_options&, + const repository_location&, + const dir_path&); // Low-level fetch API (fetch.cxx). // diff --git a/bpkg/forward.hxx b/bpkg/forward.hxx index 3845003..744c516 100644 --- a/bpkg/forward.hxx +++ b/bpkg/forward.hxx @@ -15,6 +15,7 @@ namespace bpkg // // class repository; + class repository_fragment; class selected_package; } diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 10a1512..0f64b30 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -137,15 +137,27 @@ namespace bpkg } dir_path - repository_state (const repository_location& l) + repository_state (const repository_location& rl) { - switch (l.type ()) + switch (rl.type ()) { case repository_type::pkg: case repository_type::dir: return dir_path (); // No state. case repository_type::git: { + // Strip the fragment, so all the repository fragments of the same + // git repository can reuse the state. So, for example, the state is + // shared for the fragments fetched from the following git repository + // locations: + // + // https://www.example.com/foo.git#master + // git://example.com/foo#stable + // + repository_url u (rl.url ()); + u.fragment = nullopt; + + repository_location l (u, rl.type ()); return dir_path (sha256 (l.canonical_name ()).abbreviated_string (12)); } } diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index e7be005..55b1244 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -67,8 +67,7 @@ namespace bpkg // Note that the semantics used to produce this name is repository type- // specific and can base on the repository canonical name or (potentially a // subset of) the location URL. In particular, a state directory could be - // shared by multiple repository locations of the same type (@@ TODO: if we - // ever do this, then we will need to complicate the removal logic). + // shared by multiple repository locations of the same type. // dir_path repository_state (const repository_location&); diff --git a/bpkg/package.cxx b/bpkg/package.cxx index 2880368..4880623 100644 --- a/bpkg/package.cxx +++ b/bpkg/package.cxx @@ -29,81 +29,110 @@ namespace bpkg // available_package // - // Check if the package is available from the specified repository, its - // prerequisite repositories, or one of their complements, recursively. - // Return the first repository that contains the package or NULL if none - // are. + // Check if the package is available from the specified repository fragment, + // its prerequisite repositories, or one of their complements, recursively. + // Return the first repository fragment that contains the package or NULL if + // none are. // // Note that we can end up with a repository dependency cycle since the - // root repository can be the default complement for git repositories (see - // rep_fetch() implementation for details). Thus we need to make sure that - // the repository is not in the dependency chain yet. + // root repository can be the default complement for dir and git + // repositories (see rep_fetch() implementation for details). Thus we need + // to make sure that the repository fragment is not in the dependency chain + // yet. // - using repositories = vector>>; + using repository_fragments = + vector>>; - static shared_ptr - find (const shared_ptr& r, + static shared_ptr + find (const shared_ptr& rf, const shared_ptr& ap, - repositories& chain, + repository_fragments& chain, bool prereq) { // Prerequisites are not searched through recursively. // assert (!prereq || chain.empty ()); - auto pr = [&r] (const shared_ptr& i) -> bool {return i == r;}; - auto i (find_if (chain.begin (), chain.end (), pr)); + auto i (find_if (chain.begin (), chain.end (), + [&rf] (const shared_ptr& i) -> bool + { + return i == rf; + })); if (i != chain.end ()) return nullptr; - chain.emplace_back (r); + chain.emplace_back (rf); - unique_ptr deleter ( - &chain, [] (repositories* r) {r->pop_back ();}); + unique_ptr deleter ( + &chain, [] (repository_fragments* rf) {rf->pop_back ();}); - const auto& ps (r->prerequisites); - const auto& cs (r->complements); + const auto& cs (rf->complements); + const auto& ps (rf->prerequisites); - // @@ The same repository can be present in the location set multiple times - // with different fragment values. Given that we may traverse the same - // repository tree multiple times, which is inefficient but harmless. - // Let's leave it this way for now as it likely to be changed with - // adding support for repository fragment objects. - // for (const package_location& pl: ap->locations) { - const lazy_shared_ptr& lr (pl.repository); + const lazy_shared_ptr& lrf (pl.repository_fragment); // First check the repository itself. // - if (lr.object_id () == r->name) - return r; + if (lrf.object_id () == rf->name) + return rf; - // Then check all the complements and prerequisites without - // loading them. + // Then check all the complements and prerequisites repository fragments + // without loading them. Though, we still need to load complement and + // prerequisite repositories. // - if (cs.find (lr) != cs.end () || (prereq && ps.find (lr) != ps.end ())) - return lr.load (); + auto pr = [&lrf] (const repository::fragment_type& i) + { + return i.fragment == lrf; + }; + + for (const lazy_shared_ptr& r: cs) + { + const auto& frs (r.load ()->fragments); + + if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) + return lrf.load (); + } + + if (prereq) + { + for (const lazy_weak_ptr& r: ps) + { + const auto& frs (r.load ()->fragments); + + if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) + return lrf.load (); + } + } // Finally, load the complements and prerequisites and check them // recursively. // for (const lazy_shared_ptr& cr: cs) { - // Should we consider prerequisites of our complements as our - // prerequisites? I'd say not. - // - if (shared_ptr r = find (cr.load (), ap, chain, false)) - return r; + for (const auto& fr: cr.load ()->fragments) + { + // Should we consider prerequisites of our complements as our + // prerequisites? I'd say not. + // + if (shared_ptr r = + find (fr.fragment.load (), ap, chain, false)) + return r; + } } if (prereq) { for (const lazy_weak_ptr& pr: ps) { - if (shared_ptr r = find (pr.load (), ap, chain, false)) - return r; + for (const auto& fr: pr.load ()->fragments) + { + if (shared_ptr r = + find (fr.fragment.load (), ap, chain, false)) + return r; + } } } } @@ -111,17 +140,17 @@ namespace bpkg return nullptr; } - shared_ptr - filter (const shared_ptr& r, + shared_ptr + filter (const shared_ptr& r, const shared_ptr& ap, bool prereq) { - repositories chain; + repository_fragments chain; return find (r, ap, chain, prereq); } vector> - filter (const shared_ptr& r, + filter (const shared_ptr& r, result&& apr, bool prereq) { @@ -136,34 +165,36 @@ namespace bpkg return aps; } - pair, shared_ptr> - filter_one (const shared_ptr& r, + pair, shared_ptr> + filter_one (const shared_ptr& r, result&& apr, bool prereq) { - using result = pair, shared_ptr>; + using result = pair, + shared_ptr>; for (shared_ptr ap: pointer_result (apr)) { - if (shared_ptr pr = filter (r, ap, prereq)) + if (shared_ptr pr = filter (r, ap, prereq)) return result (move (ap), move (pr)); } return result (); } - vector, shared_ptr>> - filter (const vector>& rps, + vector, shared_ptr>> + filter (const vector>& rps, odb::result&& apr, bool prereq) { - vector, shared_ptr>> aps; + vector, + shared_ptr>> aps; for (shared_ptr ap: pointer_result (apr)) { - for (const shared_ptr r: rps) + for (const shared_ptr r: rps) { - shared_ptr ar (filter (r, ap, prereq)); + shared_ptr ar (filter (r, ap, prereq)); if (ar != nullptr) { @@ -235,20 +266,20 @@ namespace bpkg if (check_external) { - using query = query; + using query = query; query q ( query::package::id.name == n && compare_version_eq (query::package::id.version, v, true, false)); - for (const auto& pr: db.query (q)) + for (const auto& prf: db.query (q)) { - const shared_ptr& r (pr.repository); + const shared_ptr& rf (prf.repository_fragment); - if (r->location.directory_based ()) + if (rf->location.directory_based ()) fail << "external package " << n << '/' << v << " is already available from " - << r->location.canonical_name (); + << rf->location.canonical_name (); } } diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 5b0db8c..ff05301 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -241,28 +241,62 @@ namespace bpkg : (?).type ()}) \ from(bpkg::repository_location (std::move ((?).url), (?).type)) - // repository + // repository_fragment + // + // Some repository types (normally version control-based) can be + // fragmented. For example, a git repository consists of multiple commits + // (fragments) which could contain different sets of packages and even + // prerequisite/complement repositories. Note also that the same fragment + // could be shared by multiple repository objects. We assume a fragment to + // be immutable, so it's complement, prerequisite and package sets can never + // change. + // + // For repository types that do not support fragmentation, there should + // be a single repository_fragment with the name and location equal to the + // ones of the containing repository. Such a fragment can not be shared but + // can be changed. // + // One of the consequences of the above is that a fragment can either be + // shared or be mutable. + // + class repository; + #pragma db object pointer(shared_ptr) session - class repository + class repository_fragment { public: - // We use a weak pointer for prerequisite repositories because we - // could have cycles. No cycles in complements, thought. + // We use a weak pointer for prerequisite repositories because we could + // have cycles. No cycles in complements, thought. + // + // Note that these point to repositories, not repository fragments. // using complements_type = std::set, compare_lazy_ptr>; using prerequisites_type = std::set, compare_lazy_ptr>; + // Repository fragment id is a repository canonical name that identifies + // just this fragment (for example, for git it is a canonical name of + // the repository URL with the full, non-abbreviated commit id). + // + // Note that while this works naturally for git where the fragment (full + // commit id) is also a valid fragment filter, it may not fit some future + // repository types. Let's deal with it when we see such a beast. + // string name; // Object id (canonical name). + + // For version control-based repositories it is used for a package + // checkout, that may involve communication with the remote repository. + // repository_location location; - complements_type complements; + + complements_type complements; prerequisites_type prerequisites; public: explicit - repository (repository_location l): location (move (l)) + repository_fragment (repository_location l) + : location (move (l)) { name = location.canonical_name (); } @@ -275,14 +309,66 @@ namespace bpkg set(this.location = std::move (?); \ assert (this.name == this.location.canonical_name ())) - #pragma db member(complements) id_column("repository") \ + #pragma db member(complements) id_column("repository_fragment") \ value_column("complement") value_not_null - #pragma db member(prerequisites) id_column("repository") \ + #pragma db member(prerequisites) id_column("repository_fragment") \ value_column("prerequisite") value_not_null private: friend class odb::access; + repository_fragment () = default; + }; + + #pragma db view object(repository_fragment) \ + query(repository_fragment::name != "" && (?)) + struct repository_fragment_count + { + #pragma db column("count(*)") + size_t result; + + operator size_t () const {return result;} + }; + + // repository + // + #pragma db object pointer(shared_ptr) session + class repository + { + public: + #pragma db value + struct fragment_type + { + string friendly_name; // User-friendly fragment name (e.g, tag, etc). + lazy_shared_ptr fragment; + }; + + using fragments_type = std::vector; + + string name; // Object id (canonical name). + repository_location location; + fragments_type fragments; + + public: + explicit + repository (repository_location l): location (move (l)) + { + name = location.canonical_name (); + } + + // Database mapping. + // + #pragma db member(name) id + + #pragma db member(location) column("") \ + set(this.location = std::move (?); \ + assert (this.name == this.location.canonical_name ())) + + #pragma db member(fragments) id_column("repository") \ + value_column("") value_not_null + + private: + friend class odb::access; repository () = default; }; @@ -300,18 +386,8 @@ namespace bpkg #pragma db value struct package_location { - using repository_type = bpkg::repository; - - // Fragment is optional, repository type-specific information that can be - // used to identify the repository fragment/partition/view/etc that this - // package came from. For example, for a version control-based repository - // this could be a commit id. - // - // The location is the package location within this repository fragment. - // - lazy_shared_ptr repository; - string fragment; - path location; + lazy_shared_ptr repository_fragment; + path location; // Package location within the repository fragment. }; // dependencies @@ -361,20 +437,21 @@ namespace bpkg available_package_id id; upstream_version version; - // List of repositories to which this package version belongs (yes, - // in our world, it can be in multiple, unrelated repositories). + // List of repository fragments to which this package version belongs + // (yes, in our world, it can be in multiple, unrelated repositories) + // together with locations within these repository fragments. // - // Note that if the repository is the special root repository (its - // location is empty), then this is a transient (or "fake") object - // for an existing package archive or package directory. In this - // case the location is the path to the archive/directory and to - // determine which one it is, use file/dir_exists(). While on the - // topic of fake available_package objects, when one is created for - // a selected package (see make_available()), this list is left empty - // with the thinking being that since the package is already in at - // least fetched state, we shouldn't be needing its location. + // Note that if the entry is the special root repository fragment (its + // location is empty), then this is a transient (or "fake") object for an + // existing package archive or package directory. In this case the + // location is the path to the archive/directory and to determine which + // one it is, use file/dir_exists(). While on the topic of fake + // available_package objects, when one is created for a selected package + // (see make_available()), this list is left empty with the thinking being + // that since the package is already in at least fetched state, we + // shouldn't be needing its location. // - vector locations; //@@ Map? + vector locations; // Package manifest data. // @@ -496,28 +573,28 @@ namespace bpkg operator size_t () const {return result;} }; - // Only return packages that are in the specified repositories, their + // Only return packages that are in the specified repository fragments, their // complements or prerequisites (if prereq is true), recursively. While you // could maybe come up with a (barely comprehensible) view/query to achieve // this, doing it on the "client side" is definitely more straightforward. // vector> - filter (const shared_ptr&, + filter (const shared_ptr&, odb::result&&, bool prereq = true); - pair, shared_ptr> - filter_one (const shared_ptr&, + pair, shared_ptr> + filter_one (const shared_ptr&, odb::result&&, bool prereq = true); - shared_ptr - filter (const shared_ptr&, + shared_ptr + filter (const shared_ptr&, const shared_ptr&, bool prereq = true); - vector, shared_ptr>> - filter (const vector>&, + vector, shared_ptr>> + filter (const vector>&, odb::result&&, bool prereq = true); @@ -606,41 +683,38 @@ namespace bpkg package_state state; package_substate substate; - // The hold flags indicate whether this package and/or version - // should be retained in the configuration. A held package will - // not be automatically removed. A held version will not be - // automatically upgraded. Note also that the two flags are - // orthogonal: we may want to keep a specific version of the - // package as long as it has dependents. + // The hold flags indicate whether this package and/or version should be + // retained in the configuration. A held package will not be automatically + // removed. A held version will not be automatically upgraded. Note also + // that the two flags are orthogonal: we may want to keep a specific + // version of the package as long as it has dependents. // bool hold_package; bool hold_version; - // Repository from which this package came. Note that it is not - // a pointer to the repository object because it could be wiped - // out (e.g., as a result of rep-fetch). We call such packages - // "orphans". While we can get a list of orphan's prerequisites - // (by loading its manifest), we wouldn't know which repository - // to use as a base to resolve them. As a result, an orphan that - // is not already configured (and thus has all its prerequisites - // resolved) is not very useful and can only be purged. + // Repository fragment from which this package came. Note that it is not a + // pointer to the repository_fragment object because it could be wiped out + // (e.g., as a result of rep-fetch). We call such packages "orphans". + // While we can get a list of orphan's prerequisites (by loading its + // manifest), we wouldn't know which repository fragment to use as a base + // to resolve them. As a result, an orphan that is not already configured + // (and thus has all its prerequisites resolved) is not very useful and + // can only be purged. // - repository_location repository; + repository_location repository_fragment; - // Path to the archive of this package, if any. If not absolute, - // then it is relative to the configuration directory. The purge - // flag indicates whether the archive should be removed when the - // packaged is purged. If the archive is not present, it should - // be false. + // Path to the archive of this package, if any. If not absolute, then it + // is relative to the configuration directory. The purge flag indicates + // whether the archive should be removed when the packaged is purged. If + // the archive is not present, it should be false. // optional archive; bool purge_archive; - // Path to the source directory of this package, if any. If not - // absolute, then it is relative to the configuration directory. - // The purge flag indicates whether the directory should be - // removed when the packaged is purged. If the source directory - // is not present, it should be false. + // Path to the source directory of this package, if any. If not absolute, + // then it is relative to the configuration directory. The purge flag + // indicates whether the directory should be removed when the packaged is + // purged. If the source directory is not present, it should be false. // optional src_root; bool purge_src; @@ -683,11 +757,12 @@ namespace bpkg return // pkg-unpack / // - (!repository.empty () && repository.directory_based ()) || + (!repository_fragment.empty () && + repository_fragment.directory_based ()) || // pkg-unpack --existing // - (repository.empty () && !archive); + (repository_fragment.empty () && !archive); } // Represent the wildcard version with the "*" string. Represent naturally @@ -899,45 +974,66 @@ namespace bpkg optional constraint; }; - // Return a list of repositories that depend on this repository as a - // complement. + // Return a count of repositories that contain this repository fragment. // - // Note that the last db object pragma is required to produce an object - // loading view. + #pragma db view table("repository_fragments") + struct fragment_repository_count + { + #pragma db column("count(*)") + size_t result; + + operator size_t () const {return result;} + }; + + // Return a list of repositories that contain this repository fragment. // - #pragma db view object(repository = complement) \ - table("repository_complements" = "rc" inner: \ - "rc.complement = " + complement::name) \ - object(repository inner: "rc.repository = " + repository::name) - struct repository_complement_dependent + #pragma db view object(repository) \ + table("repository_fragments" = "rfs" inner: \ + "rfs.repository = " + repository::name) \ + object(repository_fragment inner: "rfs.fragment = " + \ + repository_fragment::name) + struct fragment_repository { shared_ptr object; operator const shared_ptr () const {return object;} }; - // Return a list of repositories that depend on this repository as a - // prerequisite. + // Return a list of repository fragments that depend on this repository as a + // complement. // - // Note that the last db object pragma is required to produce an object - // loading view. + #pragma db view object(repository = complement) \ + table("repository_fragment_complements" = "rfc" inner: \ + "rfc.complement = " + complement::name) \ + object(repository_fragment inner: "rfc.repository_fragment = " + \ + repository_fragment::name) + struct repository_complement_dependent + { + shared_ptr object; + + operator const shared_ptr () const {return object;} + }; + + // Return a list of repository fragments that depend on this repository as a + // prerequisite. // - #pragma db view object(repository = prerequisite) \ - table("repository_prerequisites" = "rp" inner: \ - "rp.prerequisite = " + prerequisite::name) \ - object(repository inner: "rp.repository = " + repository::name) + #pragma db view object(repository = prerequisite) \ + table("repository_fragment_prerequisites" = "rfp" inner: \ + "rfp.prerequisite = " + prerequisite::name) \ + object(repository_fragment inner: "rfp.repository_fragment = " + \ + repository_fragment::name) struct repository_prerequisite_dependent { - shared_ptr object; + shared_ptr object; - operator const shared_ptr () const {return object;} + operator const shared_ptr () const {return object;} }; - // Return a list of packages available from this repository. + // Return a list of packages available from this repository fragment. // - #pragma db view object(repository) query(distinct) \ + #pragma db view object(repository_fragment) \ table("available_package_locations" = "pl" inner: \ - "pl.repository = " + repository::name) \ + "pl.repository_fragment = " + repository_fragment::name) \ object(available_package = package inner: \ "pl.name = " + package::id.name + "AND" + \ "pl.version_epoch = " + package::id.version.epoch + "AND" + \ @@ -947,18 +1043,18 @@ namespace bpkg package::id.version.canonical_release + "AND" + \ "pl.version_revision = " + package::id.version.revision + "AND" + \ "pl.version_iteration = " + package::id.version.iteration) - struct repository_package + struct repository_fragment_package { shared_ptr package; // Must match the alias (see above). operator const shared_ptr () const {return package;} }; - // Return a list of repositories the packages come from. + // Return a list of repository fragments the packages come from. // - #pragma db view object(repository) query(distinct) \ + #pragma db view object(repository_fragment) \ table("available_package_locations" = "pl" inner: \ - "pl.repository = " + repository::name) \ + "pl.repository_fragment = " + repository_fragment::name) \ object(available_package = package inner: \ "pl.name = " + package::id.name + "AND" + \ "pl.version_epoch = " + package::id.version.epoch + "AND" + \ @@ -968,13 +1064,12 @@ namespace bpkg package::id.version.canonical_release + "AND" + \ "pl.version_revision = " + package::id.version.revision + "AND" + \ "pl.version_iteration = " + package::id.version.iteration) - struct package_repository + struct package_repository_fragment { #pragma db column(package::id) available_package_id package_id; - using repository_type = bpkg::repository; - shared_ptr repository; + shared_ptr repository_fragment; }; // Version comparison operators. diff --git a/bpkg/package.xml b/bpkg/package.xml index 10014e7..a6cc552 100644 --- a/bpkg/package.xml +++ b/bpkg/package.xml @@ -1,6 +1,6 @@ - +
@@ -8,17 +8,17 @@
- - +
+ - - - + + + - - + + @@ -27,21 +27,53 @@
- - +
+ + + + + + + + + + + + + + + + +
+ + + + + + + +
+ + + + + - + - - - + + + + + + @@ -72,8 +104,7 @@ - - + @@ -99,9 +130,9 @@ - - - + + + @@ -209,8 +240,8 @@ - - + + diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index aea5420..9f3d732 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -9,7 +9,7 @@ #include #include // strlen() #include // cout -#include // find(), find_if() +#include // find_if() #include @@ -125,38 +125,37 @@ namespace bpkg return db.query (q); } - // Try to find a package that optionally satisfies the specified - // version constraint. Look in the specified repository, its - // prerequisite repositories, and their complements, recursively - // (note: recursivity applies to complements, not prerequisites). - // Return the package and the repository in which it was found or - // NULL for both if not found. Note that a stub satisfies any - // constraint. + // Try to find a package that optionally satisfies the specified version + // constraint. Look in the specified repository fragment, its prerequisite + // repositories, and their complements, recursively (note: recursivity + // applies to complements, not prerequisites). Return the package and the + // repository fragment in which it was found or NULL for both if not found. + // Note that a stub satisfies any constraint. // - static pair, shared_ptr> + static pair, shared_ptr> find_available (database& db, const string& name, - const shared_ptr& r, + const shared_ptr& rf, const optional& c, bool prereq = true) { - // Filter the result based on the repository to which each version - // belongs. + // Filter the result based on the repository fragment to which each + // version belongs. // - return filter_one (r, query_available (db, name, c), prereq); + return filter_one (rf, query_available (db, name, c), prereq); } // Create a transient (or fake, if you prefer) available_package object // corresponding to the specified selected object. Note that the package - // locations list is left empty and that the returned repository could be - // NULL if the package is an orphan. + // locations list is left empty and that the returned repository fragment + // could be NULL if the package is an orphan. // // Note also that in our model we assume that make_available() is only // called if there is no real available_package. This makes sure that if // the package moves (e.g., from testing to stable), then we will be using // stable to resolve its dependencies. // - static pair, shared_ptr> + static pair, shared_ptr> make_available (const common_options& options, const dir_path& c, database& db, @@ -168,16 +167,17 @@ namespace bpkg return make_pair (make_shared (sp->name, sp->version), nullptr); - // First see if we can find its repository. + // First see if we can find its repository fragment. // - // Note that this is package's "old" repository and there is no guarantee - // that its dependencies are still resolvable from it. But this is our - // best chance (we could go nuclear and point all orphans to the root - // repository but that feels a bit too drastic at the moment). + // Note that this is package's "old" repository fragment and there is no + // guarantee that its dependencies are still resolvable from it. But this + // is our best chance (we could go nuclear and point all orphans to the + // root repository fragment but that feels a bit too drastic at the + // moment). // - shared_ptr ar ( - db.find ( - sp->repository.canonical_name ())); + shared_ptr af ( + db.find ( + sp->repository_fragment.canonical_name ())); // The package is in at least fetched state, which means we should // be able to get its manifest. @@ -193,7 +193,7 @@ namespace bpkg // m.version = sp->version; - return make_pair (make_shared (move (m)), move (ar)); + return make_pair (make_shared (move (m)), move (af)); } // A "dependency-ordered" list of packages and their prerequisites. @@ -258,7 +258,10 @@ namespace bpkg shared_ptr selected; // NULL if not selected. shared_ptr available; // Can be NULL, fake/transient. - shared_ptr repository; // Can be NULL (orphan) or root. + + // Can be NULL (orphan) or root. + // + shared_ptr repository_fragment; const string& name () const @@ -440,8 +443,8 @@ namespace bpkg // Packages collection of whose prerequisites has been postponed due the // inability to find a version satisfying the pre-entered constraint from // repositories available to this package. The idea is that this - // constraint could still be satisfied from a repository of some other - // package (that we haven't processed yet) that also depends on this + // constraint could still be satisfied from a repository fragment of some + // other package (that we haven't processed yet) that also depends on this // prerequisite. // using postponed_packages = set; @@ -616,8 +619,9 @@ namespace bpkg // already configured since that would mean all its prerequisites are // configured as well. Note that this is not merely an optimization: the // package could be an orphan in which case the below logic will fail (no - // repository in which to search for prerequisites). By skipping the - // prerequisite check we are able to gracefully handle configured orphans. + // repository fragment in which to search for prerequisites). By skipping + // the prerequisite check we are able to gracefully handle configured + // orphans. // void collect_build_prerequisites (const common_options& options, @@ -649,7 +653,7 @@ namespace bpkg })); const shared_ptr& ap (pkg.available); - const shared_ptr& ar (pkg.repository); + const shared_ptr& af (pkg.repository_fragment); const string& name (ap->id.name); for (const dependency_alternatives& da: ap->dependencies) @@ -756,7 +760,9 @@ namespace bpkg // shared_ptr dsp (db.find (dn)); - pair, shared_ptr> rp; + pair, + shared_ptr> rp; + shared_ptr& dap (rp.first); bool force (false); @@ -781,7 +787,9 @@ namespace bpkg // system package we pick the latest one (its exact version // doesn't really matter). // - shared_ptr root (db.load ("")); + shared_ptr root ( + db.load ("")); + rp = system ? find_available (db, dn, root, nullopt) : find_available (db, @@ -807,11 +815,12 @@ namespace bpkg // if (dap == nullptr) { - // And if we have no repository to look in, then that means the - // package is an orphan (we delay this check until we actually - // need the repository to allow orphans without prerequisites). + // And if we have no repository fragment to look in, then that means + // the package is an orphan (we delay this check until we actually + // need the repository fragment to allow orphans without + // prerequisites). // - if (ar == nullptr) + if (af == nullptr) fail << "package " << pkg.available_name_version () << " is orphaned" << info << "explicitly upgrade it to a new version"; @@ -833,8 +842,8 @@ namespace bpkg // Also note that for the user-specified dependency version we rely // on its presence in repositories of the first dependent met. As // a result, we may fail too early if the version doesn't belong to - // its repository, but belongs to the one of some dependent that we - // haven't met yet. Can we just search all repositories for an + // its repositories, but belongs to the ones of some dependent that + // we haven't met yet. Can we just search all repositories for an // available package of this version and just take it, if present? // We could, but then which repository should we pick? The wrong // choice can introduce some unwanted repositories and package @@ -846,7 +855,7 @@ namespace bpkg // the package is recognized. An unrecognized package means the // broken/stale repository (see below). // - rp = find_available (db, dn, ar, !system ? d.constraint : nullopt); + rp = find_available (db, dn, af, !system ? d.constraint : nullopt); if (dap == nullptr) { @@ -868,8 +877,8 @@ namespace bpkg dr << " of package " << name; - if (!ar->location.empty () && (!dep_constr || system)) - dr << info << "repository " << ar->location << " appears to " + if (!af->location.empty () && (!dep_constr || system)) + dr << info << "repository " << af->location << " appears to " << "be broken" << info << "or the repository state could be stale" << info << "run 'bpkg rep-fetch' to update"; @@ -1395,7 +1404,7 @@ namespace bpkg return build_package { build_package::adjust, move (dsp), - nullptr, // No available package/repository. + nullptr, // No available package/repository fragment. nullptr, nullopt, // Hold package. nullopt, // Hold version. @@ -1515,8 +1524,8 @@ namespace bpkg // this dependency. If the result is a NULL available_package, then it is // either no longer used and can be dropped, or no changes to the dependency // are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository it must come from, and the - // system flag. + // upgrade/downgrade to as well as the repository fragment it must come + // from, and the system flag. // // If the explicitly specified dependency version can not be found in the // dependents repositories, then return the "no changes are necessary" @@ -1528,7 +1537,7 @@ namespace bpkg struct evaluate_result { shared_ptr available; - shared_ptr repository; + shared_ptr repository_fragment; bool unused; bool system; // Is meaningless if unused. }; @@ -1541,7 +1550,7 @@ namespace bpkg const shared_ptr&, const version& desired, bool desired_sys, - const set>&, + const set>&, const package_dependents&, bool ignore_unsatisfiable); @@ -1567,7 +1576,7 @@ namespace bpkg l5 ([&]{trace << *sp << ": unused";}); return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, true /* unused */, false /* system */}; } @@ -1598,15 +1607,16 @@ namespace bpkg l5 ([&]{trace << *sp << ": unchanged";}); return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, false /* unused */, false /* system */}; } - // Build a set of repositories the dependent packages now come from. Also - // cache the dependents and the constraints they apply to this dependency. + // Build a set of repository fragments the dependent packages now come + // from. Also cache the dependents and the constraints they apply to this + // dependency. // - set> repos; + set> repo_frags; package_dependents dependents; for (auto& pd: pds) @@ -1622,7 +1632,7 @@ namespace bpkg assert (!dap->locations.empty ()); for (const auto& pl: dap->locations) - repos.insert (pl.repository.load ()); + repo_frags.insert (pl.repository_fragment.load ()); } dependents.emplace_back (move (dsp), move (pd.constraint)); @@ -1632,7 +1642,7 @@ namespace bpkg sp, dv, dsys, - repos, + repo_frags, dependents, ignore_unsatisfiable); } @@ -1642,7 +1652,7 @@ namespace bpkg const shared_ptr& sp, const version& dv, bool dsys, - const set>& repos, + const set>& rfs, const package_dependents& dependents, bool ignore_unsatisfiable) { @@ -1658,19 +1668,22 @@ namespace bpkg ? query_available (db, nm, nullopt) : query_available (db, nm, dependency_constraint (dv))); - vector, shared_ptr>> ars ( - filter (vector> (repos.begin (), repos.end ()), - move (apr))); + vector, + shared_ptr>> afs ( + filter (vector> ( + rfs.begin (), + rfs.end ()), + move (apr))); auto no_change = [] () { return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, false /* unused */, false /* system */}; }; - if (ars.empty ()) + if (afs.empty ()) { if (ignore_unsatisfiable) { @@ -1706,9 +1719,9 @@ namespace bpkg assert (!dsys || system_repository.find (nm) != nullptr); - for (auto& ar: ars) + for (auto& af: afs) { - shared_ptr& ap (ar.first); + shared_ptr& ap (af.first); const version& av (!dsys ? ap->version : *ap->system_version ()); // If we aim to upgrade to the highest possible version and it tends to @@ -1771,7 +1784,7 @@ namespace bpkg << package_string (nm, av, dsys);}); return evaluate_result { - move (ap), move (ar.second), false /* unused */, dsys}; + move (ap), move (af.second), false /* unused */, dsys}; } // If we aim to upgrade to the highest possible version, then what we @@ -1875,7 +1888,8 @@ namespace bpkg // Evaluate a package (not necessarily dependency) and return a new desired // version. If the result is absent (nullopt), then no changes to the // package are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository it must come from. + // upgrade/downgrade to as well as the repository fragment it must come + // from. // // If the system package cannot be upgraded to the source one, not being // found in the dependents repositories, then return nullopt if @@ -1892,18 +1906,19 @@ namespace bpkg assert (sp != nullptr); - // Build a set of repositories the dependent packages come from. Also - // cache the dependents and the constraints they apply to this dependency. + // Build a set of repository fragment the dependent packages come from. + // Also cache the dependents and the constraints they apply to this + // dependency. // - set> repos; + set> repo_frags; package_dependents dependents; auto pds (db.query ( query::name == sp->name)); - // Only collect repositories (for best version selection) of (immediate) - // dependents that have a hit (direct or indirect) in recs. Note, however, - // that we collect constraints from all the dependents. + // Only collect repository fragments (for best version selection) of + // (immediate) dependents that have a hit (direct or indirect) in recs. + // Note, however, that we collect constraints from all the dependents. // bool upgrade (false); @@ -1916,8 +1931,8 @@ namespace bpkg continue; // While we already know that the dependency upgrade is required, we - // continue to iterate over dependents, collecting the repositories and - // the constraints. + // continue to iterate over dependents, collecting the repository + // fragments and the constraints. // upgrade = true; @@ -1930,7 +1945,7 @@ namespace bpkg assert (!dap->locations.empty ()); for (const auto& pl: dap->locations) - repos.insert (pl.repository.load ()); + repo_frags.insert (pl.repository_fragment.load ()); } } @@ -1947,7 +1962,7 @@ namespace bpkg sp, version () /* desired */, false /*desired_sys */, - repos, + repo_frags, dependents, ignore_unsatisfiable)); @@ -2374,14 +2389,21 @@ namespace bpkg // map pvs; - using query = query; - - for (const auto& rp: db.query ( - (query::repository::name == r->name) + - order_by_version_desc (query::package::id.version))) + for (const repository::fragment_type& rf: r->fragments) { - const shared_ptr& p (rp); - pvs.insert (make_pair (p->id.name, p->version)); + using query = query; + + for (const auto& rp: db.query ( + (query::repository_fragment::name == + rf.fragment.load ()->name) + + order_by_version_desc (query::package::id.version))) + { + const shared_ptr& p (rp); + auto i (pvs.insert (make_pair (p->id.name, p->version))); + + if (!i.second && i.first->second < p->version) + i.first->second = p->version; + } } // Populate the argument list with the latest package versions. @@ -2422,15 +2444,27 @@ namespace bpkg ? nullopt : optional (v)); - shared_ptr ap ( - find_available (db, n, r, c, false /* prereq */).first); + bool complements (false); + shared_ptr ap; + + for (const repository::fragment_type& rf: r->fragments) + { + shared_ptr fr (rf.fragment.load ()); + ap = find_available (db, n, fr, c, false /* prereq */).first; + + if (ap != nullptr) + break; + + if (!fr->complements.empty ()) + complements = true; + } if (ap == nullptr) { diag_record dr (fail); dr << "package " << pkg << " is not found in " << r->name; - if (!r->complements.empty ()) + if (complements) dr << " or its complements"; } @@ -2485,7 +2519,7 @@ namespace bpkg transaction t (db); - shared_ptr root (db.load ("")); + shared_ptr root (db.load ("")); // Here is what happens here: for unparsed package args we are going to // try and guess whether we are dealing with a package archive, package @@ -2504,7 +2538,7 @@ namespace bpkg // Reduce all the potential variations (archive, directory, package // name, package name/version) to a single available_package object. // - shared_ptr ar; + shared_ptr af; shared_ptr ap; if (!arg_parsed (pa)) @@ -2544,12 +2578,9 @@ namespace bpkg m.version, move (pa.options)); - ar = root; + af = root; ap = make_shared (move (m)); - ap->locations.push_back ( - package_location {root, - string () /* fragment */, - move (a)}); + ap->locations.push_back (package_location {root, move (a)}); } } catch (const invalid_path&) @@ -2596,12 +2627,12 @@ namespace bpkg l4 ([&]{trace << "directory '" << d << "': " << arg_string (pa);}); - // Supporting this would complicate things a bit, but we may add - // support for it one day. - // - if (pa.options.dependency ()) - fail << "package directory '" << d - << "' may not be built as a dependency"; + // Supporting this would complicate things a bit, but we may + // add support for it one day. + // + if (pa.options.dependency ()) + fail << "package directory '" << d + << "' may not be built as a dependency"; // Fix-up the package version to properly decide if we need to // upgrade/downgrade the package. Note that throwing failed @@ -2626,11 +2657,8 @@ namespace bpkg move (pa.options)); ap = make_shared (move (m)); - ar = root; - ap->locations.push_back ( - package_location {root, - string () /* fragment */, - move (d)}); + af = root; + ap->locations.push_back (package_location {root, move (d)}); } } catch (const invalid_path&) @@ -2691,7 +2719,7 @@ namespace bpkg root, dependency_constraint (pa.version))); ap = move (rp.first); - ar = move (rp.second); + af = move (rp.second); } } catch (const failed&) @@ -2895,7 +2923,7 @@ namespace bpkg auto rp (make_available (o, c, db, sp)); ap = rp.first; - ar = rp.second; // Could be NULL (orphan). + af = rp.second; // Could be NULL (orphan). } // We will keep the output directory only if the external package is @@ -2913,7 +2941,7 @@ namespace bpkg build_package::build, move (sp), move (ap), - move (ar), + move (af), true, // Hold package. !pa.version.empty (), // Hold version. {}, // Constraints. @@ -3068,9 +3096,13 @@ namespace bpkg { struct dep { - string name; // Empty if up/down-grade. - shared_ptr available; // NULL if drop. - shared_ptr repository; // NULL if drop. + string name; // Empty if up/down-grade. + + // Both are NULL if drop. + // + shared_ptr available; + shared_ptr repository_fragment; + bool system; }; vector deps; @@ -3103,8 +3135,8 @@ namespace bpkg build_package bp { nullopt, // Action. nullptr, // Selected package. - nullptr, // Available package. - nullptr, // Available package repository. + nullptr, // Available package/repository fragment. + nullptr, false, // Hold package. !p.version.empty (), // Hold version. {}, // Constraints. @@ -3175,14 +3207,14 @@ namespace bpkg build_package::build, move (sp), d.available, - d.repository, - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. + d.repository_fragment, + nullopt, // Hold package. + nullopt, // Hold version. + {}, // Constraints. d.system, keep_out, - {""}, // Required by (command line). - 0}; // Adjustments. + {""}, // Required by (command line). + 0}; // Adjustments. pkgs.collect_build (o, c, db, p, &postponed /* recursively */); } @@ -3373,7 +3405,7 @@ namespace bpkg if (!diag) deps.push_back (dep {sp->name, move (er->available), - move (er->repository), + move (er->repository_fragment), er->system}); r = true; } @@ -3752,7 +3784,7 @@ namespace bpkg const shared_ptr& ap (p.available); const package_location& pl (ap->locations[0]); - if (pl.repository.object_id () == "") // Special root. + if (pl.repository_fragment.object_id () == "") // Special root. p.keep_out = !exists (pl.location); // Directory case. else { @@ -3761,12 +3793,12 @@ namespace bpkg // See if the package comes from the directory-based repository, and // so is external. // - // Note that such repositories are always preferred over others (see - // below). + // Note that such repository fragments are always preferred over + // others (see below). // for (const package_location& l: ap->locations) { - if (l.repository.load ()->location.directory_based ()) + if (l.repository_fragment.load ()->location.directory_based ()) { p.keep_out = true; break; @@ -3875,20 +3907,22 @@ namespace bpkg // const package_location& pl (ap->locations[0]); // Got to have one. - if (pl.repository.object_id () != "") // Special root? + if (pl.repository_fragment.object_id () != "") // Special root? { transaction t (db, !simulate /* start */); - // Go through package repositories to decide if we should fetch, - // checkout or unpack depending on the available repository basis. - // Preferring a local one over the remotes and the dir repository - // type over the others seems like a sensible thing to do. + // Go through package repository fragments to decide if we should + // fetch, checkout or unpack depending on the available repository + // basis. Preferring a local one over the remotes and the dir + // repository type over the others seems like a sensible thing to + // do. // optional basis; for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl ( + l.repository_fragment.load ()->location); if (!basis || rl.local ()) // First or local? { @@ -3963,7 +3997,7 @@ namespace bpkg if (verbose && !o.no_result ()) { - const repository_location& rl (sp->repository); + const repository_location& rl (sp->repository_fragment); repository_basis basis ( !rl.empty () @@ -4016,7 +4050,7 @@ namespace bpkg else { const package_location& pl (ap->locations[0]); - assert (pl.repository.object_id () == ""); // Special root. + assert (pl.repository_fragment.object_id () == ""); // Special root. transaction t (db, !simulate /* start */); sp = pkg_unpack (o, diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx index 1759991..18ce2b9 100644 --- a/bpkg/pkg-checkout.cxx +++ b/bpkg/pkg-checkout.cxx @@ -27,14 +27,15 @@ namespace bpkg checkout (const common_options& o, const repository_location& rl, const dir_path& dir, - const string& fragment, const shared_ptr& ap) { switch (rl.type ()) { case repository_type::git: { - git_checkout (o, dir, fragment); + assert (rl.fragment ()); + + git_checkout (o, dir, *rl.fragment ()); if (exists (dir / path (".gitmodules"))) { @@ -45,7 +46,7 @@ namespace bpkg text << "checking out " << package_string (ap->id.name, ap->version); - git_checkout_submodules (o, dir); + git_checkout_submodules (o, rl, dir); } break; @@ -103,14 +104,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick a version control-based repository. Preferring a local one over - // the remotes seems like a sensible thing to do. + // Pick a version control-based repository fragment. Preferring a local + // one over the remotes seems like a sensible thing to do. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl (l.repository_fragment.load ()->location); if (rl.version_control_based () && (pl == nullptr || rl.local ())) { @@ -127,22 +128,18 @@ namespace bpkg if (verb > 1) text << "checking out " << pl->location.leaf () << " " - << "from " << pl->repository->name; - - const repository_location& rl (pl->repository->location); + << "from " << pl->repository_fragment->name; - // Note: for now we assume this is a git repository. If/when we add other - // version control-based repositories, this will need adjustment. + // Checkout the repository fragment. // + const repository_location& rl (pl->repository_fragment->location); - // Currently the git repository state already contains the checked out - // working tree so all we need to do is distribute it to the package - // directory. - // dir_path sd (c / repos_dir / repository_state (rl)); + checkout (o, rl, sd, ap); - checkout (o, rl, sd, pl->fragment, ap); - + // Calculate the package path that points into the checked out fragment + // directory. + // sd /= path_cast (pl->location); // Verify the package prerequisites are all configured since the dist @@ -219,7 +216,7 @@ namespace bpkg p->version = move (v); p->state = package_state::unpacked; - p->repository = rl; + p->repository_fragment = rl; p->src_root = d.leaf (); p->purge_src = true; p->manifest_checksum = move (mc); diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx index 7aa4183..e351cfd 100644 --- a/bpkg/pkg-configure.cxx +++ b/bpkg/pkg-configure.cxx @@ -208,7 +208,7 @@ namespace bpkg package_substate::system, false, // Don't hold package. false, // Don't hold version. - repository_location (), // Root repository. + repository_location (), // Root repository fragment. nullopt, // No source archive. false, // No auto-purge (does not get there). nullopt, // No source directory. @@ -278,12 +278,12 @@ namespace bpkg if (p != nullptr) fail << "package " << n << " already exists in configuration " << c; - shared_ptr rep (db.load ("")); // Root. + shared_ptr root (db.load ("")); using query = query; query q (query::id.name == n); - if (filter_one (rep, db.query (q)).first == nullptr) + if (filter_one (root, db.query (q)).first == nullptr) fail << "unknown package " << n; p = pkg_configure_system (n, v.empty () ? wildcard_version : v, t); diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx index a5efa47..36aaa21 100644 --- a/bpkg/pkg-fetch.cxx +++ b/bpkg/pkg-fetch.cxx @@ -58,7 +58,7 @@ namespace bpkg p->version = move (v); p->state = package_state::fetched; - p->repository = move (rl); + p->repository_fragment = move (rl); p->archive = move (a); p->purge_archive = purge; @@ -153,8 +153,8 @@ namespace bpkg // pkg_fetch_check (c, t, m.name, replace); - // Use the special root repository as the repository of this - // package. + // Use the special root repository fragment as the repository fragment of + // this package. // return pkg_fetch (c, t, @@ -197,14 +197,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick an archive-based repository. Preferring a local one over the - // remotes seems like a sensible thing to do. + // Pick an archive-based repository fragment. Preferring a local one over + // the remotes seems like a sensible thing to do. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl (l.repository_fragment.load ()->location); if (rl.archive_based () && (pl == nullptr || rl.local ())) { @@ -221,14 +221,16 @@ namespace bpkg if (verb > 1) text << "fetching " << pl->location.leaf () << " " - << "from " << pl->repository->name; + << "from " << pl->repository_fragment->name; auto_rmfile arm; path a (c / pl->location.leaf ()); if (!simulate) { - pkg_fetch_archive (co, pl->repository->location, pl->location, a); + pkg_fetch_archive ( + co, pl->repository_fragment->location, pl->location, a); + arm = auto_rmfile (a); // We can't be fetching an archive for a transient object. @@ -239,7 +241,7 @@ namespace bpkg if (sha256sum != *ap->sha256sum) { fail << "checksum mismatch for " << n << " " << v << - info << pl->repository->name << " has " << *ap->sha256sum << + info << pl->repository_fragment->name << " has " << *ap->sha256sum << info << "fetched archive has " << sha256sum << info << "consider re-fetching package list and trying again" << info << "if problem persists, consider reporting this to " @@ -253,7 +255,7 @@ namespace bpkg move (n), move (v), move (a), - pl->repository->location, + pl->repository_fragment->location, true /* purge */, simulate)); diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx index ea39b25..f621281 100644 --- a/bpkg/pkg-status.cxx +++ b/bpkg/pkg-status.cxx @@ -66,7 +66,8 @@ namespace bpkg bool known (false); bool build (false); { - shared_ptr rep (db.load ("")); // Root. + shared_ptr root ( + db.load ("")); using query = query; @@ -74,7 +75,7 @@ namespace bpkg { auto r (db.query (q)); known = !r.empty (); - build = filter_one (rep, move (r)).first != nullptr; + build = filter_one (root, move (r)).first != nullptr; } if (known) @@ -109,7 +110,7 @@ namespace bpkg pointer_result ( db.query (q))) { - bool build (filter (rep, ap)); + bool build (filter (root, ap)); apkgs.push_back (apkg {move (ap), build}); } } diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx index ace8d53..984a41e 100644 --- a/bpkg/pkg-unpack.cxx +++ b/bpkg/pkg-unpack.cxx @@ -106,7 +106,7 @@ namespace bpkg p->version = move (v); p->state = package_state::unpacked; - p->repository = move (rl); + p->repository_fragment = move (rl); p->src_root = move (d); p->purge_src = purge; p->manifest_checksum = move (mc); @@ -172,8 +172,8 @@ namespace bpkg o, c, t, d, m.name, m.version, true /* check_external */)) m.version = move (*v); - // Use the special root repository as the repository of this - // package. + // Use the special root repository fragment as the repository fragment of + // this package. // return pkg_unpack (o, c, @@ -215,14 +215,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick a directory-based repository. They are always local, so we pick - // the first one. + // Pick a directory-based repository fragment. They are always local, so we + // pick the first one. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - if (l.repository.load ()->location.directory_based ()) + if (l.repository_fragment.load ()->location.directory_based ()) { pl = &l; break; @@ -235,9 +235,9 @@ namespace bpkg if (verb > 1) text << "unpacking " << pl->location.leaf () << " " - << "from " << pl->repository->name; + << "from " << pl->repository_fragment->name; - const repository_location& rl (pl->repository->location); + const repository_location& rl (pl->repository_fragment->location); return pkg_unpack (o, c, diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx index 3569584..280c780 100644 --- a/bpkg/rep-add.cxx +++ b/bpkg/rep-add.cxx @@ -40,7 +40,7 @@ namespace bpkg updated = true; } - shared_ptr root (db.load ("")); + shared_ptr root (db.load ("")); bool added ( root->complements.insert (lazy_shared_ptr (db, r)).second); diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index c92db10..af07cb5 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -52,13 +52,14 @@ namespace bpkg pair rmc ( pkg_fetch_repositories (co, rl, ignore_unknown)); - pkg_repository_manifests& rms (rmc.first); + rep_fetch_data::fragment fr; + fr.repositories = move (rmc.first); bool a (co.auth () != auth::none && (co.auth () == auth::all || rl.remote ())); shared_ptr cert; - const optional& cert_pem (rms.back ().certificate); + optional cert_pem (move (fr.repositories.back ().certificate)); if (a) { @@ -79,6 +80,8 @@ namespace bpkg << rl.canonical_name () << info << "try again"; + fr.packages = move (pms); + if (a) { signature_manifest sm ( @@ -93,15 +96,7 @@ namespace bpkg authenticate_repository (co, conf, cert_pem, *cert, sm, rl); } - vector fps; - fps.reserve (pms.size ()); - - for (package_manifest& m: pms) - fps.emplace_back ( - rep_fetch_data::package {move (m), - string () /* repository_state */}); - - return rep_fetch_data {move (rms), move (fps), move (cert)}; + return rep_fetch_data {{move (fr)}, move (cert_pem), move (cert)}; } template @@ -109,7 +104,7 @@ namespace bpkg parse_manifest (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional& fragment) // Used for diagnostics. { try { @@ -119,8 +114,15 @@ namespace bpkg } catch (const manifest_parsing& e) { - fail (e.name, e.line, e.column) << e.description << - info << "repository " << rl << ' ' << frag << endf; + diag_record dr (fail (e.name, e.line, e.column)); + + dr << e.description << + info << "repository " << rl; + + if (fragment) + dr << ' ' << *fragment; + + dr << endf; } catch (const io_error& e) { @@ -136,11 +138,11 @@ namespace bpkg parse_repository_manifests (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional& fragment) { M r; if (exists (f)) - r = parse_manifest (f, iu, rl, frag); + r = parse_manifest (f, iu, rl, fragment); else r.emplace_back (repository_manifest ()); // Add the base repository. @@ -156,11 +158,11 @@ namespace bpkg parse_directory_manifests (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional& fragment) { M r; if (exists (f)) - r = parse_manifest (f, iu, rl, frag); + r = parse_manifest (f, iu, rl, fragment); else { r.push_back (package_manifest ()); @@ -172,30 +174,32 @@ namespace bpkg // Parse package manifests referenced by the package directory manifests. // - static vector + static vector parse_package_manifests (const common_options& co, const dir_path& repo_dir, - const string& repo_fragment, vector&& sms, bool iu, const repository_location& rl, - const string& frag) + const optional& fragment) // For diagnostics. { - vector fps; - fps.reserve (sms.size ()); + vector r; + r.reserve (sms.size ()); for (package_manifest& sm: sms) { assert (sm.location); - auto package_info = [&sm, &rl, &frag] (diag_record& dr) + auto package_info = [&sm, &rl, &fragment] (diag_record& dr) { dr << "package "; if (!sm.location->current ()) dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. - dr << "in repository " << rl << ' ' << frag; + dr << "in repository " << rl; + + if (fragment) + dr << ' ' << *fragment; }; auto failure = [&package_info] (const char* desc) @@ -240,10 +244,10 @@ namespace bpkg if (v) sm.version = move (*v); - fps.emplace_back (rep_fetch_data::package {move (sm), repo_fragment}); + r.emplace_back (move (sm)); } - return fps; + return r; } static rep_fetch_data @@ -255,30 +259,31 @@ namespace bpkg dir_path rd (path_cast (rl.path ())); - dir_repository_manifests rms ( - parse_repository_manifests ( - rd / repositories_file, - ignore_unknown, - rl, - string () /* frag */)); + rep_fetch_data::fragment fr; + + fr.repositories = parse_repository_manifests ( + rd / repositories_file, + ignore_unknown, + rl, + string () /* fragment */); dir_package_manifests pms ( parse_directory_manifests ( rd / packages_file, ignore_unknown, rl, - string () /* frag */)); - - vector fps ( - parse_package_manifests (co, - rd, - string () /* repo_fragment */, - move (pms), - ignore_unknown, - rl, - string () /* frag */)); - - return rep_fetch_data {move (rms), move (fps), nullptr}; + string () /* fragment */)); + + fr.packages = parse_package_manifests (co, + rd, + move (pms), + ignore_unknown, + rl, + string () /* fragment */); + + return rep_fetch_data {{move (fr)}, + nullopt /* cert_pem */, + nullptr /* certificate */}; } static rep_fetch_data @@ -358,21 +363,23 @@ namespace bpkg // - For each package location parse the package manifest and add it to // the resulting list. // - git_repository_manifests rms; - vector fps; + rep_fetch_data r; - for (const git_fragment& fr: git_fetch (co, rl, td)) + for (git_fragment& gf: git_fetch (co, rl, td)) { - git_checkout (co, td, fr.commit); + git_checkout (co, td, gf.commit); + + rep_fetch_data::fragment fr; + fr.id = move (gf.commit); + fr.friendly_name = move (gf.friendly_name); // Parse repository manifests. // - if (rms.empty ()) - rms = parse_repository_manifests ( + fr.repositories = parse_repository_manifests ( td / repositories_file, ignore_unknown, rl, - fr.name); + fr.friendly_name); // Parse package skeleton manifests. // @@ -381,7 +388,7 @@ namespace bpkg td / packages_file, ignore_unknown, rl, - fr.name)); + fr.friendly_name)); // Checkout submodules, if required. // @@ -391,25 +398,20 @@ namespace bpkg if (!exists (d) || empty (d)) { - git_checkout_submodules (co, td); + git_checkout_submodules (co, rl, td); break; } } // Parse package manifests. // - vector cps ( - parse_package_manifests (co, - td, - fr.commit, - move (pms), - ignore_unknown, - rl, - fr.name)); - - fps.insert (fps.end (), - make_move_iterator (cps.begin ()), - make_move_iterator (cps.end ())); + fr.packages = parse_package_manifests (co, + td, + move (pms), + ignore_unknown, + rl, + fr.friendly_name); + r.fragments.push_back (move (fr)); } // Move the state directory to its proper place. @@ -424,7 +426,7 @@ namespace bpkg filesystem_state_changed = true; } - return rep_fetch_data {move (rms), move (fps), nullptr /* certificate */}; + return r; } rep_fetch_data @@ -444,72 +446,89 @@ namespace bpkg return rep_fetch_data (); } - using repositories = set>; - - // If reason is absent, then don't print the "fetching ..." progress line. + // Return an existing repository fragment or create a new one. Update the + // existing object unless it is immutable (see repository_fragment class + // description for details). Don't fetch the complement and prerequisite + // repositories. // - static void - rep_fetch (const common_options& co, - const dir_path& conf, - transaction& t, - const shared_ptr& r, - repositories& fetched, - repositories& removed, - bool shallow, - const optional& reason) + static shared_ptr + rep_fragment (const common_options& co, + const dir_path& conf, + transaction& t, + const repository_location& rl, + rep_fetch_data::fragment&& fr) { - tracer trace ("rep_fetch(rep)"); + tracer trace ("rep_fragment"); database& db (t.database ()); tracer_guard tg (db, trace); - // Check that the repository is not fetched yet and register it as fetched - // otherwise. - // - // Note that we can end up with a repository dependency cycle via - // prerequisites. Thus we register the repository before recursing into its - // dependencies. - // - if (!fetched.insert (r).second) // Is already fetched. - return; + bool mut (fr.id.empty ()); // Is the fragment mutable? - const repository_location& rl (r->location); - l4 ([&]{trace << r->name << " " << rl;}); - - // Cancel the repository removal. - // - // Note that this is an optimization as the rep_remove() function checks - // for reachability of the repository being removed. + // Calculate the fragment location. // - removed.erase (r); + repository_location rfl; - // The fetch_*() functions below will be quiet at level 1, which can be - // quite confusing if the download hangs. - // - if (verb && reason) + switch (rl.type ()) { - diag_record dr (text); + case repository_type::pkg: + case repository_type::dir: + { + assert (mut); - dr << "fetching " << r->name; + rfl = rl; + break; + } + case repository_type::git: + { + assert (!mut); - if (!reason->empty ()) - dr << " (" << *reason << ")"; + repository_url url (rl.url ()); + url.fragment = move (fr.id); + + rfl = repository_location (url, rl.type ()); + break; + } } - // Load the repository and package manifests and use them to populate the - // prerequisite and complement repository sets as well as available - // packages. + shared_ptr rf ( + db.find (rfl.canonical_name ())); + + // Return the existing repository fragment if it is immutable. // - rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + bool exists (rf != nullptr); + + if (exists) + { + // Note that the user could change the repository URL the fragment was + // fetched from. In this case we need to sync the fragment location with + // the repository location to make sure that we use a proper URL for the + // fragment checkout. Not doing so we, for example, may try to fetch git + // submodules from the URL that is not available anymore. + // + if (rfl.url () != rf->location.url ()) + { + rf->location = move (rfl); + db.update (rf); + } - // Create the new prerequisite and complement repository sets. While doing - // this we may also reset the shallow flag if discover that any of these - // sets have changed. + if (!mut) + return rf; + } + + // Create or update the repository fragment. // - repository::complements_type complements; - repository::prerequisites_type prerequisites; + if (exists) + { + assert (mut); + + rf->complements.clear (); + rf->prerequisites.clear (); + } + else + rf = make_shared (move (rfl)); - for (repository_manifest& rm: rfd.repositories) + for (repository_manifest& rm: fr.repositories) { repository_role rr (rm.effective_role ()); @@ -535,27 +554,34 @@ namespace bpkg } } - // Create the new repository if it is not in the database yet. Otherwise - // update its location. + // Create the new repository if it is not in the database yet, otherwise + // update its location if it is changed, unless the repository is a + // top-level one (and so its location is authoritative). Such a change + // may root into the top-level repository location change made by the + // user. // - shared_ptr pr (db.find (l.canonical_name ())); + shared_ptr r (db.find (l.canonical_name ())); - if (pr == nullptr) + if (r == nullptr) { - pr = make_shared (move (l)); - db.persist (pr); // Enter into session, important if recursive. - - shallow = false; + r = make_shared (move (l)); + db.persist (r); // Enter into session, important if recursive. } - else if (pr->location.url () != l.url ()) + else if (r->location.url () != l.url ()) { - pr->location = move (l); - db.update (r); + shared_ptr root ( + db.load ("")); + + repository_fragment::complements_type& ua (root->complements); - shallow = false; + if (ua.find (r) == ua.end ()) + { + r->location = move (l); + db.update (r); + } } - // @@ What if we have duplicated? Ideally, we would like to check + // @@ What if we have duplicates? Ideally, we would like to check // this once and as early as possible. The original idea was to // do it during manifest parsing and serialization. But at that // stage we have no way of completing relative locations (which @@ -574,14 +600,14 @@ namespace bpkg { case repository_role::complement: { - l4 ([&]{trace << pr->name << " complement of " << r->name;}); - complements.insert (lazy_shared_ptr (db, pr)); + l4 ([&]{trace << r->name << " complement of " << rf->name;}); + rf->complements.insert (lazy_shared_ptr (db, r)); break; } case repository_role::prerequisite: { - l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); - prerequisites.insert (lazy_weak_ptr (db, pr)); + l4 ([&]{trace << r->name << " prerequisite of " << rf->name;}); + rf->prerequisites.insert (lazy_weak_ptr (db, r)); break; } case repository_role::base: @@ -603,8 +629,8 @@ namespace bpkg case repository_type::git: case repository_type::dir: { - if (complements.empty () && prerequisites.empty ()) - complements.insert (lazy_shared_ptr (db, string ())); + if (rf->complements.empty () && rf->prerequisites.empty ()) + rf->complements.insert (lazy_shared_ptr (db, string ())); break; } @@ -617,83 +643,10 @@ namespace bpkg } } - // Reset the shallow flag if the set of complements and/or prerequisites - // has changed. - // - // Note that weak pointers are generally incomparable (as can point to - // expired objects), and thus we can't compare the prerequisite sets - // directly. - // - if (shallow) - shallow = r->complements == complements && - equal (r->prerequisites.begin (), r->prerequisites.end (), - prerequisites.begin (), prerequisites.end (), - [] (const lazy_weak_ptr& x, - const lazy_weak_ptr& y) - { - return x.object_id () == y.object_id (); - }); - - // Fetch prerequisites and complements, unless this is a shallow fetch. - // - if (!shallow) - { - // Register complements and prerequisites for potential removal unless - // they are fetched. Clear repository dependency sets afterwards. - // - auto rm = [&fetched, &removed] (const lazy_shared_ptr& rp) - { - shared_ptr r (rp.load ()); - if (fetched.find (r) == fetched.end ()) - removed.insert (move (r)); - }; - - for (const lazy_shared_ptr& cr: r->complements) - { - // Remove the complement unless it is the root repository (see - // rep_fetch() for details). - // - if (cr.object_id () != "") - rm (cr); - } - - for (const lazy_weak_ptr& pr: r->prerequisites) - rm (lazy_shared_ptr (pr)); - - r->complements = move (complements); - r->prerequisites = move (prerequisites); - - // Fetch complements. - // - for (const auto& cr: r->complements) - { - if (cr.object_id () != "") - rep_fetch (co, - conf, - t, - cr.load (), - fetched, - removed, - false /* shallow */, - "complements " + r->name); - } - - // Fetch prerequisites. - // - for (const auto& pr: r->prerequisites) - rep_fetch (co, - conf, - t, - pr.load (), - fetched, - removed, - false /* shallow */, - "prerequisite of " + r->name); - - // Save the changes to the repository object. - // - db.update (r); - } + if (exists) + db.update (rf); + else + db.persist (rf); // "Suspend" session while persisting packages to reduce memory // consumption. @@ -701,15 +654,14 @@ namespace bpkg session& s (session::current ()); session::reset_current (); - // Remove this repository from locations of the available packages it - // contains. + // Remove this repository fragment from locations of the available + // packages it contains. // - rep_remove_package_locations (t, r->name); + if (exists) + rep_remove_package_locations (t, rf->name); - for (rep_fetch_data::package& fp: rfd.packages) + for (package_manifest& pm: fr.packages) { - package_manifest& pm (fp.manifest); - // Fix-up the external package version iteration number. // if (rl.directory_based ()) @@ -724,7 +676,7 @@ namespace bpkg co, conf, t, - path_cast (rl.path () / *fp.manifest.location), + path_cast (rl.path () / *pm.location), pm.name, pm.version, false /* check_external */)); @@ -765,7 +717,7 @@ namespace bpkg // can pick any to show to the user. // const string& r1 (rl.canonical_name ()); - const string& r2 (p->locations[0].repository.object_id ()); + const string& r2 (p->locations[0].repository_fragment.object_id ()); fail << "checksum mismatch for " << pm.name << " " << pm.version << info << r1 << " has " << *pm.sha256sum << @@ -775,12 +727,8 @@ namespace bpkg } } - // Note that the repository may already be present in the location set - // multiple times with different fragments. - // p->locations.push_back ( - package_location {lazy_shared_ptr (db, r), - move (fp.repository_fragment), + package_location {lazy_shared_ptr (db, rf), move (*pm.location)}); if (persist) @@ -790,6 +738,203 @@ namespace bpkg } session::current (s); // "Resume". + + return rf; + } + + using repositories = set>; + using repository_fragments = set>; + + // If reason is absent, then don't print the "fetching ..." progress line. + // + static void + rep_fetch (const common_options& co, + const dir_path& conf, + transaction& t, + const shared_ptr& r, + repositories& fetched_repositories, + repositories& removed_repositories, + repository_fragments& removed_fragments, + bool shallow, + const optional& reason) + { + tracer trace ("rep_fetch(rep)"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Check that the repository is not fetched yet and register it as fetched + // otherwise. + // + // Note that we can end up with a repository dependency cycle via + // prerequisites. Thus we register the repository before recursing into its + // dependencies. + // + if (!fetched_repositories.insert (r).second) // Is already fetched. + return; + + const repository_location& rl (r->location); + l4 ([&]{trace << r->name << " " << rl;}); + + // Cancel the repository removal. + // + // Note that this is an optimization as the rep_remove() function checks + // for reachability of the repository being removed. + // + removed_repositories.erase (r); + + // The fetch_*() functions below will be quiet at level 1, which can be + // quite confusing if the download hangs. + // + if (verb && reason) + { + diag_record dr (text); + + dr << "fetching " << r->name; + + if (!reason->empty ()) + dr << " (" << *reason << ")"; + } + + // Save the current complements and prerequisites to later check if the + // shallow repository fetch is possible and to register them for removal + // if that's not the case. + // + repository_fragment::complements_type old_complements; + repository_fragment::prerequisites_type old_prerequisites; + + auto collect_deps = [] (const shared_ptr& rf, + repository_fragment::complements_type& cs, + repository_fragment::prerequisites_type& ps) + { + for (const auto& cr: rf->complements) + cs.insert (cr); + + for (const auto& pr: rf->prerequisites) + ps.insert (pr); + }; + + // While traversing fragments also register them for removal. + // + for (const repository::fragment_type& fr: r->fragments) + { + shared_ptr rf (fr.fragment.load ()); + + collect_deps (rf, old_complements, old_prerequisites); + removed_fragments.insert (rf); + } + + // Cleanup the repository fragments list. + // + r->fragments.clear (); + + // Load the repository and package manifests and use them to populate the + // repository fragments list, as well as its prerequisite and complement + // repository sets. + // + rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + + repository_fragment::complements_type new_complements; + repository_fragment::prerequisites_type new_prerequisites; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + string nm (fr.friendly_name); // Don't move, still may be used. + + shared_ptr rf ( + rep_fragment (co, conf, t, rl, move (fr))); + + collect_deps (rf, new_complements, new_prerequisites); + + // Cancel the repository fragment removal. + // + // Note that this is an optimization as the rep_remove_fragment() + // function checks if the fragment is dangling prio to the removal. + // + removed_fragments.erase (rf); + + r->fragments.push_back ( + repository::fragment_type { + move (nm), lazy_shared_ptr (db, move (rf))}); + } + + // Save the changes to the repository object. + // + db.update (r); + + // Reset the shallow flag if the set of complements and/or prerequisites + // has changed. + // + // Note that weak pointers are generally incomparable (as can point to + // expired objects), and thus we can't compare the prerequisite sets + // directly. + // + if (shallow) + shallow = new_complements == old_complements && + equal (new_prerequisites.begin (), new_prerequisites.end (), + old_prerequisites.begin (), old_prerequisites.end (), + [] (const lazy_weak_ptr& x, + const lazy_weak_ptr& y) + { + return x.object_id () == y.object_id (); + }); + + // Fetch prerequisites and complements, unless this is a shallow fetch. + // + if (!shallow) + { + // Register old complements and prerequisites for potential removal + // unless they are fetched. + // + auto rm = [&fetched_repositories, &removed_repositories] ( + const lazy_shared_ptr& rp) + { + shared_ptr r (rp.load ()); + if (fetched_repositories.find (r) == fetched_repositories.end ()) + removed_repositories.insert (move (r)); + }; + + for (const lazy_shared_ptr& cr: old_complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + rm (cr); + } + + for (const lazy_weak_ptr& pr: old_prerequisites) + rm (lazy_shared_ptr (pr)); + + const string& rn (rl.canonical_name ()); + + // Fetch complements and prerequisites. + // + for (const auto& cr: new_complements) + { + if (cr.object_id () != "") + rep_fetch (co, + conf, + t, + cr.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + false /* shallow */, + "complements " + rn); + } + + for (const auto& pr: new_prerequisites) + rep_fetch (co, + conf, + t, + pr.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + false /* shallow */, + "prerequisite of " + rn); + } } static void @@ -806,10 +951,12 @@ namespace bpkg tracer_guard tg (db, trace); // As a fist step we fetch repositories recursively building the list of - // the former prerequisites and complements to be considered for removal. + // the former repository prerequisites, complements and fragments to be + // considered for removal. // // We delay the actual removal until we fetch all the required repositories // as a dependency dropped by one repository can appear for another one. + // The same is true about repository fragments. // try { @@ -819,26 +966,42 @@ namespace bpkg // filesystem_state_changed = false; - repositories fetched; - repositories removed; + repositories fetched_repositories; + repositories removed_repositories; + repository_fragments removed_fragments; // Fetch the requested repositories, recursively. // for (const lazy_shared_ptr& r: repos) - rep_fetch (o, conf, t, r.load (), fetched, removed, shallow, reason); + rep_fetch (o, + conf, + t, + r.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + shallow, + reason); // Remove dangling repositories. // - for (const shared_ptr& r: removed) + for (const shared_ptr& r: removed_repositories) rep_remove (conf, t, r); + // Remove dangling repository fragments. + // + for (const shared_ptr& rf: removed_fragments) + rep_remove_fragment (conf, t, rf); + // Finally, make sure that the external packages are available from a // single directory-based repository. // // Sort the packages by name and version. This way the external packages - // with the same upstream version and revision will be adjacent. + // with the same upstream version and revision will be adjacent. Note + // that here we rely on the fact that the directory-based repositories + // have a single fragment. // - using query = query; + using query = query; const auto& qv (query::package::id.version); query q ("ORDER BY" + query::package::id.name + "," + @@ -849,18 +1012,18 @@ namespace bpkg qv.iteration); available_package_id ap; - shared_ptr rp; + shared_ptr rf; - for (const auto& pr: db.query (q)) + for (const auto& prf: db.query (q)) { - const shared_ptr& r (pr.repository); - if (!r->location.directory_based ()) + const shared_ptr& f (prf.repository_fragment); + if (!f->location.directory_based ()) continue; // Fail if the external package is of the same upstream version and // revision as the previous one. // - const available_package_id& id (pr.package_id); + const available_package_id& id (prf.package_id); if (id.name == ap.name && compare_version_eq (id.version, ap.version, true, false)) @@ -871,12 +1034,12 @@ namespace bpkg fail << "external package " << id.name << '/' << version (v.epoch, v.upstream, v.release, v.revision, 0) << " is available from two repositories" << - info << "repository " << rp->location << - info << "repository " << r->location; + info << "repository " << rf->location << + info << "repository " << f->location; } ap = id; - rp = r; + rf = f; } } catch (const failed&) @@ -913,8 +1076,11 @@ namespace bpkg transaction t (db); - shared_ptr root (db.load ("")); - repository::complements_type& ua (root->complements); // User-added repos. + shared_ptr root (db.load ("")); + + // User-added repos. + // + repository_fragment::complements_type& ua (root->complements); for (const repository_location& rl: rls) { @@ -950,8 +1116,11 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr root (db.load ("")); - repository::complements_type& ua (root->complements); // User-added repos. + shared_ptr root (db.load ("")); + + // User-added repos. + // + repository_fragment::complements_type& ua (root->complements); optional reason; diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 89f4f2a..7e7777f 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -28,20 +28,23 @@ namespace bpkg struct rep_fetch_data { - using repository = repository_manifest; - - struct package + struct fragment { - package_manifest manifest; - string repository_fragment; // See package_location::fragment. + // Empty for fragment-less repositories. + // + string id; + string friendly_name; // User-friendly fragment name (e.g, tag, etc). + + vector repositories; + vector packages; }; - std::vector repositories; - std::vector packages; + vector fragments; - // For base repo (can be NULL). + // For base pkg repo (can be nullopt/NULL). // - shared_ptr certificate; + optional cert_pem; + shared_ptr certificate; // Authenticated. }; rep_fetch_data diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx index 0e7f911..7dd4a28 100644 --- a/bpkg/rep-info.cxx +++ b/bpkg/rep-info.cxx @@ -4,7 +4,8 @@ #include -#include // cout +#include // cout +#include // find_if() #include // sha256_to_fingerprint() #include @@ -72,7 +73,7 @@ namespace bpkg if (all || cert_info) { shared_ptr& cert (rfd.certificate); - const optional& cert_pem (rfd.repositories.back ().certificate); + const optional& cert_pem (rfd.cert_pem); if (cert_pem) { @@ -145,23 +146,89 @@ namespace bpkg { if (o.manifest ()) { + // Merge repository manifest lists, adding the fragment value to + // prerequisite/complement repository manifests, and picking the + // latest base repository manifest. + // + vector rms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (repository_manifest& rm: fr.repositories) + { + if (rm.effective_role () != repository_role::base) + { + if (!fr.id.empty ()) + rm.fragment = fr.id; + + rms.push_back (move (rm)); + } + } + } + + // Append the latest base repository manifest. + // + // Note that there must be at least one fragment with at least a + // base repository being present. + // + assert (!rfd.fragments.empty () && + !rfd.fragments.back ().repositories.empty ()); + + rms.push_back (move (rfd.fragments.back ().repositories.back ())); + // Note: serializing without any extra repository_manifests info. // manifest_serializer s (cout, "STDOUT"); - for (const repository_manifest& rm: rfd.repositories) + + for (const repository_manifest& rm: rms) rm.serialize (s); + s.next ("", ""); // End of stream. } else { - for (const repository_manifest& rm: rfd.repositories) + // Merge complements/prerequisites from all fragments "upgrading" + // prerequisites to complements and preferring locations from the + // latest fragments. + // + vector rms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) { - repository_role rr (rm.effective_role ()); + for (repository_manifest& rm: fr.repositories) + { + if (rm.effective_role () == repository_role::base) + continue; + + repository_location l (rm.location, rl); // Complete. + + auto i (find_if (rms.begin (), rms.end (), + [&l] (const repository_manifest& i) + { + return i.location.canonical_name () == + l.canonical_name (); + })); + + if (i == rms.end ()) + { + rm.location = move (l); + rms.push_back (move (rm)); + } + else + { + if (rm.effective_role () == repository_role::complement) + i->role = rm.effective_role (); - if (rr == repository_role::base) - continue; // Entry for this repository. + i->location = move (l); + } + } + } + + for (const repository_manifest& rm: rms) + { + repository_role rr (rm.effective_role ()); - repository_location l (rm.location, rl); // Complete. + const repository_location& l (rm.location); const string& n (l.canonical_name ()); switch (rr) @@ -185,21 +252,56 @@ namespace bpkg { if (o.manifest ()) { + // Merge package manifest lists, adding the fragment. + // + vector pms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (package_manifest& pm: fr.packages) + { + if (!fr.id.empty ()) + pm.fragment = fr.id; + + pms.push_back (move (pm)); + } + } + // Note: serializing without any extra package_manifests info. // manifest_serializer s (cout, "STDOUT"); - for (const rep_fetch_data::package& p: rfd.packages) - p.manifest.serialize (s); + for (const package_manifest& pm: pms) + pm.serialize (s); s.next ("", ""); // End of stream. } else { + // Merge packages from all the fragments. + // + vector pms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (package_manifest& pm: fr.packages) + { + auto i (find_if (pms.begin (), pms.end (), + [&pm] (const package_manifest& i) + { + return i.name == pm.name && + i.version == pm.version; + })); + + if (i == pms.end ()) + pms.push_back (move (pm)); + } + } + // Separate package list from the general repository info. // cout << endl; - for (const rep_fetch_data::package& p: rfd.packages) - cout << p.manifest.name << "/" << p.manifest.version << endl; + for (const package_manifest& pm: pms) + cout << pm.name << "/" << pm.version << endl; } } } diff --git a/bpkg/rep-list.cxx b/bpkg/rep-list.cxx index 8d308dd..f1e251c 100644 --- a/bpkg/rep-list.cxx +++ b/bpkg/rep-list.cxx @@ -20,7 +20,7 @@ namespace bpkg // // Each line has the following form: // - // [(complement|prerequisite) ] + // [(complement|prerequisite) ] [ ()] // // and is indented with 2 additional spaces for each recursion level. // @@ -44,34 +44,42 @@ namespace bpkg indent += " "; - if (o.complements ()) + auto print_repo = [&o, &indent, &chain] ( + const shared_ptr& r, + const char* role, + const repository::fragment_type& fr) { - for (const lazy_shared_ptr& rp: r->complements) - { - // Skip the root complement (see rep_fetch() for details). - // - if (rp.object_id () == "") - continue; + cout << indent << role << ' ' << r->name << ' ' << r->location; - shared_ptr r (rp.load ()); + if (!fr.friendly_name.empty ()) + cout << " (" << fr.friendly_name << ")"; - cout << indent << "complement " - << r->location.canonical_name () << " " << r->location << endl; + cout << endl; - print_dependencies (o, r, indent, chain); - } - } + print_dependencies (o, r, indent, chain); + }; - if (o.prerequisites ()) + for (const repository::fragment_type& rfr: r->fragments) { - for (const lazy_weak_ptr& rp: r->prerequisites) - { - shared_ptr r (rp.load ()); + shared_ptr fr (rfr.fragment.load ()); - cout << indent << "prerequisite " - << r->location.canonical_name () << " " << r->location << endl; + if (o.complements ()) + { + for (const lazy_shared_ptr& rp: fr->complements) + { + // Skip the root complement (see rep_fetch() for details). + // + if (rp.object_id () == "") + continue; + + print_repo (rp.load (), "complement", rfr); + } + } - print_dependencies (o, r, indent, chain); + if (o.prerequisites ()) + { + for (const lazy_weak_ptr& rp: fr->prerequisites) + print_repo (rp.load (), "prerequisite", rfr); } } @@ -104,7 +112,7 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr root (db.load ("")); + shared_ptr root (db.load ("")); for (const lazy_shared_ptr& rp: root->complements) { diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx index fafe5a8..056883d 100644 --- a/bpkg/rep-remove.cxx +++ b/bpkg/rep-remove.cxx @@ -44,21 +44,45 @@ namespace bpkg if (!traversed.insert (r).second) // We have already been here. return false; - for (const auto& rc: db.query ( + // Iterate over repository fragments that depend on this repository as a + // complement. + // + for (const auto& rf: db.query ( query::complement::name == nm)) { - const shared_ptr& r (rc); - if (r->name.empty () /* Root? */ || reachable (db, r, traversed)) + const shared_ptr& f (rf); + + if (f->name.empty ()) // Root? return true; + + // Iterate over repositories that contain this repository fragment. + // + for (const auto& fr: db.query ( + query::repository_fragment::name == f->name)) + { + if (reachable (db, fr, traversed)) + return true; + } } - for (const auto& rd: db.query ( + // Iterate over repository fragments that depend on this repository as a + // prerequisite. + // + for (const auto& rf: db.query ( query::prerequisite::name == nm)) { - // Note that the root repository has no prerequisites. + // Note that the root repository fragment has no prerequisites. // - if (reachable (db, rd, traversed)) - return true; + const shared_ptr& f (rf); + + // Iterate over repositories that contain this repository fragment. + // + for (const auto& fr: db.query ( + query::repository_fragment::name == f->name)) + { + if (reachable (db, fr, traversed)) + return true; + } } return false; @@ -72,22 +96,28 @@ namespace bpkg } void - rep_remove_package_locations (transaction& t, const string& name) + rep_remove_package_locations (transaction& t, const string& fragment_name) { + tracer trace ("rep_remove_package_locations"); + database& db (t.database ()); + tracer_guard tg (db, trace); + + using query = query; - for (const auto& rp: db.query ( - query::repository::name == name)) + for (const auto& rp: db.query ( + query::repository_fragment::name == fragment_name)) { const shared_ptr& p (rp); vector& ls (p->locations); - for (auto i (ls.cbegin ()); i != ls.cend (); ) + for (auto i (ls.cbegin ()); i != ls.cend (); ++i) { - if (i->repository.object_id () == name) - i = ls.erase (i); - else - ++i; + if (i->repository_fragment.object_id () == fragment_name) + { + ls.erase (i); + break; + } } if (ls.empty ()) @@ -117,17 +147,29 @@ namespace bpkg transaction& t, const shared_ptr& r) { - const string& nm (r->name); - assert (!nm.empty ()); // Can't be the root repository. + assert (!r->name.empty ()); // Can't be the root repository. + + tracer trace ("rep_remove"); database& db (t.database ()); + tracer_guard tg (db, trace); if (reachable (db, r)) return; - rep_remove_package_locations (t, nm); + // Note that it is essential to erase the repository object from the + // database prior to the repository fragments it contains as they must be + // un-referenced first. + // + db.erase (r); - // Cleanup the repository state if present. + // Remove dangling repository fragments. + // + for (const repository::fragment_type& fr: r->fragments) + rep_remove_fragment (c, t, fr.fragment.load ()); + + // Cleanup the repository state if present and there are no more + // repositories referring this state. // // Note that this step is irreversible on failure. If something goes wrong // we will end up with a state-less fetched repository and the @@ -145,14 +187,60 @@ namespace bpkg dir_path sd (c / repos_dir / d); if (exists (sd)) - rmdir (sd); + { + // There is no way to get the list of repositories that share this + // state other than traversing all repositories of this type. + // + bool rm (true); + + using query = query; + + for (shared_ptr r: + pointer_result ( + db.query ( + query::name != "" && + query::location.type == to_string (r->location.type ())))) + { + if (repository_state (r->location) == d) + { + rm = false; + break; + } + } + + if (rm) + rmdir (sd); + } } + } - // Note that it is essential to erase the repository object from the - // database prior to its complements and prerequisites removal as they - // must be un-referenced first. + void + rep_remove_fragment (const dir_path& c, + transaction& t, + const shared_ptr& rf) + { + tracer trace ("rep_remove_fragment"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Bail out if the repository fragment is still used. // - db.erase (r); + using query = query; + + if (db.query_value ( + "fragment=" + query::_val (rf->name)) != 0) + return; + + // Remove the repository fragment from locations of the available packages + // it contains. Note that this must be done before the repository fragment + // removal. + // + rep_remove_package_locations (t, rf->name); + + // Remove the repository fragment. + // + db.erase (rf); // Remove dangling complements and prerequisites. // @@ -167,7 +255,7 @@ namespace bpkg rep_remove (c, t, r); }; - for (const lazy_shared_ptr& cr: r->complements) + for (const lazy_shared_ptr& cr: rf->complements) { // Remove the complement unless it is the root repository (see // rep_fetch() for details). @@ -176,8 +264,15 @@ namespace bpkg remove (cr); } - for (const lazy_weak_ptr& pr: r->prerequisites) + for (const lazy_weak_ptr& pr: rf->prerequisites) remove (lazy_shared_ptr (pr)); + + // If there are no repositories stayed in the database then no repository + // fragments nor packages should stay either. + // + assert (db.query_value () != 0 || + (db.query_value () == 0 && + db.query_value () == 0)); } void @@ -187,12 +282,13 @@ namespace bpkg bool quiet) { tracer trace ("rep_remove_clean"); + tracer_guard tg (db, trace); assert (!transaction::has_current ()); - // Clean repositories and available packages. At the end only repositories - // that were explicitly added by the user and the special root repository - // should remain. + // Clean repositories, repository fragments and available packages. At the + // end only repositories that were explicitly added by the user and the + // special root repository should remain. // { // Note that we don't rely on being in session nor create one. @@ -201,19 +297,19 @@ namespace bpkg db.erase_query (); - shared_ptr root (db.load ("")); - repository::complements_type& ua (root->complements); + db.erase_query ( + query::name != ""); + + shared_ptr root (db.load ("")); + repository_fragment::complements_type& ua (root->complements); for (shared_ptr r: pointer_result (db.query ())) { if (r->name == "") - { - l5 ([&]{trace << "skipping root";}); - } + l5 ([&]{trace << "skipping root repository";}); else if (ua.find (lazy_shared_ptr (db, r)) != ua.end ()) { - r->complements.clear (); - r->prerequisites.clear (); + r->fragments.clear (); db.update (r); if (verb >= (quiet ? 2 : 1) && !o.no_result ()) @@ -298,8 +394,8 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr root (db.load ("")); - repository::complements_type& ua (root->complements); + shared_ptr root (db.load ("")); + repository_fragment::complements_type& ua (root->complements); if (o.all ()) { @@ -392,11 +488,12 @@ namespace bpkg // assert (!o.all () || ua.empty ()); - // If we removed all the user-added repositories then no repositories nor - // packages should stay in the database. + // If we removed all the user-added repositories then no repositories, + // repository fragments or packages should stay in the database. // assert (!ua.empty () || (db.query_value () == 0 && + db.query_value () == 0 && db.query_value () == 0)); t.commit (); diff --git a/bpkg/rep-remove.hxx b/bpkg/rep-remove.hxx index 68d8a36..477f308 100644 --- a/bpkg/rep-remove.hxx +++ b/bpkg/rep-remove.hxx @@ -17,13 +17,22 @@ namespace bpkg rep_remove (const rep_remove_options&, cli::scanner& args); // Remove a repository if it is not reachable from the root (and thus is not - // required by any user-added repository). + // required by any user-added repository), also removing its unused + // repository fragments. // void rep_remove (const dir_path& conf, transaction&, const shared_ptr&); + // Remove a repository fragment if it is not referenced by any repository, + // also removing its unreachable complements and prerequisites. + // + void + rep_remove_fragment (const dir_path& conf, + transaction&, + const shared_ptr&); + // Bring the configuration to the clean state as if repositories were added // but never fetched. Leave selected packages intact. // @@ -45,11 +54,11 @@ namespace bpkg database&, bool quiet = true); - // Remove a repository from locations of the available packages it - // contains. Remove packages that come from only this repository. + // Remove a repository fragment from locations of the available packages it + // contains. Remove packages that come from only this repository fragment. // void - rep_remove_package_locations (transaction&, const string& repository_name); + rep_remove_package_locations (transaction&, const string& fragment_name); } #endif // BPKG_REP_REMOVE_HXX -- cgit v1.1