diff options
37 files changed, 1472 insertions, 745 deletions
diff --git a/bpkg/cfg-create.cxx b/bpkg/cfg-create.cxx index b62163e..04d97f4 100644 --- a/bpkg/cfg-create.cxx +++ b/bpkg/cfg-create.cxx @@ -96,10 +96,31 @@ namespace bpkg // database db (open (c, trace, true)); - // Add the special, root repository object with empty location. + // Add the special, root repository object with empty location and + // containing a single repository fragment having an empty location as + // well. + // + // Note that the root repository serves as a complement for dir and git + // repositories that have neither prerequisites nor complements. The + // root repository fragment is used for transient available package + // locations and as a search starting point for held packages (see + // pkg-build for details). // transaction t (db); - db.persist (repository (repository_location ())); + + shared_ptr<repository_fragment> fr ( + make_shared<repository_fragment> (repository_location ())); + + db.persist (fr); + + shared_ptr<repository> r ( + make_shared<repository> (repository_location ())); + + r->fragments.push_back ( + repository::fragment_type {string () /* friendly_name */, move (fr)}); + + db.persist (r); + t.commit (); if (verb && !o.no_result ()) diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index e6836a6..9991075 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -5,7 +5,7 @@ #include <bpkg/fetch.hxx> #include <map> -#include <algorithm> // find(), find_if(), replace() +#include <algorithm> // find(), find_if(), replace(), sort() #include <libbutl/utility.mxx> // digit(), xdigit() #include <libbutl/process.mxx> @@ -864,18 +864,21 @@ namespace bpkg // Add a fragment to the resulting list, suppressing duplicates. // - auto add_frag = [&r] (const git_fragment& f) + // Note that the timestamp is set to zero. It is properly filled by the + // sort() lambda (see below). + // + auto add_frag = [&r] (const string& c, string n = string ()) { auto i (find_if (r.begin (), r.end (), - [&f] (const git_fragment& i) + [&c] (const git_fragment& i) { - return i.commit == f.commit; + return i.commit == c; })); if (i == r.end ()) - r.push_back (f); - else if (i->name.empty ()) - i->name = f.name; + r.push_back (git_fragment {c, 0 /* timestamp */, move (n)}); + else if (i->friendly_name.empty ()) + i->friendly_name = move (n); }; // Add a commit to the list for subsequent fetching. @@ -890,11 +893,9 @@ namespace bpkg // auto friendly_name = [] (const string& n) -> string { - return n.compare (0, 11, "refs/heads/") == 0 - ? "branch " + string (n, 11) - : n.compare (0, 10, "refs/tags/") == 0 - ? "tag " + string (n, 10) - : "reference " + n; + // Strip 'refs/' prefix if present. + // + return n.compare (0, 5, "refs/") == 0 ? string (n, 5) : n; }; if (rfs.empty ()) @@ -912,7 +913,7 @@ namespace bpkg // Add the commit to the resulting list. // - add_frag (git_fragment {rf.commit, friendly_name (rf.name)}); + add_frag (rf.commit, friendly_name (rf.name)); // Skip the commit if it is already fetched. // @@ -939,7 +940,7 @@ namespace bpkg // if (rf.commit && commit_fetched (co, dir, *rf.commit)) { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); continue; } @@ -951,7 +952,7 @@ namespace bpkg if (commit_fetched (co, dir, rfc.commit)) { - add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); + add_frag (rfc.commit, friendly_name (rfc.name)); continue; } } @@ -982,7 +983,7 @@ namespace bpkg const ref& rfc (reference (*rf.name, true /* abbr_commit */)); - add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); + add_frag (rfc.commit, friendly_name (rfc.name)); add_commit (rfc.commit, fetch_list); } // If commit is specified and the shallow fetch is possible, then we @@ -990,7 +991,7 @@ namespace bpkg // else if (shallow) { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (*rf.commit, fetch_list); } // If commit is specified and the shallow fetch is not possible, but @@ -1001,7 +1002,7 @@ namespace bpkg { // Note that commits we return and fetch are likely to differ. // - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (reference (*rf.name, false /* abbr_commit */).commit, fetch_list); @@ -1011,7 +1012,7 @@ namespace bpkg // else { - add_frag (git_fragment {*rf.commit, string ()}); + add_frag (*rf.commit); add_commit (*rf.commit, fetch_list); if (!commit_advertized (co, url, *rf.commit)) @@ -1020,10 +1021,50 @@ namespace bpkg } } + // Set timestamps for commits and sort them in the timestamp ascending + // order. + // + auto sort = [&co, &dir] (git_fragments&& frs) -> git_fragments + { + for (git_fragment& fr: frs) + { + // Add '^{commit}' suffix to strip some unwanted output that appears + // for tags. + // + string s (git_string (co, "commit timestamp", + co.git_option (), + "-C", dir, + "show", + "-s", + "--format=%ct", + fr.commit + "^{commit}")); + try + { + fr.timestamp = static_cast<time_t> (stoull (s)); + } + // Catches both std::invalid_argument and std::out_of_range that + // inherit from std::logic_error. + // + catch (const logic_error&) + { + fail << "'" << s << "' doesn't appear to contain a git commit " + "timestamp" << endg; + } + } + + std::sort (frs.begin (), frs.end (), + [] (const git_fragment& x, const git_fragment& y) + { + return x.timestamp < y.timestamp; + }); + + return frs; + }; + // Bail out if all commits are already fetched. // if (scs.empty () && dcs.empty ()) - return r; + return sort (move (r)); auto fetch = [&co, &url, &dir] (const strings& refspecs, bool shallow) { @@ -1125,11 +1166,11 @@ namespace bpkg // for (auto& fr: r) { - if (fr.name.empty ()) - fr.name = "commit " + fr.commit.substr (0, 12); + if (fr.friendly_name.empty ()) + fr.friendly_name = fr.commit.substr (0, 12); } - return r; + return sort (move (r)); } // Checkout the repository submodules (see git_checkout_submodules() @@ -1377,50 +1418,62 @@ namespace bpkg init (co, dir, url); } - vector<git_fragment> - git_fetch (const common_options& co, - const repository_location& rl, - const dir_path& dir) + // Update the repository remote origin URL, if changed. + // + static void + sync_origin_url (const common_options& co, + const repository_location& rl, + const dir_path& dir) { - git_ref_filters rfs; repository_url url (rl.url ()); + url.fragment = nullopt; - if (url.fragment) - try - { - rfs = parse_git_ref_filters (*url.fragment); - url.fragment = nullopt; - } - catch (const invalid_argument& e) - { - fail << "unable to fetch " << url << ": " << e; - } - - // Update the repository URL, if changed. - // repository_url u (origin_url (co, dir)); if (url != u) { - // Note that the repository canonical name can not change under the - // legal scenarios that lead to the location change. Changed canonical - // name means that the repository was manually amended. We could fix-up - // such repositories as well but want to leave the backdoor for tests. + // Note that the repository canonical name with the fragment part + // stripped can not change under the legal scenarios that lead to the + // location change. Changed canonical name means that the repository was + // manually amended. We could fix-up such repositories as well but want + // to leave the backdoor for tests. // - u.fragment = rl.url ().fragment; // Restore the fragment. - repository_location l (u, rl.type ()); - - if (rl.canonical_name () == l.canonical_name ()) + if (repository_location (url, rl.type ()).canonical_name () == + repository_location (u, rl.type ()).canonical_name ()) { if (verb) + { + u.fragment = rl.url ().fragment; // Restore the fragment. + info << "location changed for " << rl.canonical_name () << info << "new location " << rl << - info << "old location " << l; + info << "old location " << repository_location (u, rl.type ()); + } origin_url (co, dir, url); } } + } + vector<git_fragment> + git_fetch (const common_options& co, + const repository_location& rl, + const dir_path& dir) + { + git_ref_filters rfs; + const repository_url& url (rl.url ()); + + if (url.fragment) + try + { + rfs = parse_git_ref_filters (*url.fragment); + } + catch (const invalid_argument& e) + { + fail << "unable to fetch " << url << ": " << e; + } + + sync_origin_url (co, rl, dir); return fetch (co, dir, dir_path () /* submodule */, rfs); } @@ -1462,8 +1515,17 @@ namespace bpkg } void - git_checkout_submodules (const common_options& co, const dir_path& dir) + git_checkout_submodules (const common_options& co, + const repository_location& rl, + const dir_path& dir) { + // Note that commits could come from different repository URLs that may + // contain different sets of commits. Thus, we need to switch to the URL + // the checked out commit came from to properly complete submodule + // relative URLs. + // + sync_origin_url (co, rl, dir); + checkout_submodules (co, dir, dir / dir_path (".git"), diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx index d05b8da..9a8efd0 100644 --- a/bpkg/fetch.hxx +++ b/bpkg/fetch.hxx @@ -5,6 +5,8 @@ #ifndef BPKG_FETCH_HXX #define BPKG_FETCH_HXX +#include <ctime> // time_t + #include <libbutl/process.mxx> #include <libbpkg/manifest.hxx> @@ -60,15 +62,21 @@ namespace bpkg // Fetch a git repository in the specifid directory (previously created by // git_init() for the references obtained with the repository URL fragment - // filters, returning commit ids these references resolve to. After fetching - // the repository working tree state is unspecified (see git_checkout ()). + // filters, returning commit ids these references resolve to in the earliest + // to latest order. Update the remote repository URL, if changed. After + // fetching the repository working tree state is unspecified (see + // git_checkout()). // // Note that submodules are not fetched. // struct git_fragment { - string commit; - string name; // User-friendly name (like 'branch foo', 'tag bar', ...). + // User-friendly fragment name is either a ref (tags/v1.2.3, heads/master, + // HEAD) or an abbreviated commit id (0123456789ab). + // + string commit; + std::time_t timestamp; + string friendly_name; }; vector<git_fragment> @@ -86,10 +94,13 @@ namespace bpkg const string& commit); // Fetch (if necessary) and checkout submodules, recursively, in a working - // tree previously checked out by git_checkout(). + // tree previously checked out by git_checkout(). Update the remote + // repository URL, if changed. // void - git_checkout_submodules (const common_options&, const dir_path&); + git_checkout_submodules (const common_options&, + const repository_location&, + const dir_path&); // Low-level fetch API (fetch.cxx). // diff --git a/bpkg/forward.hxx b/bpkg/forward.hxx index 3845003..744c516 100644 --- a/bpkg/forward.hxx +++ b/bpkg/forward.hxx @@ -15,6 +15,7 @@ namespace bpkg // <bpkg/package.hxx> // class repository; + class repository_fragment; class selected_package; } diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 10a1512..0f64b30 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -137,15 +137,27 @@ namespace bpkg } dir_path - repository_state (const repository_location& l) + repository_state (const repository_location& rl) { - switch (l.type ()) + switch (rl.type ()) { case repository_type::pkg: case repository_type::dir: return dir_path (); // No state. case repository_type::git: { + // Strip the fragment, so all the repository fragments of the same + // git repository can reuse the state. So, for example, the state is + // shared for the fragments fetched from the following git repository + // locations: + // + // https://www.example.com/foo.git#master + // git://example.com/foo#stable + // + repository_url u (rl.url ()); + u.fragment = nullopt; + + repository_location l (u, rl.type ()); return dir_path (sha256 (l.canonical_name ()).abbreviated_string (12)); } } diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index e7be005..55b1244 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -67,8 +67,7 @@ namespace bpkg // Note that the semantics used to produce this name is repository type- // specific and can base on the repository canonical name or (potentially a // subset of) the location URL. In particular, a state directory could be - // shared by multiple repository locations of the same type (@@ TODO: if we - // ever do this, then we will need to complicate the removal logic). + // shared by multiple repository locations of the same type. // dir_path repository_state (const repository_location&); diff --git a/bpkg/package.cxx b/bpkg/package.cxx index 2880368..4880623 100644 --- a/bpkg/package.cxx +++ b/bpkg/package.cxx @@ -29,81 +29,110 @@ namespace bpkg // available_package // - // Check if the package is available from the specified repository, its - // prerequisite repositories, or one of their complements, recursively. - // Return the first repository that contains the package or NULL if none - // are. + // Check if the package is available from the specified repository fragment, + // its prerequisite repositories, or one of their complements, recursively. + // Return the first repository fragment that contains the package or NULL if + // none are. // // Note that we can end up with a repository dependency cycle since the - // root repository can be the default complement for git repositories (see - // rep_fetch() implementation for details). Thus we need to make sure that - // the repository is not in the dependency chain yet. + // root repository can be the default complement for dir and git + // repositories (see rep_fetch() implementation for details). Thus we need + // to make sure that the repository fragment is not in the dependency chain + // yet. // - using repositories = vector<reference_wrapper<const shared_ptr<repository>>>; + using repository_fragments = + vector<reference_wrapper<const shared_ptr<repository_fragment>>>; - static shared_ptr<repository> - find (const shared_ptr<repository>& r, + static shared_ptr<repository_fragment> + find (const shared_ptr<repository_fragment>& rf, const shared_ptr<available_package>& ap, - repositories& chain, + repository_fragments& chain, bool prereq) { // Prerequisites are not searched through recursively. // assert (!prereq || chain.empty ()); - auto pr = [&r] (const shared_ptr<repository>& i) -> bool {return i == r;}; - auto i (find_if (chain.begin (), chain.end (), pr)); + auto i (find_if (chain.begin (), chain.end (), + [&rf] (const shared_ptr<repository_fragment>& i) -> bool + { + return i == rf; + })); if (i != chain.end ()) return nullptr; - chain.emplace_back (r); + chain.emplace_back (rf); - unique_ptr<repositories, void (*)(repositories*)> deleter ( - &chain, [] (repositories* r) {r->pop_back ();}); + unique_ptr<repository_fragments, void (*)(repository_fragments*)> deleter ( + &chain, [] (repository_fragments* rf) {rf->pop_back ();}); - const auto& ps (r->prerequisites); - const auto& cs (r->complements); + const auto& cs (rf->complements); + const auto& ps (rf->prerequisites); - // @@ The same repository can be present in the location set multiple times - // with different fragment values. Given that we may traverse the same - // repository tree multiple times, which is inefficient but harmless. - // Let's leave it this way for now as it likely to be changed with - // adding support for repository fragment objects. - // for (const package_location& pl: ap->locations) { - const lazy_shared_ptr<repository>& lr (pl.repository); + const lazy_shared_ptr<repository_fragment>& lrf (pl.repository_fragment); // First check the repository itself. // - if (lr.object_id () == r->name) - return r; + if (lrf.object_id () == rf->name) + return rf; - // Then check all the complements and prerequisites without - // loading them. + // Then check all the complements and prerequisites repository fragments + // without loading them. Though, we still need to load complement and + // prerequisite repositories. // - if (cs.find (lr) != cs.end () || (prereq && ps.find (lr) != ps.end ())) - return lr.load (); + auto pr = [&lrf] (const repository::fragment_type& i) + { + return i.fragment == lrf; + }; + + for (const lazy_shared_ptr<repository>& r: cs) + { + const auto& frs (r.load ()->fragments); + + if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) + return lrf.load (); + } + + if (prereq) + { + for (const lazy_weak_ptr<repository>& r: ps) + { + const auto& frs (r.load ()->fragments); + + if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) + return lrf.load (); + } + } // Finally, load the complements and prerequisites and check them // recursively. // for (const lazy_shared_ptr<repository>& cr: cs) { - // Should we consider prerequisites of our complements as our - // prerequisites? I'd say not. - // - if (shared_ptr<repository> r = find (cr.load (), ap, chain, false)) - return r; + for (const auto& fr: cr.load ()->fragments) + { + // Should we consider prerequisites of our complements as our + // prerequisites? I'd say not. + // + if (shared_ptr<repository_fragment> r = + find (fr.fragment.load (), ap, chain, false)) + return r; + } } if (prereq) { for (const lazy_weak_ptr<repository>& pr: ps) { - if (shared_ptr<repository> r = find (pr.load (), ap, chain, false)) - return r; + for (const auto& fr: pr.load ()->fragments) + { + if (shared_ptr<repository_fragment> r = + find (fr.fragment.load (), ap, chain, false)) + return r; + } } } } @@ -111,17 +140,17 @@ namespace bpkg return nullptr; } - shared_ptr<repository> - filter (const shared_ptr<repository>& r, + shared_ptr<repository_fragment> + filter (const shared_ptr<repository_fragment>& r, const shared_ptr<available_package>& ap, bool prereq) { - repositories chain; + repository_fragments chain; return find (r, ap, chain, prereq); } vector<shared_ptr<available_package>> - filter (const shared_ptr<repository>& r, + filter (const shared_ptr<repository_fragment>& r, result<available_package>&& apr, bool prereq) { @@ -136,34 +165,36 @@ namespace bpkg return aps; } - pair<shared_ptr<available_package>, shared_ptr<repository>> - filter_one (const shared_ptr<repository>& r, + pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> + filter_one (const shared_ptr<repository_fragment>& r, result<available_package>&& apr, bool prereq) { - using result = pair<shared_ptr<available_package>, shared_ptr<repository>>; + using result = pair<shared_ptr<available_package>, + shared_ptr<repository_fragment>>; for (shared_ptr<available_package> ap: pointer_result (apr)) { - if (shared_ptr<repository> pr = filter (r, ap, prereq)) + if (shared_ptr<repository_fragment> pr = filter (r, ap, prereq)) return result (move (ap), move (pr)); } return result (); } - vector<pair<shared_ptr<available_package>, shared_ptr<repository>>> - filter (const vector<shared_ptr<repository>>& rps, + vector<pair<shared_ptr<available_package>, shared_ptr<repository_fragment>>> + filter (const vector<shared_ptr<repository_fragment>>& rps, odb::result<available_package>&& apr, bool prereq) { - vector<pair<shared_ptr<available_package>, shared_ptr<repository>>> aps; + vector<pair<shared_ptr<available_package>, + shared_ptr<repository_fragment>>> aps; for (shared_ptr<available_package> ap: pointer_result (apr)) { - for (const shared_ptr<repository> r: rps) + for (const shared_ptr<repository_fragment> r: rps) { - shared_ptr<repository> ar (filter (r, ap, prereq)); + shared_ptr<repository_fragment> ar (filter (r, ap, prereq)); if (ar != nullptr) { @@ -235,20 +266,20 @@ namespace bpkg if (check_external) { - using query = query<package_repository>; + using query = query<package_repository_fragment>; query q ( query::package::id.name == n && compare_version_eq (query::package::id.version, v, true, false)); - for (const auto& pr: db.query<package_repository> (q)) + for (const auto& prf: db.query<package_repository_fragment> (q)) { - const shared_ptr<repository>& r (pr.repository); + const shared_ptr<repository_fragment>& rf (prf.repository_fragment); - if (r->location.directory_based ()) + if (rf->location.directory_based ()) fail << "external package " << n << '/' << v << " is already available from " - << r->location.canonical_name (); + << rf->location.canonical_name (); } } diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 5b0db8c..ff05301 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -241,28 +241,62 @@ namespace bpkg : (?).type ()}) \ from(bpkg::repository_location (std::move ((?).url), (?).type)) - // repository + // repository_fragment + // + // Some repository types (normally version control-based) can be + // fragmented. For example, a git repository consists of multiple commits + // (fragments) which could contain different sets of packages and even + // prerequisite/complement repositories. Note also that the same fragment + // could be shared by multiple repository objects. We assume a fragment to + // be immutable, so it's complement, prerequisite and package sets can never + // change. + // + // For repository types that do not support fragmentation, there should + // be a single repository_fragment with the name and location equal to the + // ones of the containing repository. Such a fragment can not be shared but + // can be changed. // + // One of the consequences of the above is that a fragment can either be + // shared or be mutable. + // + class repository; + #pragma db object pointer(shared_ptr) session - class repository + class repository_fragment { public: - // We use a weak pointer for prerequisite repositories because we - // could have cycles. No cycles in complements, thought. + // We use a weak pointer for prerequisite repositories because we could + // have cycles. No cycles in complements, thought. + // + // Note that these point to repositories, not repository fragments. // using complements_type = std::set<lazy_shared_ptr<repository>, compare_lazy_ptr>; using prerequisites_type = std::set<lazy_weak_ptr<repository>, compare_lazy_ptr>; + // Repository fragment id is a repository canonical name that identifies + // just this fragment (for example, for git it is a canonical name of + // the repository URL with the full, non-abbreviated commit id). + // + // Note that while this works naturally for git where the fragment (full + // commit id) is also a valid fragment filter, it may not fit some future + // repository types. Let's deal with it when we see such a beast. + // string name; // Object id (canonical name). + + // For version control-based repositories it is used for a package + // checkout, that may involve communication with the remote repository. + // repository_location location; - complements_type complements; + + complements_type complements; prerequisites_type prerequisites; public: explicit - repository (repository_location l): location (move (l)) + repository_fragment (repository_location l) + : location (move (l)) { name = location.canonical_name (); } @@ -275,14 +309,66 @@ namespace bpkg set(this.location = std::move (?); \ assert (this.name == this.location.canonical_name ())) - #pragma db member(complements) id_column("repository") \ + #pragma db member(complements) id_column("repository_fragment") \ value_column("complement") value_not_null - #pragma db member(prerequisites) id_column("repository") \ + #pragma db member(prerequisites) id_column("repository_fragment") \ value_column("prerequisite") value_not_null private: friend class odb::access; + repository_fragment () = default; + }; + + #pragma db view object(repository_fragment) \ + query(repository_fragment::name != "" && (?)) + struct repository_fragment_count + { + #pragma db column("count(*)") + size_t result; + + operator size_t () const {return result;} + }; + + // repository + // + #pragma db object pointer(shared_ptr) session + class repository + { + public: + #pragma db value + struct fragment_type + { + string friendly_name; // User-friendly fragment name (e.g, tag, etc). + lazy_shared_ptr<repository_fragment> fragment; + }; + + using fragments_type = std::vector<fragment_type>; + + string name; // Object id (canonical name). + repository_location location; + fragments_type fragments; + + public: + explicit + repository (repository_location l): location (move (l)) + { + name = location.canonical_name (); + } + + // Database mapping. + // + #pragma db member(name) id + + #pragma db member(location) column("") \ + set(this.location = std::move (?); \ + assert (this.name == this.location.canonical_name ())) + + #pragma db member(fragments) id_column("repository") \ + value_column("") value_not_null + + private: + friend class odb::access; repository () = default; }; @@ -300,18 +386,8 @@ namespace bpkg #pragma db value struct package_location { - using repository_type = bpkg::repository; - - // Fragment is optional, repository type-specific information that can be - // used to identify the repository fragment/partition/view/etc that this - // package came from. For example, for a version control-based repository - // this could be a commit id. - // - // The location is the package location within this repository fragment. - // - lazy_shared_ptr<repository_type> repository; - string fragment; - path location; + lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; + path location; // Package location within the repository fragment. }; // dependencies @@ -361,20 +437,21 @@ namespace bpkg available_package_id id; upstream_version version; - // List of repositories to which this package version belongs (yes, - // in our world, it can be in multiple, unrelated repositories). + // List of repository fragments to which this package version belongs + // (yes, in our world, it can be in multiple, unrelated repositories) + // together with locations within these repository fragments. // - // Note that if the repository is the special root repository (its - // location is empty), then this is a transient (or "fake") object - // for an existing package archive or package directory. In this - // case the location is the path to the archive/directory and to - // determine which one it is, use file/dir_exists(). While on the - // topic of fake available_package objects, when one is created for - // a selected package (see make_available()), this list is left empty - // with the thinking being that since the package is already in at - // least fetched state, we shouldn't be needing its location. + // Note that if the entry is the special root repository fragment (its + // location is empty), then this is a transient (or "fake") object for an + // existing package archive or package directory. In this case the + // location is the path to the archive/directory and to determine which + // one it is, use file/dir_exists(). While on the topic of fake + // available_package objects, when one is created for a selected package + // (see make_available()), this list is left empty with the thinking being + // that since the package is already in at least fetched state, we + // shouldn't be needing its location. // - vector<package_location> locations; //@@ Map? + vector<package_location> locations; // Package manifest data. // @@ -496,28 +573,28 @@ namespace bpkg operator size_t () const {return result;} }; - // Only return packages that are in the specified repositories, their + // Only return packages that are in the specified repository fragments, their // complements or prerequisites (if prereq is true), recursively. While you // could maybe come up with a (barely comprehensible) view/query to achieve // this, doing it on the "client side" is definitely more straightforward. // vector<shared_ptr<available_package>> - filter (const shared_ptr<repository>&, + filter (const shared_ptr<repository_fragment>&, odb::result<available_package>&&, bool prereq = true); - pair<shared_ptr<available_package>, shared_ptr<repository>> - filter_one (const shared_ptr<repository>&, + pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> + filter_one (const shared_ptr<repository_fragment>&, odb::result<available_package>&&, bool prereq = true); - shared_ptr<repository> - filter (const shared_ptr<repository>&, + shared_ptr<repository_fragment> + filter (const shared_ptr<repository_fragment>&, const shared_ptr<available_package>&, bool prereq = true); - vector<pair<shared_ptr<available_package>, shared_ptr<repository>>> - filter (const vector<shared_ptr<repository>>&, + vector<pair<shared_ptr<available_package>, shared_ptr<repository_fragment>>> + filter (const vector<shared_ptr<repository_fragment>>&, odb::result<available_package>&&, bool prereq = true); @@ -606,41 +683,38 @@ namespace bpkg package_state state; package_substate substate; - // The hold flags indicate whether this package and/or version - // should be retained in the configuration. A held package will - // not be automatically removed. A held version will not be - // automatically upgraded. Note also that the two flags are - // orthogonal: we may want to keep a specific version of the - // package as long as it has dependents. + // The hold flags indicate whether this package and/or version should be + // retained in the configuration. A held package will not be automatically + // removed. A held version will not be automatically upgraded. Note also + // that the two flags are orthogonal: we may want to keep a specific + // version of the package as long as it has dependents. // bool hold_package; bool hold_version; - // Repository from which this package came. Note that it is not - // a pointer to the repository object because it could be wiped - // out (e.g., as a result of rep-fetch). We call such packages - // "orphans". While we can get a list of orphan's prerequisites - // (by loading its manifest), we wouldn't know which repository - // to use as a base to resolve them. As a result, an orphan that - // is not already configured (and thus has all its prerequisites - // resolved) is not very useful and can only be purged. + // Repository fragment from which this package came. Note that it is not a + // pointer to the repository_fragment object because it could be wiped out + // (e.g., as a result of rep-fetch). We call such packages "orphans". + // While we can get a list of orphan's prerequisites (by loading its + // manifest), we wouldn't know which repository fragment to use as a base + // to resolve them. As a result, an orphan that is not already configured + // (and thus has all its prerequisites resolved) is not very useful and + // can only be purged. // - repository_location repository; + repository_location repository_fragment; - // Path to the archive of this package, if any. If not absolute, - // then it is relative to the configuration directory. The purge - // flag indicates whether the archive should be removed when the - // packaged is purged. If the archive is not present, it should - // be false. + // Path to the archive of this package, if any. If not absolute, then it + // is relative to the configuration directory. The purge flag indicates + // whether the archive should be removed when the packaged is purged. If + // the archive is not present, it should be false. // optional<path> archive; bool purge_archive; - // Path to the source directory of this package, if any. If not - // absolute, then it is relative to the configuration directory. - // The purge flag indicates whether the directory should be - // removed when the packaged is purged. If the source directory - // is not present, it should be false. + // Path to the source directory of this package, if any. If not absolute, + // then it is relative to the configuration directory. The purge flag + // indicates whether the directory should be removed when the packaged is + // purged. If the source directory is not present, it should be false. // optional<dir_path> src_root; bool purge_src; @@ -683,11 +757,12 @@ namespace bpkg return // pkg-unpack <name>/<version> // - (!repository.empty () && repository.directory_based ()) || + (!repository_fragment.empty () && + repository_fragment.directory_based ()) || // pkg-unpack --existing <dir> // - (repository.empty () && !archive); + (repository_fragment.empty () && !archive); } // Represent the wildcard version with the "*" string. Represent naturally @@ -899,45 +974,66 @@ namespace bpkg optional<dependency_constraint> constraint; }; - // Return a list of repositories that depend on this repository as a - // complement. + // Return a count of repositories that contain this repository fragment. // - // Note that the last db object pragma is required to produce an object - // loading view. + #pragma db view table("repository_fragments") + struct fragment_repository_count + { + #pragma db column("count(*)") + size_t result; + + operator size_t () const {return result;} + }; + + // Return a list of repositories that contain this repository fragment. // - #pragma db view object(repository = complement) \ - table("repository_complements" = "rc" inner: \ - "rc.complement = " + complement::name) \ - object(repository inner: "rc.repository = " + repository::name) - struct repository_complement_dependent + #pragma db view object(repository) \ + table("repository_fragments" = "rfs" inner: \ + "rfs.repository = " + repository::name) \ + object(repository_fragment inner: "rfs.fragment = " + \ + repository_fragment::name) + struct fragment_repository { shared_ptr<repository> object; operator const shared_ptr<repository> () const {return object;} }; - // Return a list of repositories that depend on this repository as a - // prerequisite. + // Return a list of repository fragments that depend on this repository as a + // complement. // - // Note that the last db object pragma is required to produce an object - // loading view. + #pragma db view object(repository = complement) \ + table("repository_fragment_complements" = "rfc" inner: \ + "rfc.complement = " + complement::name) \ + object(repository_fragment inner: "rfc.repository_fragment = " + \ + repository_fragment::name) + struct repository_complement_dependent + { + shared_ptr<repository_fragment> object; + + operator const shared_ptr<repository_fragment> () const {return object;} + }; + + // Return a list of repository fragments that depend on this repository as a + // prerequisite. // - #pragma db view object(repository = prerequisite) \ - table("repository_prerequisites" = "rp" inner: \ - "rp.prerequisite = " + prerequisite::name) \ - object(repository inner: "rp.repository = " + repository::name) + #pragma db view object(repository = prerequisite) \ + table("repository_fragment_prerequisites" = "rfp" inner: \ + "rfp.prerequisite = " + prerequisite::name) \ + object(repository_fragment inner: "rfp.repository_fragment = " + \ + repository_fragment::name) struct repository_prerequisite_dependent { - shared_ptr<repository> object; + shared_ptr<repository_fragment> object; - operator const shared_ptr<repository> () const {return object;} + operator const shared_ptr<repository_fragment> () const {return object;} }; - // Return a list of packages available from this repository. + // Return a list of packages available from this repository fragment. // - #pragma db view object(repository) query(distinct) \ + #pragma db view object(repository_fragment) \ table("available_package_locations" = "pl" inner: \ - "pl.repository = " + repository::name) \ + "pl.repository_fragment = " + repository_fragment::name) \ object(available_package = package inner: \ "pl.name = " + package::id.name + "AND" + \ "pl.version_epoch = " + package::id.version.epoch + "AND" + \ @@ -947,18 +1043,18 @@ namespace bpkg package::id.version.canonical_release + "AND" + \ "pl.version_revision = " + package::id.version.revision + "AND" + \ "pl.version_iteration = " + package::id.version.iteration) - struct repository_package + struct repository_fragment_package { shared_ptr<available_package> package; // Must match the alias (see above). operator const shared_ptr<available_package> () const {return package;} }; - // Return a list of repositories the packages come from. + // Return a list of repository fragments the packages come from. // - #pragma db view object(repository) query(distinct) \ + #pragma db view object(repository_fragment) \ table("available_package_locations" = "pl" inner: \ - "pl.repository = " + repository::name) \ + "pl.repository_fragment = " + repository_fragment::name) \ object(available_package = package inner: \ "pl.name = " + package::id.name + "AND" + \ "pl.version_epoch = " + package::id.version.epoch + "AND" + \ @@ -968,13 +1064,12 @@ namespace bpkg package::id.version.canonical_release + "AND" + \ "pl.version_revision = " + package::id.version.revision + "AND" + \ "pl.version_iteration = " + package::id.version.iteration) - struct package_repository + struct package_repository_fragment { #pragma db column(package::id) available_package_id package_id; - using repository_type = bpkg::repository; - shared_ptr<repository_type> repository; + shared_ptr<bpkg::repository_fragment> repository_fragment; }; // Version comparison operators. diff --git a/bpkg/package.xml b/bpkg/package.xml index 10014e7..a6cc552 100644 --- a/bpkg/package.xml +++ b/bpkg/package.xml @@ -1,6 +1,6 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="sqlite" version="1"> <model version="4"> - <table name="repository" kind="object"> + <table name="repository_fragment" kind="object"> <column name="name" type="TEXT" null="true"/> <column name="url" type="TEXT" null="true"/> <column name="type" type="TEXT" null="true"/> @@ -8,17 +8,17 @@ <column name="name"/> </primary-key> </table> - <table name="repository_complements" kind="container"> - <column name="repository" type="TEXT" null="true"/> + <table name="repository_fragment_complements" kind="container"> + <column name="repository_fragment" type="TEXT" null="true"/> <column name="complement" type="TEXT" null="true"/> - <foreign-key name="repository_fk" on-delete="CASCADE"> - <column name="repository"/> - <references table="repository"> + <foreign-key name="repository_fragment_fk" on-delete="CASCADE"> + <column name="repository_fragment"/> + <references table="repository_fragment"> <column name="name"/> </references> </foreign-key> - <index name="repository_complements_repository_i"> - <column name="repository"/> + <index name="repository_fragment_complements_repository_fragment_i"> + <column name="repository_fragment"/> </index> <foreign-key name="complement_fk" deferrable="DEFERRED"> <column name="complement"/> @@ -27,21 +27,53 @@ </references> </foreign-key> </table> - <table name="repository_prerequisites" kind="container"> - <column name="repository" type="TEXT" null="true"/> + <table name="repository_fragment_prerequisites" kind="container"> + <column name="repository_fragment" type="TEXT" null="true"/> <column name="prerequisite" type="TEXT" null="true"/> + <foreign-key name="repository_fragment_fk" on-delete="CASCADE"> + <column name="repository_fragment"/> + <references table="repository_fragment"> + <column name="name"/> + </references> + </foreign-key> + <index name="repository_fragment_prerequisites_repository_fragment_i"> + <column name="repository_fragment"/> + </index> + <foreign-key name="prerequisite_fk" deferrable="DEFERRED"> + <column name="prerequisite"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + </table> + <table name="repository" kind="object"> + <column name="name" type="TEXT" null="true"/> + <column name="url" type="TEXT" null="true"/> + <column name="type" type="TEXT" null="true"/> + <primary-key> + <column name="name"/> + </primary-key> + </table> + <table name="repository_fragments" kind="container"> + <column name="repository" type="TEXT" null="true"/> + <column name="index" type="INTEGER" null="true"/> + <column name="friendly_name" type="TEXT" null="true"/> + <column name="fragment" type="TEXT" null="true"/> <foreign-key name="repository_fk" on-delete="CASCADE"> <column name="repository"/> <references table="repository"> <column name="name"/> </references> </foreign-key> - <index name="repository_prerequisites_repository_i"> + <index name="repository_fragments_repository_i"> <column name="repository"/> </index> - <foreign-key name="prerequisite_fk" deferrable="DEFERRED"> - <column name="prerequisite"/> - <references table="repository"> + <index name="repository_fragments_index_i"> + <column name="index"/> + </index> + <foreign-key name="fragment_fk" deferrable="DEFERRED"> + <column name="fragment"/> + <references table="repository_fragment"> <column name="name"/> </references> </foreign-key> @@ -72,8 +104,7 @@ <column name="version_canonical_release" type="TEXT" null="true" options="COLLATE BINARY"/> <column name="version_revision" type="INTEGER" null="true"/> <column name="version_iteration" type="INTEGER" null="true"/> - <column name="repository" type="TEXT" null="true"/> - <column name="fragment" type="TEXT" null="true"/> + <column name="repository_fragment" type="TEXT" null="true"/> <column name="location" type="TEXT" null="true"/> <foreign-key name="object_id_fk" on-delete="CASCADE"> <column name="name"/> @@ -99,9 +130,9 @@ <column name="version_revision"/> <column name="version_iteration"/> </index> - <foreign-key name="repository_fk" deferrable="DEFERRED"> - <column name="repository"/> - <references table="repository"> + <foreign-key name="repository_fragment_fk" deferrable="DEFERRED"> + <column name="repository_fragment"/> + <references table="repository_fragment"> <column name="name"/> </references> </foreign-key> @@ -209,8 +240,8 @@ <column name="substate" type="TEXT" null="true"/> <column name="hold_package" type="INTEGER" null="true"/> <column name="hold_version" type="INTEGER" null="true"/> - <column name="repository_url" type="TEXT" null="true"/> - <column name="repository_type" type="TEXT" null="true"/> + <column name="repository_fragment_url" type="TEXT" null="true"/> + <column name="repository_fragment_type" type="TEXT" null="true"/> <column name="archive" type="TEXT" null="true"/> <column name="purge_archive" type="INTEGER" null="true"/> <column name="src_root" type="TEXT" null="true"/> diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index aea5420..9f3d732 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -9,7 +9,7 @@ #include <list> #include <cstring> // strlen() #include <iostream> // cout -#include <algorithm> // find(), find_if() +#include <algorithm> // find_if() #include <libbutl/url.mxx> @@ -125,38 +125,37 @@ namespace bpkg return db.query<available_package> (q); } - // Try to find a package that optionally satisfies the specified - // version constraint. Look in the specified repository, its - // prerequisite repositories, and their complements, recursively - // (note: recursivity applies to complements, not prerequisites). - // Return the package and the repository in which it was found or - // NULL for both if not found. Note that a stub satisfies any - // constraint. + // Try to find a package that optionally satisfies the specified version + // constraint. Look in the specified repository fragment, its prerequisite + // repositories, and their complements, recursively (note: recursivity + // applies to complements, not prerequisites). Return the package and the + // repository fragment in which it was found or NULL for both if not found. + // Note that a stub satisfies any constraint. // - static pair<shared_ptr<available_package>, shared_ptr<repository>> + static pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> find_available (database& db, const string& name, - const shared_ptr<repository>& r, + const shared_ptr<repository_fragment>& rf, const optional<dependency_constraint>& c, bool prereq = true) { - // Filter the result based on the repository to which each version - // belongs. + // Filter the result based on the repository fragment to which each + // version belongs. // - return filter_one (r, query_available (db, name, c), prereq); + return filter_one (rf, query_available (db, name, c), prereq); } // Create a transient (or fake, if you prefer) available_package object // corresponding to the specified selected object. Note that the package - // locations list is left empty and that the returned repository could be - // NULL if the package is an orphan. + // locations list is left empty and that the returned repository fragment + // could be NULL if the package is an orphan. // // Note also that in our model we assume that make_available() is only // called if there is no real available_package. This makes sure that if // the package moves (e.g., from testing to stable), then we will be using // stable to resolve its dependencies. // - static pair<shared_ptr<available_package>, shared_ptr<repository>> + static pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> make_available (const common_options& options, const dir_path& c, database& db, @@ -168,16 +167,17 @@ namespace bpkg return make_pair (make_shared<available_package> (sp->name, sp->version), nullptr); - // First see if we can find its repository. + // First see if we can find its repository fragment. // - // Note that this is package's "old" repository and there is no guarantee - // that its dependencies are still resolvable from it. But this is our - // best chance (we could go nuclear and point all orphans to the root - // repository but that feels a bit too drastic at the moment). + // Note that this is package's "old" repository fragment and there is no + // guarantee that its dependencies are still resolvable from it. But this + // is our best chance (we could go nuclear and point all orphans to the + // root repository fragment but that feels a bit too drastic at the + // moment). // - shared_ptr<repository> ar ( - db.find<repository> ( - sp->repository.canonical_name ())); + shared_ptr<repository_fragment> af ( + db.find<repository_fragment> ( + sp->repository_fragment.canonical_name ())); // The package is in at least fetched state, which means we should // be able to get its manifest. @@ -193,7 +193,7 @@ namespace bpkg // m.version = sp->version; - return make_pair (make_shared<available_package> (move (m)), move (ar)); + return make_pair (make_shared<available_package> (move (m)), move (af)); } // A "dependency-ordered" list of packages and their prerequisites. @@ -258,7 +258,10 @@ namespace bpkg shared_ptr<selected_package> selected; // NULL if not selected. shared_ptr<available_package> available; // Can be NULL, fake/transient. - shared_ptr<bpkg::repository> repository; // Can be NULL (orphan) or root. + + // Can be NULL (orphan) or root. + // + shared_ptr<bpkg::repository_fragment> repository_fragment; const string& name () const @@ -440,8 +443,8 @@ namespace bpkg // Packages collection of whose prerequisites has been postponed due the // inability to find a version satisfying the pre-entered constraint from // repositories available to this package. The idea is that this - // constraint could still be satisfied from a repository of some other - // package (that we haven't processed yet) that also depends on this + // constraint could still be satisfied from a repository fragment of some + // other package (that we haven't processed yet) that also depends on this // prerequisite. // using postponed_packages = set<const build_package*>; @@ -616,8 +619,9 @@ namespace bpkg // already configured since that would mean all its prerequisites are // configured as well. Note that this is not merely an optimization: the // package could be an orphan in which case the below logic will fail (no - // repository in which to search for prerequisites). By skipping the - // prerequisite check we are able to gracefully handle configured orphans. + // repository fragment in which to search for prerequisites). By skipping + // the prerequisite check we are able to gracefully handle configured + // orphans. // void collect_build_prerequisites (const common_options& options, @@ -649,7 +653,7 @@ namespace bpkg })); const shared_ptr<available_package>& ap (pkg.available); - const shared_ptr<repository>& ar (pkg.repository); + const shared_ptr<repository_fragment>& af (pkg.repository_fragment); const string& name (ap->id.name); for (const dependency_alternatives& da: ap->dependencies) @@ -756,7 +760,9 @@ namespace bpkg // shared_ptr<selected_package> dsp (db.find<selected_package> (dn)); - pair<shared_ptr<available_package>, shared_ptr<repository>> rp; + pair<shared_ptr<available_package>, + shared_ptr<repository_fragment>> rp; + shared_ptr<available_package>& dap (rp.first); bool force (false); @@ -781,7 +787,9 @@ namespace bpkg // system package we pick the latest one (its exact version // doesn't really matter). // - shared_ptr<repository> root (db.load<repository> ("")); + shared_ptr<repository_fragment> root ( + db.load<repository_fragment> ("")); + rp = system ? find_available (db, dn, root, nullopt) : find_available (db, @@ -807,11 +815,12 @@ namespace bpkg // if (dap == nullptr) { - // And if we have no repository to look in, then that means the - // package is an orphan (we delay this check until we actually - // need the repository to allow orphans without prerequisites). + // And if we have no repository fragment to look in, then that means + // the package is an orphan (we delay this check until we actually + // need the repository fragment to allow orphans without + // prerequisites). // - if (ar == nullptr) + if (af == nullptr) fail << "package " << pkg.available_name_version () << " is orphaned" << info << "explicitly upgrade it to a new version"; @@ -833,8 +842,8 @@ namespace bpkg // Also note that for the user-specified dependency version we rely // on its presence in repositories of the first dependent met. As // a result, we may fail too early if the version doesn't belong to - // its repository, but belongs to the one of some dependent that we - // haven't met yet. Can we just search all repositories for an + // its repositories, but belongs to the ones of some dependent that + // we haven't met yet. Can we just search all repositories for an // available package of this version and just take it, if present? // We could, but then which repository should we pick? The wrong // choice can introduce some unwanted repositories and package @@ -846,7 +855,7 @@ namespace bpkg // the package is recognized. An unrecognized package means the // broken/stale repository (see below). // - rp = find_available (db, dn, ar, !system ? d.constraint : nullopt); + rp = find_available (db, dn, af, !system ? d.constraint : nullopt); if (dap == nullptr) { @@ -868,8 +877,8 @@ namespace bpkg dr << " of package " << name; - if (!ar->location.empty () && (!dep_constr || system)) - dr << info << "repository " << ar->location << " appears to " + if (!af->location.empty () && (!dep_constr || system)) + dr << info << "repository " << af->location << " appears to " << "be broken" << info << "or the repository state could be stale" << info << "run 'bpkg rep-fetch' to update"; @@ -1395,7 +1404,7 @@ namespace bpkg return build_package { build_package::adjust, move (dsp), - nullptr, // No available package/repository. + nullptr, // No available package/repository fragment. nullptr, nullopt, // Hold package. nullopt, // Hold version. @@ -1515,8 +1524,8 @@ namespace bpkg // this dependency. If the result is a NULL available_package, then it is // either no longer used and can be dropped, or no changes to the dependency // are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository it must come from, and the - // system flag. + // upgrade/downgrade to as well as the repository fragment it must come + // from, and the system flag. // // If the explicitly specified dependency version can not be found in the // dependents repositories, then return the "no changes are necessary" @@ -1528,7 +1537,7 @@ namespace bpkg struct evaluate_result { shared_ptr<available_package> available; - shared_ptr<bpkg::repository> repository; + shared_ptr<bpkg::repository_fragment> repository_fragment; bool unused; bool system; // Is meaningless if unused. }; @@ -1541,7 +1550,7 @@ namespace bpkg const shared_ptr<selected_package>&, const version& desired, bool desired_sys, - const set<shared_ptr<repository>>&, + const set<shared_ptr<repository_fragment>>&, const package_dependents&, bool ignore_unsatisfiable); @@ -1567,7 +1576,7 @@ namespace bpkg l5 ([&]{trace << *sp << ": unused";}); return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, true /* unused */, false /* system */}; } @@ -1598,15 +1607,16 @@ namespace bpkg l5 ([&]{trace << *sp << ": unchanged";}); return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, false /* unused */, false /* system */}; } - // Build a set of repositories the dependent packages now come from. Also - // cache the dependents and the constraints they apply to this dependency. + // Build a set of repository fragments the dependent packages now come + // from. Also cache the dependents and the constraints they apply to this + // dependency. // - set<shared_ptr<repository>> repos; + set<shared_ptr<repository_fragment>> repo_frags; package_dependents dependents; for (auto& pd: pds) @@ -1622,7 +1632,7 @@ namespace bpkg assert (!dap->locations.empty ()); for (const auto& pl: dap->locations) - repos.insert (pl.repository.load ()); + repo_frags.insert (pl.repository_fragment.load ()); } dependents.emplace_back (move (dsp), move (pd.constraint)); @@ -1632,7 +1642,7 @@ namespace bpkg sp, dv, dsys, - repos, + repo_frags, dependents, ignore_unsatisfiable); } @@ -1642,7 +1652,7 @@ namespace bpkg const shared_ptr<selected_package>& sp, const version& dv, bool dsys, - const set<shared_ptr<repository>>& repos, + const set<shared_ptr<repository_fragment>>& rfs, const package_dependents& dependents, bool ignore_unsatisfiable) { @@ -1658,19 +1668,22 @@ namespace bpkg ? query_available (db, nm, nullopt) : query_available (db, nm, dependency_constraint (dv))); - vector<pair<shared_ptr<available_package>, shared_ptr<repository>>> ars ( - filter (vector<shared_ptr<repository>> (repos.begin (), repos.end ()), - move (apr))); + vector<pair<shared_ptr<available_package>, + shared_ptr<repository_fragment>>> afs ( + filter (vector<shared_ptr<repository_fragment>> ( + rfs.begin (), + rfs.end ()), + move (apr))); auto no_change = [] () { return evaluate_result {nullptr /* available */, - nullptr /* repository */, + nullptr /* repository_fragment */, false /* unused */, false /* system */}; }; - if (ars.empty ()) + if (afs.empty ()) { if (ignore_unsatisfiable) { @@ -1706,9 +1719,9 @@ namespace bpkg assert (!dsys || system_repository.find (nm) != nullptr); - for (auto& ar: ars) + for (auto& af: afs) { - shared_ptr<available_package>& ap (ar.first); + shared_ptr<available_package>& ap (af.first); const version& av (!dsys ? ap->version : *ap->system_version ()); // If we aim to upgrade to the highest possible version and it tends to @@ -1771,7 +1784,7 @@ namespace bpkg << package_string (nm, av, dsys);}); return evaluate_result { - move (ap), move (ar.second), false /* unused */, dsys}; + move (ap), move (af.second), false /* unused */, dsys}; } // If we aim to upgrade to the highest possible version, then what we @@ -1875,7 +1888,8 @@ namespace bpkg // Evaluate a package (not necessarily dependency) and return a new desired // version. If the result is absent (nullopt), then no changes to the // package are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository it must come from. + // upgrade/downgrade to as well as the repository fragment it must come + // from. // // If the system package cannot be upgraded to the source one, not being // found in the dependents repositories, then return nullopt if @@ -1892,18 +1906,19 @@ namespace bpkg assert (sp != nullptr); - // Build a set of repositories the dependent packages come from. Also - // cache the dependents and the constraints they apply to this dependency. + // Build a set of repository fragment the dependent packages come from. + // Also cache the dependents and the constraints they apply to this + // dependency. // - set<shared_ptr<repository>> repos; + set<shared_ptr<repository_fragment>> repo_frags; package_dependents dependents; auto pds (db.query<package_dependent> ( query<package_dependent>::name == sp->name)); - // Only collect repositories (for best version selection) of (immediate) - // dependents that have a hit (direct or indirect) in recs. Note, however, - // that we collect constraints from all the dependents. + // Only collect repository fragments (for best version selection) of + // (immediate) dependents that have a hit (direct or indirect) in recs. + // Note, however, that we collect constraints from all the dependents. // bool upgrade (false); @@ -1916,8 +1931,8 @@ namespace bpkg continue; // While we already know that the dependency upgrade is required, we - // continue to iterate over dependents, collecting the repositories and - // the constraints. + // continue to iterate over dependents, collecting the repository + // fragments and the constraints. // upgrade = true; @@ -1930,7 +1945,7 @@ namespace bpkg assert (!dap->locations.empty ()); for (const auto& pl: dap->locations) - repos.insert (pl.repository.load ()); + repo_frags.insert (pl.repository_fragment.load ()); } } @@ -1947,7 +1962,7 @@ namespace bpkg sp, version () /* desired */, false /*desired_sys */, - repos, + repo_frags, dependents, ignore_unsatisfiable)); @@ -2374,14 +2389,21 @@ namespace bpkg // map<string, version> pvs; - using query = query<repository_package>; - - for (const auto& rp: db.query<repository_package> ( - (query::repository::name == r->name) + - order_by_version_desc (query::package::id.version))) + for (const repository::fragment_type& rf: r->fragments) { - const shared_ptr<available_package>& p (rp); - pvs.insert (make_pair (p->id.name, p->version)); + using query = query<repository_fragment_package>; + + for (const auto& rp: db.query<repository_fragment_package> ( + (query::repository_fragment::name == + rf.fragment.load ()->name) + + order_by_version_desc (query::package::id.version))) + { + const shared_ptr<available_package>& p (rp); + auto i (pvs.insert (make_pair (p->id.name, p->version))); + + if (!i.second && i.first->second < p->version) + i.first->second = p->version; + } } // Populate the argument list with the latest package versions. @@ -2422,15 +2444,27 @@ namespace bpkg ? nullopt : optional<dependency_constraint> (v)); - shared_ptr<available_package> ap ( - find_available (db, n, r, c, false /* prereq */).first); + bool complements (false); + shared_ptr<available_package> ap; + + for (const repository::fragment_type& rf: r->fragments) + { + shared_ptr<repository_fragment> fr (rf.fragment.load ()); + ap = find_available (db, n, fr, c, false /* prereq */).first; + + if (ap != nullptr) + break; + + if (!fr->complements.empty ()) + complements = true; + } if (ap == nullptr) { diag_record dr (fail); dr << "package " << pkg << " is not found in " << r->name; - if (!r->complements.empty ()) + if (complements) dr << " or its complements"; } @@ -2485,7 +2519,7 @@ namespace bpkg transaction t (db); - shared_ptr<repository> root (db.load<repository> ("")); + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); // Here is what happens here: for unparsed package args we are going to // try and guess whether we are dealing with a package archive, package @@ -2504,7 +2538,7 @@ namespace bpkg // Reduce all the potential variations (archive, directory, package // name, package name/version) to a single available_package object. // - shared_ptr<repository> ar; + shared_ptr<repository_fragment> af; shared_ptr<available_package> ap; if (!arg_parsed (pa)) @@ -2544,12 +2578,9 @@ namespace bpkg m.version, move (pa.options)); - ar = root; + af = root; ap = make_shared<available_package> (move (m)); - ap->locations.push_back ( - package_location {root, - string () /* fragment */, - move (a)}); + ap->locations.push_back (package_location {root, move (a)}); } } catch (const invalid_path&) @@ -2596,12 +2627,12 @@ namespace bpkg l4 ([&]{trace << "directory '" << d << "': " << arg_string (pa);}); - // Supporting this would complicate things a bit, but we may add - // support for it one day. - // - if (pa.options.dependency ()) - fail << "package directory '" << d - << "' may not be built as a dependency"; + // Supporting this would complicate things a bit, but we may + // add support for it one day. + // + if (pa.options.dependency ()) + fail << "package directory '" << d + << "' may not be built as a dependency"; // Fix-up the package version to properly decide if we need to // upgrade/downgrade the package. Note that throwing failed @@ -2626,11 +2657,8 @@ namespace bpkg move (pa.options)); ap = make_shared<available_package> (move (m)); - ar = root; - ap->locations.push_back ( - package_location {root, - string () /* fragment */, - move (d)}); + af = root; + ap->locations.push_back (package_location {root, move (d)}); } } catch (const invalid_path&) @@ -2691,7 +2719,7 @@ namespace bpkg root, dependency_constraint (pa.version))); ap = move (rp.first); - ar = move (rp.second); + af = move (rp.second); } } catch (const failed&) @@ -2895,7 +2923,7 @@ namespace bpkg auto rp (make_available (o, c, db, sp)); ap = rp.first; - ar = rp.second; // Could be NULL (orphan). + af = rp.second; // Could be NULL (orphan). } // We will keep the output directory only if the external package is @@ -2913,7 +2941,7 @@ namespace bpkg build_package::build, move (sp), move (ap), - move (ar), + move (af), true, // Hold package. !pa.version.empty (), // Hold version. {}, // Constraints. @@ -3068,9 +3096,13 @@ namespace bpkg { struct dep { - string name; // Empty if up/down-grade. - shared_ptr<available_package> available; // NULL if drop. - shared_ptr<bpkg::repository> repository; // NULL if drop. + string name; // Empty if up/down-grade. + + // Both are NULL if drop. + // + shared_ptr<available_package> available; + shared_ptr<bpkg::repository_fragment> repository_fragment; + bool system; }; vector<dep> deps; @@ -3103,8 +3135,8 @@ namespace bpkg build_package bp { nullopt, // Action. nullptr, // Selected package. - nullptr, // Available package. - nullptr, // Available package repository. + nullptr, // Available package/repository fragment. + nullptr, false, // Hold package. !p.version.empty (), // Hold version. {}, // Constraints. @@ -3175,14 +3207,14 @@ namespace bpkg build_package::build, move (sp), d.available, - d.repository, - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. + d.repository_fragment, + nullopt, // Hold package. + nullopt, // Hold version. + {}, // Constraints. d.system, keep_out, - {""}, // Required by (command line). - 0}; // Adjustments. + {""}, // Required by (command line). + 0}; // Adjustments. pkgs.collect_build (o, c, db, p, &postponed /* recursively */); } @@ -3373,7 +3405,7 @@ namespace bpkg if (!diag) deps.push_back (dep {sp->name, move (er->available), - move (er->repository), + move (er->repository_fragment), er->system}); r = true; } @@ -3752,7 +3784,7 @@ namespace bpkg const shared_ptr<available_package>& ap (p.available); const package_location& pl (ap->locations[0]); - if (pl.repository.object_id () == "") // Special root. + if (pl.repository_fragment.object_id () == "") // Special root. p.keep_out = !exists (pl.location); // Directory case. else { @@ -3761,12 +3793,12 @@ namespace bpkg // See if the package comes from the directory-based repository, and // so is external. // - // Note that such repositories are always preferred over others (see - // below). + // Note that such repository fragments are always preferred over + // others (see below). // for (const package_location& l: ap->locations) { - if (l.repository.load ()->location.directory_based ()) + if (l.repository_fragment.load ()->location.directory_based ()) { p.keep_out = true; break; @@ -3875,20 +3907,22 @@ namespace bpkg // const package_location& pl (ap->locations[0]); // Got to have one. - if (pl.repository.object_id () != "") // Special root? + if (pl.repository_fragment.object_id () != "") // Special root? { transaction t (db, !simulate /* start */); - // Go through package repositories to decide if we should fetch, - // checkout or unpack depending on the available repository basis. - // Preferring a local one over the remotes and the dir repository - // type over the others seems like a sensible thing to do. + // Go through package repository fragments to decide if we should + // fetch, checkout or unpack depending on the available repository + // basis. Preferring a local one over the remotes and the dir + // repository type over the others seems like a sensible thing to + // do. // optional<repository_basis> basis; for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl ( + l.repository_fragment.load ()->location); if (!basis || rl.local ()) // First or local? { @@ -3963,7 +3997,7 @@ namespace bpkg if (verbose && !o.no_result ()) { - const repository_location& rl (sp->repository); + const repository_location& rl (sp->repository_fragment); repository_basis basis ( !rl.empty () @@ -4016,7 +4050,7 @@ namespace bpkg else { const package_location& pl (ap->locations[0]); - assert (pl.repository.object_id () == ""); // Special root. + assert (pl.repository_fragment.object_id () == ""); // Special root. transaction t (db, !simulate /* start */); sp = pkg_unpack (o, diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx index 1759991..18ce2b9 100644 --- a/bpkg/pkg-checkout.cxx +++ b/bpkg/pkg-checkout.cxx @@ -27,14 +27,15 @@ namespace bpkg checkout (const common_options& o, const repository_location& rl, const dir_path& dir, - const string& fragment, const shared_ptr<available_package>& ap) { switch (rl.type ()) { case repository_type::git: { - git_checkout (o, dir, fragment); + assert (rl.fragment ()); + + git_checkout (o, dir, *rl.fragment ()); if (exists (dir / path (".gitmodules"))) { @@ -45,7 +46,7 @@ namespace bpkg text << "checking out " << package_string (ap->id.name, ap->version); - git_checkout_submodules (o, dir); + git_checkout_submodules (o, rl, dir); } break; @@ -103,14 +104,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick a version control-based repository. Preferring a local one over - // the remotes seems like a sensible thing to do. + // Pick a version control-based repository fragment. Preferring a local + // one over the remotes seems like a sensible thing to do. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl (l.repository_fragment.load ()->location); if (rl.version_control_based () && (pl == nullptr || rl.local ())) { @@ -127,22 +128,18 @@ namespace bpkg if (verb > 1) text << "checking out " << pl->location.leaf () << " " - << "from " << pl->repository->name; - - const repository_location& rl (pl->repository->location); + << "from " << pl->repository_fragment->name; - // Note: for now we assume this is a git repository. If/when we add other - // version control-based repositories, this will need adjustment. + // Checkout the repository fragment. // + const repository_location& rl (pl->repository_fragment->location); - // Currently the git repository state already contains the checked out - // working tree so all we need to do is distribute it to the package - // directory. - // dir_path sd (c / repos_dir / repository_state (rl)); + checkout (o, rl, sd, ap); - checkout (o, rl, sd, pl->fragment, ap); - + // Calculate the package path that points into the checked out fragment + // directory. + // sd /= path_cast<dir_path> (pl->location); // Verify the package prerequisites are all configured since the dist @@ -219,7 +216,7 @@ namespace bpkg p->version = move (v); p->state = package_state::unpacked; - p->repository = rl; + p->repository_fragment = rl; p->src_root = d.leaf (); p->purge_src = true; p->manifest_checksum = move (mc); diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx index 7aa4183..e351cfd 100644 --- a/bpkg/pkg-configure.cxx +++ b/bpkg/pkg-configure.cxx @@ -208,7 +208,7 @@ namespace bpkg package_substate::system, false, // Don't hold package. false, // Don't hold version. - repository_location (), // Root repository. + repository_location (), // Root repository fragment. nullopt, // No source archive. false, // No auto-purge (does not get there). nullopt, // No source directory. @@ -278,12 +278,12 @@ namespace bpkg if (p != nullptr) fail << "package " << n << " already exists in configuration " << c; - shared_ptr<repository> rep (db.load<repository> ("")); // Root. + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); using query = query<available_package>; query q (query::id.name == n); - if (filter_one (rep, db.query<available_package> (q)).first == nullptr) + if (filter_one (root, db.query<available_package> (q)).first == nullptr) fail << "unknown package " << n; p = pkg_configure_system (n, v.empty () ? wildcard_version : v, t); diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx index a5efa47..36aaa21 100644 --- a/bpkg/pkg-fetch.cxx +++ b/bpkg/pkg-fetch.cxx @@ -58,7 +58,7 @@ namespace bpkg p->version = move (v); p->state = package_state::fetched; - p->repository = move (rl); + p->repository_fragment = move (rl); p->archive = move (a); p->purge_archive = purge; @@ -153,8 +153,8 @@ namespace bpkg // pkg_fetch_check (c, t, m.name, replace); - // Use the special root repository as the repository of this - // package. + // Use the special root repository fragment as the repository fragment of + // this package. // return pkg_fetch (c, t, @@ -197,14 +197,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick an archive-based repository. Preferring a local one over the - // remotes seems like a sensible thing to do. + // Pick an archive-based repository fragment. Preferring a local one over + // the remotes seems like a sensible thing to do. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository.load ()->location); + const repository_location& rl (l.repository_fragment.load ()->location); if (rl.archive_based () && (pl == nullptr || rl.local ())) { @@ -221,14 +221,16 @@ namespace bpkg if (verb > 1) text << "fetching " << pl->location.leaf () << " " - << "from " << pl->repository->name; + << "from " << pl->repository_fragment->name; auto_rmfile arm; path a (c / pl->location.leaf ()); if (!simulate) { - pkg_fetch_archive (co, pl->repository->location, pl->location, a); + pkg_fetch_archive ( + co, pl->repository_fragment->location, pl->location, a); + arm = auto_rmfile (a); // We can't be fetching an archive for a transient object. @@ -239,7 +241,7 @@ namespace bpkg if (sha256sum != *ap->sha256sum) { fail << "checksum mismatch for " << n << " " << v << - info << pl->repository->name << " has " << *ap->sha256sum << + info << pl->repository_fragment->name << " has " << *ap->sha256sum << info << "fetched archive has " << sha256sum << info << "consider re-fetching package list and trying again" << info << "if problem persists, consider reporting this to " @@ -253,7 +255,7 @@ namespace bpkg move (n), move (v), move (a), - pl->repository->location, + pl->repository_fragment->location, true /* purge */, simulate)); diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx index ea39b25..f621281 100644 --- a/bpkg/pkg-status.cxx +++ b/bpkg/pkg-status.cxx @@ -66,7 +66,8 @@ namespace bpkg bool known (false); bool build (false); { - shared_ptr<repository> rep (db.load<repository> ("")); // Root. + shared_ptr<repository_fragment> root ( + db.load<repository_fragment> ("")); using query = query<available_package>; @@ -74,7 +75,7 @@ namespace bpkg { auto r (db.query<available_package> (q)); known = !r.empty (); - build = filter_one (rep, move (r)).first != nullptr; + build = filter_one (root, move (r)).first != nullptr; } if (known) @@ -109,7 +110,7 @@ namespace bpkg pointer_result ( db.query<available_package> (q))) { - bool build (filter (rep, ap)); + bool build (filter (root, ap)); apkgs.push_back (apkg {move (ap), build}); } } diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx index ace8d53..984a41e 100644 --- a/bpkg/pkg-unpack.cxx +++ b/bpkg/pkg-unpack.cxx @@ -106,7 +106,7 @@ namespace bpkg p->version = move (v); p->state = package_state::unpacked; - p->repository = move (rl); + p->repository_fragment = move (rl); p->src_root = move (d); p->purge_src = purge; p->manifest_checksum = move (mc); @@ -172,8 +172,8 @@ namespace bpkg o, c, t, d, m.name, m.version, true /* check_external */)) m.version = move (*v); - // Use the special root repository as the repository of this - // package. + // Use the special root repository fragment as the repository fragment of + // this package. // return pkg_unpack (o, c, @@ -215,14 +215,14 @@ namespace bpkg if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; - // Pick a directory-based repository. They are always local, so we pick - // the first one. + // Pick a directory-based repository fragment. They are always local, so we + // pick the first one. // const package_location* pl (nullptr); for (const package_location& l: ap->locations) { - if (l.repository.load ()->location.directory_based ()) + if (l.repository_fragment.load ()->location.directory_based ()) { pl = &l; break; @@ -235,9 +235,9 @@ namespace bpkg if (verb > 1) text << "unpacking " << pl->location.leaf () << " " - << "from " << pl->repository->name; + << "from " << pl->repository_fragment->name; - const repository_location& rl (pl->repository->location); + const repository_location& rl (pl->repository_fragment->location); return pkg_unpack (o, c, diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx index 3569584..280c780 100644 --- a/bpkg/rep-add.cxx +++ b/bpkg/rep-add.cxx @@ -40,7 +40,7 @@ namespace bpkg updated = true; } - shared_ptr<repository> root (db.load<repository> ("")); + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); bool added ( root->complements.insert (lazy_shared_ptr<repository> (db, r)).second); diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index c92db10..af07cb5 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -52,13 +52,14 @@ namespace bpkg pair<pkg_repository_manifests, string /* checksum */> rmc ( pkg_fetch_repositories (co, rl, ignore_unknown)); - pkg_repository_manifests& rms (rmc.first); + rep_fetch_data::fragment fr; + fr.repositories = move (rmc.first); bool a (co.auth () != auth::none && (co.auth () == auth::all || rl.remote ())); shared_ptr<const certificate> cert; - const optional<string>& cert_pem (rms.back ().certificate); + optional<string> cert_pem (move (fr.repositories.back ().certificate)); if (a) { @@ -79,6 +80,8 @@ namespace bpkg << rl.canonical_name () << info << "try again"; + fr.packages = move (pms); + if (a) { signature_manifest sm ( @@ -93,15 +96,7 @@ namespace bpkg authenticate_repository (co, conf, cert_pem, *cert, sm, rl); } - vector<rep_fetch_data::package> fps; - fps.reserve (pms.size ()); - - for (package_manifest& m: pms) - fps.emplace_back ( - rep_fetch_data::package {move (m), - string () /* repository_state */}); - - return rep_fetch_data {move (rms), move (fps), move (cert)}; + return rep_fetch_data {{move (fr)}, move (cert_pem), move (cert)}; } template <typename M> @@ -109,7 +104,7 @@ namespace bpkg parse_manifest (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional<string>& fragment) // Used for diagnostics. { try { @@ -119,8 +114,15 @@ namespace bpkg } catch (const manifest_parsing& e) { - fail (e.name, e.line, e.column) << e.description << - info << "repository " << rl << ' ' << frag << endf; + diag_record dr (fail (e.name, e.line, e.column)); + + dr << e.description << + info << "repository " << rl; + + if (fragment) + dr << ' ' << *fragment; + + dr << endf; } catch (const io_error& e) { @@ -136,11 +138,11 @@ namespace bpkg parse_repository_manifests (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional<string>& fragment) { M r; if (exists (f)) - r = parse_manifest<M> (f, iu, rl, frag); + r = parse_manifest<M> (f, iu, rl, fragment); else r.emplace_back (repository_manifest ()); // Add the base repository. @@ -156,11 +158,11 @@ namespace bpkg parse_directory_manifests (const path& f, bool iu, const repository_location& rl, - const string& frag) + const optional<string>& fragment) { M r; if (exists (f)) - r = parse_manifest<M> (f, iu, rl, frag); + r = parse_manifest<M> (f, iu, rl, fragment); else { r.push_back (package_manifest ()); @@ -172,30 +174,32 @@ namespace bpkg // Parse package manifests referenced by the package directory manifests. // - static vector<rep_fetch_data::package> + static vector<package_manifest> parse_package_manifests (const common_options& co, const dir_path& repo_dir, - const string& repo_fragment, vector<package_manifest>&& sms, bool iu, const repository_location& rl, - const string& frag) + const optional<string>& fragment) // For diagnostics. { - vector<rep_fetch_data::package> fps; - fps.reserve (sms.size ()); + vector<package_manifest> r; + r.reserve (sms.size ()); for (package_manifest& sm: sms) { assert (sm.location); - auto package_info = [&sm, &rl, &frag] (diag_record& dr) + auto package_info = [&sm, &rl, &fragment] (diag_record& dr) { dr << "package "; if (!sm.location->current ()) dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. - dr << "in repository " << rl << ' ' << frag; + dr << "in repository " << rl; + + if (fragment) + dr << ' ' << *fragment; }; auto failure = [&package_info] (const char* desc) @@ -240,10 +244,10 @@ namespace bpkg if (v) sm.version = move (*v); - fps.emplace_back (rep_fetch_data::package {move (sm), repo_fragment}); + r.emplace_back (move (sm)); } - return fps; + return r; } static rep_fetch_data @@ -255,30 +259,31 @@ namespace bpkg dir_path rd (path_cast<dir_path> (rl.path ())); - dir_repository_manifests rms ( - parse_repository_manifests<dir_repository_manifests> ( - rd / repositories_file, - ignore_unknown, - rl, - string () /* frag */)); + rep_fetch_data::fragment fr; + + fr.repositories = parse_repository_manifests<dir_repository_manifests> ( + rd / repositories_file, + ignore_unknown, + rl, + string () /* fragment */); dir_package_manifests pms ( parse_directory_manifests<dir_package_manifests> ( rd / packages_file, ignore_unknown, rl, - string () /* frag */)); - - vector<rep_fetch_data::package> fps ( - parse_package_manifests (co, - rd, - string () /* repo_fragment */, - move (pms), - ignore_unknown, - rl, - string () /* frag */)); - - return rep_fetch_data {move (rms), move (fps), nullptr}; + string () /* fragment */)); + + fr.packages = parse_package_manifests (co, + rd, + move (pms), + ignore_unknown, + rl, + string () /* fragment */); + + return rep_fetch_data {{move (fr)}, + nullopt /* cert_pem */, + nullptr /* certificate */}; } static rep_fetch_data @@ -358,21 +363,23 @@ namespace bpkg // - For each package location parse the package manifest and add it to // the resulting list. // - git_repository_manifests rms; - vector<rep_fetch_data::package> fps; + rep_fetch_data r; - for (const git_fragment& fr: git_fetch (co, rl, td)) + for (git_fragment& gf: git_fetch (co, rl, td)) { - git_checkout (co, td, fr.commit); + git_checkout (co, td, gf.commit); + + rep_fetch_data::fragment fr; + fr.id = move (gf.commit); + fr.friendly_name = move (gf.friendly_name); // Parse repository manifests. // - if (rms.empty ()) - rms = parse_repository_manifests<git_repository_manifests> ( + fr.repositories = parse_repository_manifests<git_repository_manifests> ( td / repositories_file, ignore_unknown, rl, - fr.name); + fr.friendly_name); // Parse package skeleton manifests. // @@ -381,7 +388,7 @@ namespace bpkg td / packages_file, ignore_unknown, rl, - fr.name)); + fr.friendly_name)); // Checkout submodules, if required. // @@ -391,25 +398,20 @@ namespace bpkg if (!exists (d) || empty (d)) { - git_checkout_submodules (co, td); + git_checkout_submodules (co, rl, td); break; } } // Parse package manifests. // - vector<rep_fetch_data::package> cps ( - parse_package_manifests (co, - td, - fr.commit, - move (pms), - ignore_unknown, - rl, - fr.name)); - - fps.insert (fps.end (), - make_move_iterator (cps.begin ()), - make_move_iterator (cps.end ())); + fr.packages = parse_package_manifests (co, + td, + move (pms), + ignore_unknown, + rl, + fr.friendly_name); + r.fragments.push_back (move (fr)); } // Move the state directory to its proper place. @@ -424,7 +426,7 @@ namespace bpkg filesystem_state_changed = true; } - return rep_fetch_data {move (rms), move (fps), nullptr /* certificate */}; + return r; } rep_fetch_data @@ -444,72 +446,89 @@ namespace bpkg return rep_fetch_data (); } - using repositories = set<shared_ptr<repository>>; - - // If reason is absent, then don't print the "fetching ..." progress line. + // Return an existing repository fragment or create a new one. Update the + // existing object unless it is immutable (see repository_fragment class + // description for details). Don't fetch the complement and prerequisite + // repositories. // - static void - rep_fetch (const common_options& co, - const dir_path& conf, - transaction& t, - const shared_ptr<repository>& r, - repositories& fetched, - repositories& removed, - bool shallow, - const optional<string>& reason) + static shared_ptr<repository_fragment> + rep_fragment (const common_options& co, + const dir_path& conf, + transaction& t, + const repository_location& rl, + rep_fetch_data::fragment&& fr) { - tracer trace ("rep_fetch(rep)"); + tracer trace ("rep_fragment"); database& db (t.database ()); tracer_guard tg (db, trace); - // Check that the repository is not fetched yet and register it as fetched - // otherwise. - // - // Note that we can end up with a repository dependency cycle via - // prerequisites. Thus we register the repository before recursing into its - // dependencies. - // - if (!fetched.insert (r).second) // Is already fetched. - return; + bool mut (fr.id.empty ()); // Is the fragment mutable? - const repository_location& rl (r->location); - l4 ([&]{trace << r->name << " " << rl;}); - - // Cancel the repository removal. - // - // Note that this is an optimization as the rep_remove() function checks - // for reachability of the repository being removed. + // Calculate the fragment location. // - removed.erase (r); + repository_location rfl; - // The fetch_*() functions below will be quiet at level 1, which can be - // quite confusing if the download hangs. - // - if (verb && reason) + switch (rl.type ()) { - diag_record dr (text); + case repository_type::pkg: + case repository_type::dir: + { + assert (mut); - dr << "fetching " << r->name; + rfl = rl; + break; + } + case repository_type::git: + { + assert (!mut); - if (!reason->empty ()) - dr << " (" << *reason << ")"; + repository_url url (rl.url ()); + url.fragment = move (fr.id); + + rfl = repository_location (url, rl.type ()); + break; + } } - // Load the repository and package manifests and use them to populate the - // prerequisite and complement repository sets as well as available - // packages. + shared_ptr<repository_fragment> rf ( + db.find<repository_fragment> (rfl.canonical_name ())); + + // Return the existing repository fragment if it is immutable. // - rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + bool exists (rf != nullptr); + + if (exists) + { + // Note that the user could change the repository URL the fragment was + // fetched from. In this case we need to sync the fragment location with + // the repository location to make sure that we use a proper URL for the + // fragment checkout. Not doing so we, for example, may try to fetch git + // submodules from the URL that is not available anymore. + // + if (rfl.url () != rf->location.url ()) + { + rf->location = move (rfl); + db.update (rf); + } - // Create the new prerequisite and complement repository sets. While doing - // this we may also reset the shallow flag if discover that any of these - // sets have changed. + if (!mut) + return rf; + } + + // Create or update the repository fragment. // - repository::complements_type complements; - repository::prerequisites_type prerequisites; + if (exists) + { + assert (mut); + + rf->complements.clear (); + rf->prerequisites.clear (); + } + else + rf = make_shared<repository_fragment> (move (rfl)); - for (repository_manifest& rm: rfd.repositories) + for (repository_manifest& rm: fr.repositories) { repository_role rr (rm.effective_role ()); @@ -535,27 +554,34 @@ namespace bpkg } } - // Create the new repository if it is not in the database yet. Otherwise - // update its location. + // Create the new repository if it is not in the database yet, otherwise + // update its location if it is changed, unless the repository is a + // top-level one (and so its location is authoritative). Such a change + // may root into the top-level repository location change made by the + // user. // - shared_ptr<repository> pr (db.find<repository> (l.canonical_name ())); + shared_ptr<repository> r (db.find<repository> (l.canonical_name ())); - if (pr == nullptr) + if (r == nullptr) { - pr = make_shared<repository> (move (l)); - db.persist (pr); // Enter into session, important if recursive. - - shallow = false; + r = make_shared<repository> (move (l)); + db.persist (r); // Enter into session, important if recursive. } - else if (pr->location.url () != l.url ()) + else if (r->location.url () != l.url ()) { - pr->location = move (l); - db.update (r); + shared_ptr<repository_fragment> root ( + db.load<repository_fragment> ("")); + + repository_fragment::complements_type& ua (root->complements); - shallow = false; + if (ua.find (r) == ua.end ()) + { + r->location = move (l); + db.update (r); + } } - // @@ What if we have duplicated? Ideally, we would like to check + // @@ What if we have duplicates? Ideally, we would like to check // this once and as early as possible. The original idea was to // do it during manifest parsing and serialization. But at that // stage we have no way of completing relative locations (which @@ -574,14 +600,14 @@ namespace bpkg { case repository_role::complement: { - l4 ([&]{trace << pr->name << " complement of " << r->name;}); - complements.insert (lazy_shared_ptr<repository> (db, pr)); + l4 ([&]{trace << r->name << " complement of " << rf->name;}); + rf->complements.insert (lazy_shared_ptr<repository> (db, r)); break; } case repository_role::prerequisite: { - l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); - prerequisites.insert (lazy_weak_ptr<repository> (db, pr)); + l4 ([&]{trace << r->name << " prerequisite of " << rf->name;}); + rf->prerequisites.insert (lazy_weak_ptr<repository> (db, r)); break; } case repository_role::base: @@ -603,8 +629,8 @@ namespace bpkg case repository_type::git: case repository_type::dir: { - if (complements.empty () && prerequisites.empty ()) - complements.insert (lazy_shared_ptr<repository> (db, string ())); + if (rf->complements.empty () && rf->prerequisites.empty ()) + rf->complements.insert (lazy_shared_ptr<repository> (db, string ())); break; } @@ -617,83 +643,10 @@ namespace bpkg } } - // Reset the shallow flag if the set of complements and/or prerequisites - // has changed. - // - // Note that weak pointers are generally incomparable (as can point to - // expired objects), and thus we can't compare the prerequisite sets - // directly. - // - if (shallow) - shallow = r->complements == complements && - equal (r->prerequisites.begin (), r->prerequisites.end (), - prerequisites.begin (), prerequisites.end (), - [] (const lazy_weak_ptr<repository>& x, - const lazy_weak_ptr<repository>& y) - { - return x.object_id () == y.object_id (); - }); - - // Fetch prerequisites and complements, unless this is a shallow fetch. - // - if (!shallow) - { - // Register complements and prerequisites for potential removal unless - // they are fetched. Clear repository dependency sets afterwards. - // - auto rm = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp) - { - shared_ptr<repository> r (rp.load ()); - if (fetched.find (r) == fetched.end ()) - removed.insert (move (r)); - }; - - for (const lazy_shared_ptr<repository>& cr: r->complements) - { - // Remove the complement unless it is the root repository (see - // rep_fetch() for details). - // - if (cr.object_id () != "") - rm (cr); - } - - for (const lazy_weak_ptr<repository>& pr: r->prerequisites) - rm (lazy_shared_ptr<repository> (pr)); - - r->complements = move (complements); - r->prerequisites = move (prerequisites); - - // Fetch complements. - // - for (const auto& cr: r->complements) - { - if (cr.object_id () != "") - rep_fetch (co, - conf, - t, - cr.load (), - fetched, - removed, - false /* shallow */, - "complements " + r->name); - } - - // Fetch prerequisites. - // - for (const auto& pr: r->prerequisites) - rep_fetch (co, - conf, - t, - pr.load (), - fetched, - removed, - false /* shallow */, - "prerequisite of " + r->name); - - // Save the changes to the repository object. - // - db.update (r); - } + if (exists) + db.update (rf); + else + db.persist (rf); // "Suspend" session while persisting packages to reduce memory // consumption. @@ -701,15 +654,14 @@ namespace bpkg session& s (session::current ()); session::reset_current (); - // Remove this repository from locations of the available packages it - // contains. + // Remove this repository fragment from locations of the available + // packages it contains. // - rep_remove_package_locations (t, r->name); + if (exists) + rep_remove_package_locations (t, rf->name); - for (rep_fetch_data::package& fp: rfd.packages) + for (package_manifest& pm: fr.packages) { - package_manifest& pm (fp.manifest); - // Fix-up the external package version iteration number. // if (rl.directory_based ()) @@ -724,7 +676,7 @@ namespace bpkg co, conf, t, - path_cast<dir_path> (rl.path () / *fp.manifest.location), + path_cast<dir_path> (rl.path () / *pm.location), pm.name, pm.version, false /* check_external */)); @@ -765,7 +717,7 @@ namespace bpkg // can pick any to show to the user. // const string& r1 (rl.canonical_name ()); - const string& r2 (p->locations[0].repository.object_id ()); + const string& r2 (p->locations[0].repository_fragment.object_id ()); fail << "checksum mismatch for " << pm.name << " " << pm.version << info << r1 << " has " << *pm.sha256sum << @@ -775,12 +727,8 @@ namespace bpkg } } - // Note that the repository may already be present in the location set - // multiple times with different fragments. - // p->locations.push_back ( - package_location {lazy_shared_ptr<repository> (db, r), - move (fp.repository_fragment), + package_location {lazy_shared_ptr<repository_fragment> (db, rf), move (*pm.location)}); if (persist) @@ -790,6 +738,203 @@ namespace bpkg } session::current (s); // "Resume". + + return rf; + } + + using repositories = set<shared_ptr<repository>>; + using repository_fragments = set<shared_ptr<repository_fragment>>; + + // If reason is absent, then don't print the "fetching ..." progress line. + // + static void + rep_fetch (const common_options& co, + const dir_path& conf, + transaction& t, + const shared_ptr<repository>& r, + repositories& fetched_repositories, + repositories& removed_repositories, + repository_fragments& removed_fragments, + bool shallow, + const optional<string>& reason) + { + tracer trace ("rep_fetch(rep)"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Check that the repository is not fetched yet and register it as fetched + // otherwise. + // + // Note that we can end up with a repository dependency cycle via + // prerequisites. Thus we register the repository before recursing into its + // dependencies. + // + if (!fetched_repositories.insert (r).second) // Is already fetched. + return; + + const repository_location& rl (r->location); + l4 ([&]{trace << r->name << " " << rl;}); + + // Cancel the repository removal. + // + // Note that this is an optimization as the rep_remove() function checks + // for reachability of the repository being removed. + // + removed_repositories.erase (r); + + // The fetch_*() functions below will be quiet at level 1, which can be + // quite confusing if the download hangs. + // + if (verb && reason) + { + diag_record dr (text); + + dr << "fetching " << r->name; + + if (!reason->empty ()) + dr << " (" << *reason << ")"; + } + + // Save the current complements and prerequisites to later check if the + // shallow repository fetch is possible and to register them for removal + // if that's not the case. + // + repository_fragment::complements_type old_complements; + repository_fragment::prerequisites_type old_prerequisites; + + auto collect_deps = [] (const shared_ptr<repository_fragment>& rf, + repository_fragment::complements_type& cs, + repository_fragment::prerequisites_type& ps) + { + for (const auto& cr: rf->complements) + cs.insert (cr); + + for (const auto& pr: rf->prerequisites) + ps.insert (pr); + }; + + // While traversing fragments also register them for removal. + // + for (const repository::fragment_type& fr: r->fragments) + { + shared_ptr<repository_fragment> rf (fr.fragment.load ()); + + collect_deps (rf, old_complements, old_prerequisites); + removed_fragments.insert (rf); + } + + // Cleanup the repository fragments list. + // + r->fragments.clear (); + + // Load the repository and package manifests and use them to populate the + // repository fragments list, as well as its prerequisite and complement + // repository sets. + // + rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); + + repository_fragment::complements_type new_complements; + repository_fragment::prerequisites_type new_prerequisites; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + string nm (fr.friendly_name); // Don't move, still may be used. + + shared_ptr<repository_fragment> rf ( + rep_fragment (co, conf, t, rl, move (fr))); + + collect_deps (rf, new_complements, new_prerequisites); + + // Cancel the repository fragment removal. + // + // Note that this is an optimization as the rep_remove_fragment() + // function checks if the fragment is dangling prio to the removal. + // + removed_fragments.erase (rf); + + r->fragments.push_back ( + repository::fragment_type { + move (nm), lazy_shared_ptr<repository_fragment> (db, move (rf))}); + } + + // Save the changes to the repository object. + // + db.update (r); + + // Reset the shallow flag if the set of complements and/or prerequisites + // has changed. + // + // Note that weak pointers are generally incomparable (as can point to + // expired objects), and thus we can't compare the prerequisite sets + // directly. + // + if (shallow) + shallow = new_complements == old_complements && + equal (new_prerequisites.begin (), new_prerequisites.end (), + old_prerequisites.begin (), old_prerequisites.end (), + [] (const lazy_weak_ptr<repository>& x, + const lazy_weak_ptr<repository>& y) + { + return x.object_id () == y.object_id (); + }); + + // Fetch prerequisites and complements, unless this is a shallow fetch. + // + if (!shallow) + { + // Register old complements and prerequisites for potential removal + // unless they are fetched. + // + auto rm = [&fetched_repositories, &removed_repositories] ( + const lazy_shared_ptr<repository>& rp) + { + shared_ptr<repository> r (rp.load ()); + if (fetched_repositories.find (r) == fetched_repositories.end ()) + removed_repositories.insert (move (r)); + }; + + for (const lazy_shared_ptr<repository>& cr: old_complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + rm (cr); + } + + for (const lazy_weak_ptr<repository>& pr: old_prerequisites) + rm (lazy_shared_ptr<repository> (pr)); + + const string& rn (rl.canonical_name ()); + + // Fetch complements and prerequisites. + // + for (const auto& cr: new_complements) + { + if (cr.object_id () != "") + rep_fetch (co, + conf, + t, + cr.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + false /* shallow */, + "complements " + rn); + } + + for (const auto& pr: new_prerequisites) + rep_fetch (co, + conf, + t, + pr.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + false /* shallow */, + "prerequisite of " + rn); + } } static void @@ -806,10 +951,12 @@ namespace bpkg tracer_guard tg (db, trace); // As a fist step we fetch repositories recursively building the list of - // the former prerequisites and complements to be considered for removal. + // the former repository prerequisites, complements and fragments to be + // considered for removal. // // We delay the actual removal until we fetch all the required repositories // as a dependency dropped by one repository can appear for another one. + // The same is true about repository fragments. // try { @@ -819,26 +966,42 @@ namespace bpkg // filesystem_state_changed = false; - repositories fetched; - repositories removed; + repositories fetched_repositories; + repositories removed_repositories; + repository_fragments removed_fragments; // Fetch the requested repositories, recursively. // for (const lazy_shared_ptr<repository>& r: repos) - rep_fetch (o, conf, t, r.load (), fetched, removed, shallow, reason); + rep_fetch (o, + conf, + t, + r.load (), + fetched_repositories, + removed_repositories, + removed_fragments, + shallow, + reason); // Remove dangling repositories. // - for (const shared_ptr<repository>& r: removed) + for (const shared_ptr<repository>& r: removed_repositories) rep_remove (conf, t, r); + // Remove dangling repository fragments. + // + for (const shared_ptr<repository_fragment>& rf: removed_fragments) + rep_remove_fragment (conf, t, rf); + // Finally, make sure that the external packages are available from a // single directory-based repository. // // Sort the packages by name and version. This way the external packages - // with the same upstream version and revision will be adjacent. + // with the same upstream version and revision will be adjacent. Note + // that here we rely on the fact that the directory-based repositories + // have a single fragment. // - using query = query<package_repository>; + using query = query<package_repository_fragment>; const auto& qv (query::package::id.version); query q ("ORDER BY" + query::package::id.name + "," + @@ -849,18 +1012,18 @@ namespace bpkg qv.iteration); available_package_id ap; - shared_ptr<repository> rp; + shared_ptr<repository_fragment> rf; - for (const auto& pr: db.query<package_repository> (q)) + for (const auto& prf: db.query<package_repository_fragment> (q)) { - const shared_ptr<repository>& r (pr.repository); - if (!r->location.directory_based ()) + const shared_ptr<repository_fragment>& f (prf.repository_fragment); + if (!f->location.directory_based ()) continue; // Fail if the external package is of the same upstream version and // revision as the previous one. // - const available_package_id& id (pr.package_id); + const available_package_id& id (prf.package_id); if (id.name == ap.name && compare_version_eq (id.version, ap.version, true, false)) @@ -871,12 +1034,12 @@ namespace bpkg fail << "external package " << id.name << '/' << version (v.epoch, v.upstream, v.release, v.revision, 0) << " is available from two repositories" << - info << "repository " << rp->location << - info << "repository " << r->location; + info << "repository " << rf->location << + info << "repository " << f->location; } ap = id; - rp = r; + rf = f; } } catch (const failed&) @@ -913,8 +1076,11 @@ namespace bpkg transaction t (db); - shared_ptr<repository> root (db.load<repository> ("")); - repository::complements_type& ua (root->complements); // User-added repos. + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); + + // User-added repos. + // + repository_fragment::complements_type& ua (root->complements); for (const repository_location& rl: rls) { @@ -950,8 +1116,11 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr<repository> root (db.load<repository> ("")); - repository::complements_type& ua (root->complements); // User-added repos. + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); + + // User-added repos. + // + repository_fragment::complements_type& ua (root->complements); optional<string> reason; diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 89f4f2a..7e7777f 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -28,20 +28,23 @@ namespace bpkg struct rep_fetch_data { - using repository = repository_manifest; - - struct package + struct fragment { - package_manifest manifest; - string repository_fragment; // See package_location::fragment. + // Empty for fragment-less repositories. + // + string id; + string friendly_name; // User-friendly fragment name (e.g, tag, etc). + + vector<repository_manifest> repositories; + vector<package_manifest> packages; }; - std::vector<repository> repositories; - std::vector<package> packages; + vector<fragment> fragments; - // For base repo (can be NULL). + // For base pkg repo (can be nullopt/NULL). // - shared_ptr<const bpkg::certificate> certificate; + optional<string> cert_pem; + shared_ptr<const bpkg::certificate> certificate; // Authenticated. }; rep_fetch_data diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx index 0e7f911..7dd4a28 100644 --- a/bpkg/rep-info.cxx +++ b/bpkg/rep-info.cxx @@ -4,7 +4,8 @@ #include <bpkg/rep-info.hxx> -#include <iostream> // cout +#include <iostream> // cout +#include <algorithm> // find_if() #include <libbutl/sha256.mxx> // sha256_to_fingerprint() #include <libbutl/manifest-serializer.mxx> @@ -72,7 +73,7 @@ namespace bpkg if (all || cert_info) { shared_ptr<const certificate>& cert (rfd.certificate); - const optional<string>& cert_pem (rfd.repositories.back ().certificate); + const optional<string>& cert_pem (rfd.cert_pem); if (cert_pem) { @@ -145,23 +146,89 @@ namespace bpkg { if (o.manifest ()) { + // Merge repository manifest lists, adding the fragment value to + // prerequisite/complement repository manifests, and picking the + // latest base repository manifest. + // + vector<repository_manifest> rms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (repository_manifest& rm: fr.repositories) + { + if (rm.effective_role () != repository_role::base) + { + if (!fr.id.empty ()) + rm.fragment = fr.id; + + rms.push_back (move (rm)); + } + } + } + + // Append the latest base repository manifest. + // + // Note that there must be at least one fragment with at least a + // base repository being present. + // + assert (!rfd.fragments.empty () && + !rfd.fragments.back ().repositories.empty ()); + + rms.push_back (move (rfd.fragments.back ().repositories.back ())); + // Note: serializing without any extra repository_manifests info. // manifest_serializer s (cout, "STDOUT"); - for (const repository_manifest& rm: rfd.repositories) + + for (const repository_manifest& rm: rms) rm.serialize (s); + s.next ("", ""); // End of stream. } else { - for (const repository_manifest& rm: rfd.repositories) + // Merge complements/prerequisites from all fragments "upgrading" + // prerequisites to complements and preferring locations from the + // latest fragments. + // + vector<repository_manifest> rms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) { - repository_role rr (rm.effective_role ()); + for (repository_manifest& rm: fr.repositories) + { + if (rm.effective_role () == repository_role::base) + continue; + + repository_location l (rm.location, rl); // Complete. + + auto i (find_if (rms.begin (), rms.end (), + [&l] (const repository_manifest& i) + { + return i.location.canonical_name () == + l.canonical_name (); + })); + + if (i == rms.end ()) + { + rm.location = move (l); + rms.push_back (move (rm)); + } + else + { + if (rm.effective_role () == repository_role::complement) + i->role = rm.effective_role (); - if (rr == repository_role::base) - continue; // Entry for this repository. + i->location = move (l); + } + } + } + + for (const repository_manifest& rm: rms) + { + repository_role rr (rm.effective_role ()); - repository_location l (rm.location, rl); // Complete. + const repository_location& l (rm.location); const string& n (l.canonical_name ()); switch (rr) @@ -185,21 +252,56 @@ namespace bpkg { if (o.manifest ()) { + // Merge package manifest lists, adding the fragment. + // + vector<package_manifest> pms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (package_manifest& pm: fr.packages) + { + if (!fr.id.empty ()) + pm.fragment = fr.id; + + pms.push_back (move (pm)); + } + } + // Note: serializing without any extra package_manifests info. // manifest_serializer s (cout, "STDOUT"); - for (const rep_fetch_data::package& p: rfd.packages) - p.manifest.serialize (s); + for (const package_manifest& pm: pms) + pm.serialize (s); s.next ("", ""); // End of stream. } else { + // Merge packages from all the fragments. + // + vector<package_manifest> pms; + + for (rep_fetch_data::fragment& fr: rfd.fragments) + { + for (package_manifest& pm: fr.packages) + { + auto i (find_if (pms.begin (), pms.end (), + [&pm] (const package_manifest& i) + { + return i.name == pm.name && + i.version == pm.version; + })); + + if (i == pms.end ()) + pms.push_back (move (pm)); + } + } + // Separate package list from the general repository info. // cout << endl; - for (const rep_fetch_data::package& p: rfd.packages) - cout << p.manifest.name << "/" << p.manifest.version << endl; + for (const package_manifest& pm: pms) + cout << pm.name << "/" << pm.version << endl; } } } diff --git a/bpkg/rep-list.cxx b/bpkg/rep-list.cxx index 8d308dd..f1e251c 100644 --- a/bpkg/rep-list.cxx +++ b/bpkg/rep-list.cxx @@ -20,7 +20,7 @@ namespace bpkg // // Each line has the following form: // - // [(complement|prerequisite) ]<name> <location> + // [(complement|prerequisite) ]<name> <location>[ (<fragment>)] // // and is indented with 2 additional spaces for each recursion level. // @@ -44,34 +44,42 @@ namespace bpkg indent += " "; - if (o.complements ()) + auto print_repo = [&o, &indent, &chain] ( + const shared_ptr<repository>& r, + const char* role, + const repository::fragment_type& fr) { - for (const lazy_shared_ptr<repository>& rp: r->complements) - { - // Skip the root complement (see rep_fetch() for details). - // - if (rp.object_id () == "") - continue; + cout << indent << role << ' ' << r->name << ' ' << r->location; - shared_ptr<repository> r (rp.load ()); + if (!fr.friendly_name.empty ()) + cout << " (" << fr.friendly_name << ")"; - cout << indent << "complement " - << r->location.canonical_name () << " " << r->location << endl; + cout << endl; - print_dependencies (o, r, indent, chain); - } - } + print_dependencies (o, r, indent, chain); + }; - if (o.prerequisites ()) + for (const repository::fragment_type& rfr: r->fragments) { - for (const lazy_weak_ptr<repository>& rp: r->prerequisites) - { - shared_ptr<repository> r (rp.load ()); + shared_ptr<repository_fragment> fr (rfr.fragment.load ()); - cout << indent << "prerequisite " - << r->location.canonical_name () << " " << r->location << endl; + if (o.complements ()) + { + for (const lazy_shared_ptr<repository>& rp: fr->complements) + { + // Skip the root complement (see rep_fetch() for details). + // + if (rp.object_id () == "") + continue; + + print_repo (rp.load (), "complement", rfr); + } + } - print_dependencies (o, r, indent, chain); + if (o.prerequisites ()) + { + for (const lazy_weak_ptr<repository>& rp: fr->prerequisites) + print_repo (rp.load (), "prerequisite", rfr); } } @@ -104,7 +112,7 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr<repository> root (db.load<repository> ("")); + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); for (const lazy_shared_ptr<repository>& rp: root->complements) { diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx index fafe5a8..056883d 100644 --- a/bpkg/rep-remove.cxx +++ b/bpkg/rep-remove.cxx @@ -44,21 +44,45 @@ namespace bpkg if (!traversed.insert (r).second) // We have already been here. return false; - for (const auto& rc: db.query<repository_complement_dependent> ( + // Iterate over repository fragments that depend on this repository as a + // complement. + // + for (const auto& rf: db.query<repository_complement_dependent> ( query<repository_complement_dependent>::complement::name == nm)) { - const shared_ptr<repository>& r (rc); - if (r->name.empty () /* Root? */ || reachable (db, r, traversed)) + const shared_ptr<repository_fragment>& f (rf); + + if (f->name.empty ()) // Root? return true; + + // Iterate over repositories that contain this repository fragment. + // + for (const auto& fr: db.query<fragment_repository> ( + query<fragment_repository>::repository_fragment::name == f->name)) + { + if (reachable (db, fr, traversed)) + return true; + } } - for (const auto& rd: db.query<repository_prerequisite_dependent> ( + // Iterate over repository fragments that depend on this repository as a + // prerequisite. + // + for (const auto& rf: db.query<repository_prerequisite_dependent> ( query<repository_prerequisite_dependent>::prerequisite::name == nm)) { - // Note that the root repository has no prerequisites. + // Note that the root repository fragment has no prerequisites. // - if (reachable (db, rd, traversed)) - return true; + const shared_ptr<repository_fragment>& f (rf); + + // Iterate over repositories that contain this repository fragment. + // + for (const auto& fr: db.query<fragment_repository> ( + query<fragment_repository>::repository_fragment::name == f->name)) + { + if (reachable (db, fr, traversed)) + return true; + } } return false; @@ -72,22 +96,28 @@ namespace bpkg } void - rep_remove_package_locations (transaction& t, const string& name) + rep_remove_package_locations (transaction& t, const string& fragment_name) { + tracer trace ("rep_remove_package_locations"); + database& db (t.database ()); + tracer_guard tg (db, trace); + + using query = query<repository_fragment_package>; - for (const auto& rp: db.query<repository_package> ( - query<repository_package>::repository::name == name)) + for (const auto& rp: db.query<repository_fragment_package> ( + query::repository_fragment::name == fragment_name)) { const shared_ptr<available_package>& p (rp); vector<package_location>& ls (p->locations); - for (auto i (ls.cbegin ()); i != ls.cend (); ) + for (auto i (ls.cbegin ()); i != ls.cend (); ++i) { - if (i->repository.object_id () == name) - i = ls.erase (i); - else - ++i; + if (i->repository_fragment.object_id () == fragment_name) + { + ls.erase (i); + break; + } } if (ls.empty ()) @@ -117,17 +147,29 @@ namespace bpkg transaction& t, const shared_ptr<repository>& r) { - const string& nm (r->name); - assert (!nm.empty ()); // Can't be the root repository. + assert (!r->name.empty ()); // Can't be the root repository. + + tracer trace ("rep_remove"); database& db (t.database ()); + tracer_guard tg (db, trace); if (reachable (db, r)) return; - rep_remove_package_locations (t, nm); + // Note that it is essential to erase the repository object from the + // database prior to the repository fragments it contains as they must be + // un-referenced first. + // + db.erase (r); - // Cleanup the repository state if present. + // Remove dangling repository fragments. + // + for (const repository::fragment_type& fr: r->fragments) + rep_remove_fragment (c, t, fr.fragment.load ()); + + // Cleanup the repository state if present and there are no more + // repositories referring this state. // // Note that this step is irreversible on failure. If something goes wrong // we will end up with a state-less fetched repository and the @@ -145,14 +187,60 @@ namespace bpkg dir_path sd (c / repos_dir / d); if (exists (sd)) - rmdir (sd); + { + // There is no way to get the list of repositories that share this + // state other than traversing all repositories of this type. + // + bool rm (true); + + using query = query<repository>; + + for (shared_ptr<repository> r: + pointer_result ( + db.query<repository> ( + query::name != "" && + query::location.type == to_string (r->location.type ())))) + { + if (repository_state (r->location) == d) + { + rm = false; + break; + } + } + + if (rm) + rmdir (sd); + } } + } - // Note that it is essential to erase the repository object from the - // database prior to its complements and prerequisites removal as they - // must be un-referenced first. + void + rep_remove_fragment (const dir_path& c, + transaction& t, + const shared_ptr<repository_fragment>& rf) + { + tracer trace ("rep_remove_fragment"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Bail out if the repository fragment is still used. // - db.erase (r); + using query = query<fragment_repository_count>; + + if (db.query_value<fragment_repository_count> ( + "fragment=" + query::_val (rf->name)) != 0) + return; + + // Remove the repository fragment from locations of the available packages + // it contains. Note that this must be done before the repository fragment + // removal. + // + rep_remove_package_locations (t, rf->name); + + // Remove the repository fragment. + // + db.erase (rf); // Remove dangling complements and prerequisites. // @@ -167,7 +255,7 @@ namespace bpkg rep_remove (c, t, r); }; - for (const lazy_shared_ptr<repository>& cr: r->complements) + for (const lazy_shared_ptr<repository>& cr: rf->complements) { // Remove the complement unless it is the root repository (see // rep_fetch() for details). @@ -176,8 +264,15 @@ namespace bpkg remove (cr); } - for (const lazy_weak_ptr<repository>& pr: r->prerequisites) + for (const lazy_weak_ptr<repository>& pr: rf->prerequisites) remove (lazy_shared_ptr<repository> (pr)); + + // If there are no repositories stayed in the database then no repository + // fragments nor packages should stay either. + // + assert (db.query_value<repository_count> () != 0 || + (db.query_value<repository_fragment_count> () == 0 && + db.query_value<available_package_count> () == 0)); } void @@ -187,12 +282,13 @@ namespace bpkg bool quiet) { tracer trace ("rep_remove_clean"); + tracer_guard tg (db, trace); assert (!transaction::has_current ()); - // Clean repositories and available packages. At the end only repositories - // that were explicitly added by the user and the special root repository - // should remain. + // Clean repositories, repository fragments and available packages. At the + // end only repositories that were explicitly added by the user and the + // special root repository should remain. // { // Note that we don't rely on being in session nor create one. @@ -201,19 +297,19 @@ namespace bpkg db.erase_query<available_package> (); - shared_ptr<repository> root (db.load<repository> ("")); - repository::complements_type& ua (root->complements); + db.erase_query<repository_fragment> ( + query<repository_fragment>::name != ""); + + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); + repository_fragment::complements_type& ua (root->complements); for (shared_ptr<repository> r: pointer_result (db.query<repository> ())) { if (r->name == "") - { - l5 ([&]{trace << "skipping root";}); - } + l5 ([&]{trace << "skipping root repository";}); else if (ua.find (lazy_shared_ptr<repository> (db, r)) != ua.end ()) { - r->complements.clear (); - r->prerequisites.clear (); + r->fragments.clear (); db.update (r); if (verb >= (quiet ? 2 : 1) && !o.no_result ()) @@ -298,8 +394,8 @@ namespace bpkg transaction t (db); session s; // Repository dependencies can have cycles. - shared_ptr<repository> root (db.load<repository> ("")); - repository::complements_type& ua (root->complements); + shared_ptr<repository_fragment> root (db.load<repository_fragment> ("")); + repository_fragment::complements_type& ua (root->complements); if (o.all ()) { @@ -392,11 +488,12 @@ namespace bpkg // assert (!o.all () || ua.empty ()); - // If we removed all the user-added repositories then no repositories nor - // packages should stay in the database. + // If we removed all the user-added repositories then no repositories, + // repository fragments or packages should stay in the database. // assert (!ua.empty () || (db.query_value<repository_count> () == 0 && + db.query_value<repository_fragment_count> () == 0 && db.query_value<available_package_count> () == 0)); t.commit (); diff --git a/bpkg/rep-remove.hxx b/bpkg/rep-remove.hxx index 68d8a36..477f308 100644 --- a/bpkg/rep-remove.hxx +++ b/bpkg/rep-remove.hxx @@ -17,13 +17,22 @@ namespace bpkg rep_remove (const rep_remove_options&, cli::scanner& args); // Remove a repository if it is not reachable from the root (and thus is not - // required by any user-added repository). + // required by any user-added repository), also removing its unused + // repository fragments. // void rep_remove (const dir_path& conf, transaction&, const shared_ptr<repository>&); + // Remove a repository fragment if it is not referenced by any repository, + // also removing its unreachable complements and prerequisites. + // + void + rep_remove_fragment (const dir_path& conf, + transaction&, + const shared_ptr<repository_fragment>&); + // Bring the configuration to the clean state as if repositories were added // but never fetched. Leave selected packages intact. // @@ -45,11 +54,11 @@ namespace bpkg database&, bool quiet = true); - // Remove a repository from locations of the available packages it - // contains. Remove packages that come from only this repository. + // Remove a repository fragment from locations of the available packages it + // contains. Remove packages that come from only this repository fragment. // void - rep_remove_package_locations (transaction&, const string& repository_name); + rep_remove_package_locations (transaction&, const string& fragment_name); } #endif // BPKG_REP_REMOVE_HXX diff --git a/doc/manual.cli b/doc/manual.cli index 99312e8..447586e 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -955,9 +955,12 @@ following synopsis: \ location: <path> +[fragment]: <string> \ The detailed description of each value follows in the subsequent sections. +The \c{fragment} value can only be present in a merged \c{packages.manifest} +file for a multi-fragment repository. As an example, if our repository contained the \c{src/} subdirectory that in turn contained the \c{libfoo} and \c{foo} packages, then the corresponding @@ -980,6 +983,15 @@ The path to the package directory relative to the repository root. It should be in the POSIX representation. +\h2#manifest-package-list-dir-fragment|\c{fragment}| + +\ +[fragment]: <string> +\ + +The repository fragment id this package belongs to. + + \h#manifest-repository|Repository Manifest| The repository manifest (only used as part of the repository manifest list @@ -996,6 +1008,7 @@ each value in subsequent sections. [summary]: <text> [description]: <text> [certificate]: <pem> +[fragment]: <string> \ See also the Repository Chaining documentation for further information @@ TODO. @@ -1172,6 +1185,15 @@ with the corresponding private key and the signature saved in the Manifest} for details. +\h2#manifest-repository-fragment|\c{fragment}| + +\ +[fragment]: <string> +\ + +The repository fragment id this repository belongs to. + + \h#manifest-repository-list|Repository List Manifest| @@ TODO See the Repository Chaining document for more information on the @@ -1182,9 +1204,12 @@ repository root directory) describes the repository. First comes a (potentially empty) sequence of repository manifests that describe the prerequisite and complement repositories. After this sequence must come the manifest for the base repository, that is, the repository that this manifest -list is describing. The location value in this last manifest must be -omitted. \N{Since we got hold of the manifest list, then we presumably -already know the location of the base repository.} +list is describing. The \c{location} value in this last manifest must be +omitted. \N{Since we got hold of the manifest list, then we presumably already +know the location of the base repository.} + +The \c{fragment} values can only be present in a merged +\c{repositories.manifest} file for a multi-fragment repository. As an example, a repository manifest list for the \c{math/testing} repository could look like this: diff --git a/tests/common/git/init b/tests/common/git/init index e80ebf4..1c15bd6 100755 --- a/tests/common/git/init +++ b/tests/common/git/init @@ -60,6 +60,7 @@ rm -f -r style.git/basic rm -f -r style-basic.git/.git rm -f style-basic.git/README rm -f style-basic.git/INSTALL +rm -f style-basic.git/repositories.manifest # Create master branch for style-basic.git. # @@ -69,10 +70,15 @@ git -C style-basic.git commit -am 'Create' # Create stable branch for style-basic. # +sleep 1 # Make sure that master commits are older than stable commits. git -C style-basic.git branch stable git -C style-basic.git checkout stable touch style-basic.git/README -git -C style-basic.git add README +cat <<EOF >style-basic.git/repositories.manifest +: 1 +email: user@example.com +EOF +git -C style-basic.git add README repositories.manifest git -C style-basic.git commit -am 'README' # Create master branch for style.git, adding style-basic.git as a submodule. diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar Binary files differindex 8be7c94..58b58bd 100644 --- a/tests/common/git/state0/libbar.tar +++ b/tests/common/git/state0/libbar.tar diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar Binary files differindex 31b5519..7aa0a9d 100644 --- a/tests/common/git/state0/libfoo.tar +++ b/tests/common/git/state0/libfoo.tar diff --git a/tests/common/git/state0/libfox.tar b/tests/common/git/state0/libfox.tar Binary files differindex b4037c5..81d6c85 100644 --- a/tests/common/git/state0/libfox.tar +++ b/tests/common/git/state0/libfox.tar diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar Binary files differindex 80731dd..481705c 100644 --- a/tests/common/git/state0/style-basic.tar +++ b/tests/common/git/state0/style-basic.tar diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar Binary files differindex 1a88771..127f82f 100644 --- a/tests/common/git/state0/style.tar +++ b/tests/common/git/state0/style.tar diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar Binary files differindex 298712c..572c56b 100644 --- a/tests/common/git/state1/libbaz.tar +++ b/tests/common/git/state1/libbaz.tar diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar Binary files differindex 2f2e3b4..4512930 100644 --- a/tests/common/git/state1/libfoo.tar +++ b/tests/common/git/state1/libfoo.tar diff --git a/tests/common/git/state1/libfox.tar b/tests/common/git/state1/libfox.tar Binary files differindex d296973..4046ded 100644 --- a/tests/common/git/state1/libfox.tar +++ b/tests/common/git/state1/libfox.tar diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar Binary files differindex 7391b66..604e8c8 100644 --- a/tests/common/git/state1/style-basic.tar +++ b/tests/common/git/state1/style-basic.tar diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar Binary files differindex 4be977b..dab3fd7 100644 --- a/tests/common/git/state1/style.tar +++ b/tests/common/git/state1/style.tar diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test index 5028bf9..47be179 100644 --- a/tests/rep-fetch.test +++ b/tests/rep-fetch.test @@ -39,7 +39,7 @@ # | | `-- root.build # | `-- * # | -# |-- circle +# |-- cycle # | |-- extra -> stable (prerequisite) # | | |-- libbar-1.1.0+1.tar.gz # | | `-- repositories.manifest @@ -50,7 +50,7 @@ # | |-- libfoo-1.0.0.tar.gz # | `-- repositories.manifest # | -# |-- no-circle +# |-- no-cycle # | |-- extra # | | |-- libbar-1.1.0+1.tar.gz # | | `-- repositories.manifest diff --git a/tests/rep-info.test b/tests/rep-info.test index cdc9e70..e0c7688 100644 --- a/tests/rep-info.test +++ b/tests/rep-info.test @@ -172,4 +172,15 @@ else libbar/1.0.0 libmbar/1.0.0 EOO + + : repository-manifests + : + : Here we test that the base repository manifest that comes from the stable + : branch is printed because the stable head commit is newer that of the + : master branch. + : + $* -r -m "$rep/style-basic.git#stable,master" >>~%EOO% + : 1 + email: user@example.com + EOO } diff --git a/tests/rep-list.test b/tests/rep-list.test index 970379a..c1f0841 100644 --- a/tests/rep-list.test +++ b/tests/rep-list.test @@ -142,6 +142,6 @@ else $* --complements --prerequisites >>~%EOO% %git:.+libbar#master .+libbar\.git#master% - % prerequisite git:.+style-basic#stable .+style-basic\.git#stable% + % prerequisite git:.+style-basic#stable .+style-basic\.git#stable \(heads/master\)% EOO } |