diff options
Diffstat (limited to 'bpkg/package.cxx')
| -rw-r--r-- | bpkg/package.cxx | 699 |
1 files changed, 416 insertions, 283 deletions
diff --git a/bpkg/package.cxx b/bpkg/package.cxx index 3532f3d..05dbc0d 100644 --- a/bpkg/package.cxx +++ b/bpkg/package.cxx @@ -6,7 +6,10 @@ #include <bpkg/database.hxx> #include <bpkg/checksum.hxx> +#include <bpkg/rep-mask.hxx> +#include <bpkg/pkg-verify.hxx> #include <bpkg/diagnostics.hxx> +#include <bpkg/satisfaction.hxx> #include <bpkg/manifest-utility.hxx> using namespace std; @@ -15,322 +18,191 @@ namespace bpkg { const version wildcard_version (0, "0", nullopt, nullopt, 0); - // available_package_id + // configuration // - bool - operator< (const available_package_id& x, const available_package_id& y) + configuration:: + configuration (optional<string> n, string t, optional<uuid_type> uid) + : id (0), + name (move (n)), + type (move (t)), + expl (false) { - int r (x.name.compare (y.name)); - return r != 0 ? r < 0 : x.version < y.version; + try + { + uuid = uid ? *uid : uuid_type::generate (); + } + catch (const system_error& e) + { + fail << "unable to generate configuration uuid: " << e; + } } - // available_package - // - odb::result<available_package> - query_available (database& db, - const package_name& name, - const optional<version_constraint>& c, - bool order) + dir_path configuration:: + effective_path (const dir_path& d) const { - using query = query<available_package>; - - query q (query::id.name == name); - const auto& vm (query::id.version); - - // If there is a constraint, then translate it to the query. Otherwise, - // get the latest version or stub versions if present. - // - if (c) + if (path.relative ()) { - assert (c->complete ()); - - // If the revision is not explicitly specified, then compare ignoring the - // revision. The idea is that when the user runs 'bpkg build libfoo/1' - // and there is 1+1 available, it should just work. The user shouldn't - // have to spell the revision explicitly. Similarly, when we have - // 'depends: libfoo == 1', then it would be strange if 1+1 did not - // satisfy this constraint. The same for libfoo <= 1 -- 1+1 should - // satisfy. - // - // Note that we always compare ignoring the iteration, as it can not be - // specified in the manifest/command line. This way the latest iteration - // will always be picked up. - // - query qs (compare_version_eq (vm, - canonical_version (wildcard_version), - false /* revision */, - false /* iteration */)); - - if (c->min_version && - c->max_version && - *c->min_version == *c->max_version) - { - const version& v (*c->min_version); - - q = q && - (compare_version_eq (vm, - canonical_version (v), - v.revision.has_value (), - false /* iteration */) || - qs); - } - else - { - query qr (true); - - if (c->min_version) - { - const version& v (*c->min_version); - canonical_version cv (v); - bool rv (v.revision); - - if (c->min_open) - qr = compare_version_gt (vm, cv, rv, false /* iteration */); - else - qr = compare_version_ge (vm, cv, rv, false /* iteration */); - } + dir_path r (d / path); - if (c->max_version) - { - const version& v (*c->max_version); - canonical_version cv (v); - bool rv (v.revision); - - if (c->max_open) - qr = qr && compare_version_lt (vm, cv, rv, false /* iteration */); - else - qr = qr && compare_version_le (vm, cv, rv, false /* iteration */); - } + string what ("linked with " + d.representation () + " configuration " + + (name ? *name : to_string (*id))); - q = q && (qr || qs); - } + normalize (r, what.c_str ()); + return r; } - - if (order) - q += order_by_version_desc (vm); - - return db.query<available_package> (q); + else + return path; } - // Check if the package is available from the specified repository fragment, - // its prerequisite repositories, or one of their complements, recursively. - // Return the first repository fragment that contains the package or NULL if - // none are. + // package_key // - // Note that we can end up with a repository dependency cycle since the - // root repository can be the default complement for dir and git - // repositories (see rep_fetch() implementation for details). Thus we need - // to make sure that the repository fragment is not in the dependency chain - // yet. - // - using repository_fragments = - vector<reference_wrapper<const shared_ptr<repository_fragment>>>; - - static shared_ptr<repository_fragment> - find (const shared_ptr<repository_fragment>& rf, - const shared_ptr<available_package>& ap, - repository_fragments& chain, - bool prereq) + string package_key:: + string () const { - // Prerequisites are not searched through recursively. - // - assert (!prereq || chain.empty ()); - - auto i (find_if (chain.begin (), chain.end (), - [&rf] (const shared_ptr<repository_fragment>& i) -> bool - { - return i == rf; - })); - - if (i != chain.end ()) - return nullptr; - - chain.emplace_back (rf); + const std::string& s (db.get ().string); + return !s.empty () ? name.string () + ' ' + s : name.string (); + } - unique_ptr<repository_fragments, void (*)(repository_fragments*)> deleter ( - &chain, [] (repository_fragments* rf) {rf->pop_back ();}); + bool package_key:: + operator< (const package_key& v) const + { + int r (name.compare (v.name)); + return r != 0 ? (r < 0) : (db < v.db); + } - const auto& cs (rf->complements); - const auto& ps (rf->prerequisites); + // package_version_key + // + string package_version_key:: + string (bool ignore_version) const + { + std::string r (name.string ()); - for (const package_location& pl: ap->locations) + if (version && !version->empty () && !ignore_version) { - const lazy_shared_ptr<repository_fragment>& lrf (pl.repository_fragment); - - // First check the repository itself. - // - if (lrf.object_id () == rf->name) - return rf; - - // Then check all the complements and prerequisites repository fragments - // without loading them. Though, we still need to load complement and - // prerequisite repositories. - // - auto pr = [&lrf] (const repository::fragment_type& i) - { - return i.fragment == lrf; - }; - - for (const lazy_weak_ptr<repository>& r: cs) - { - const auto& frs (r.load ()->fragments); - - if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) - return lrf.load (); - } - - if (prereq) - { - for (const lazy_weak_ptr<repository>& r: ps) - { - const auto& frs (r.load ()->fragments); - - if (find_if (frs.begin (), frs.end (), pr) != frs.end ()) - return lrf.load (); - } - } + r += '/'; + r += version->string (); + } - // Finally, load the complements and prerequisites and check them - // recursively. - // - for (const lazy_weak_ptr<repository>& cr: cs) - { - for (const auto& fr: cr.load ()->fragments) - { - // Should we consider prerequisites of our complements as our - // prerequisites? I'd say not. - // - if (shared_ptr<repository_fragment> r = - find (fr.fragment.load (), ap, chain, false)) - return r; - } - } + const std::string& d (db.get ().string); - if (prereq) - { - for (const lazy_weak_ptr<repository>& pr: ps) - { - for (const auto& fr: pr.load ()->fragments) - { - if (shared_ptr<repository_fragment> r = - find (fr.fragment.load (), ap, chain, false)) - return r; - } - } - } + if (!d.empty ()) + { + r += ' '; + r += d; } - return nullptr; + return r; } - shared_ptr<repository_fragment> - filter (const shared_ptr<repository_fragment>& r, - const shared_ptr<available_package>& ap, - bool prereq) + bool package_version_key:: + operator< (const package_version_key& v) const { - repository_fragments chain; - return find (r, ap, chain, prereq); + // NOTE: remember to update cmdline_adjustments::tried_earlier() if + // changing anything here. + // + if (int r = name.compare (v.name)) + return r < 0; + + return version != v.version ? (version < v.version) : (db < v.db); } - vector<shared_ptr<available_package>> - filter (const shared_ptr<repository_fragment>& r, - result<available_package>&& apr, - bool prereq) + // available_package + // + const version* available_package:: + system_version (database& db) const { - vector<shared_ptr<available_package>> aps; - - for (shared_ptr<available_package> ap: pointer_result (apr)) + if (!system_version_) { - if (filter (r, ap, prereq) != nullptr) - aps.push_back (move (ap)); + assert (db.system_repository); + + if (const system_package* sp = db.system_repository->find (id.name)) + { + // Only cache if it is authoritative. + // + if (sp->authoritative) + system_version_ = sp->version; + else + return &sp->version; + } } - return aps; + return system_version_ ? &*system_version_ : nullptr; } - pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> - filter_one (const shared_ptr<repository_fragment>& r, - result<available_package>&& apr, - bool prereq) + pair<const version*, bool> available_package:: + system_version_authoritative (database& db) const { - using result = pair<shared_ptr<available_package>, - shared_ptr<repository_fragment>>; + assert (db.system_repository); + + const system_package* sp (db.system_repository->find (id.name)); - for (shared_ptr<available_package> ap: pointer_result (apr)) + if (!system_version_) { - if (shared_ptr<repository_fragment> pr = filter (r, ap, prereq)) - return result (move (ap), move (pr)); + if (sp != nullptr) + { + // Only cache if it is authoritative. + // + if (sp->authoritative) + system_version_ = sp->version; + else + return make_pair (&sp->version, false); + } } - return result (); + return make_pair (system_version_ ? &*system_version_ : nullptr, + sp != nullptr ? sp->authoritative : false); } - vector<pair<shared_ptr<available_package>, shared_ptr<repository_fragment>>> - filter (const vector<shared_ptr<repository_fragment>>& rps, - odb::result<available_package>&& apr, - bool prereq) + void + check_any_available (const linked_databases& dbs, + transaction&, + const diag_record* drp) { - vector<pair<shared_ptr<available_package>, - shared_ptr<repository_fragment>>> aps; - - for (shared_ptr<available_package> ap: pointer_result (apr)) + bool rep (false); + bool pkg (false); + for (database& db: dbs) { - for (const shared_ptr<repository_fragment>& r: rps) + if (db.query_value<repository_count> () != 0) { - if (shared_ptr<repository_fragment> rf = filter (r, ap, prereq)) + rep = true; + + if (db.query_value<available_package_count> () != 0) { - aps.emplace_back (move (ap), move (rf)); + pkg = true; break; } } } - return aps; - } - - pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> - filter_one (const vector<shared_ptr<repository_fragment>>& rps, - odb::result<available_package>&& apr, - bool prereq) - { - using result = pair<shared_ptr<available_package>, - shared_ptr<repository_fragment>>; + if (pkg) + return; - for (shared_ptr<available_package> ap: pointer_result (apr)) - { - for (const shared_ptr<repository_fragment>& r: rps) - { - if (shared_ptr<repository_fragment> rf = filter (r, ap, prereq)) - return result (move (ap), move (rf)); - } - } + diag_record d; + const diag_record& dr (drp != nullptr ? *drp << info : d << fail); - return result (); - } - - void - check_any_available (const dir_path& c, - transaction& t, - const diag_record* dr) - { - database& db (t.database ()); + if (dbs.size () == 1) + dr << "configuration " << dbs[0].get ().config_orig << " has "; + else + dr << "specified configurations have "; - if (db.query_value<repository_count> () == 0) + if (!rep) { - diag_record d; - (dr != nullptr ? *dr << info : d << fail) - << "configuration " << c << " has no repositories" << + dr << "no repositories" << info << "use 'bpkg rep-add' to add a repository"; } - else if (db.query_value<available_package_count> () == 0) + else { - diag_record d; - (dr != nullptr ? *dr << info : d << fail) - << "configuration " << c << " has no available packages" << + dr << "no available packages" << info << "use 'bpkg rep-fetch' to fetch available packages list"; } } + void + check_any_available (database& db, transaction& t, const diag_record* dr) + { + return check_any_available (linked_databases ({db}), t, dr); + } + string package_string (const package_name& n, const version& v, bool system) { @@ -376,29 +248,157 @@ namespace bpkg // Quote the result as it contains the space character. // - return "'" + name.string () + ' ' + constraint->string () + "'"; + return '\'' + name.string () + ' ' + constraint->string () + '\''; } // selected_package // string selected_package:: - version_string () const + string (database& db) const { - return version != wildcard_version ? version.string () : "*"; + const std::string& s (db.string); + return !s.empty () ? string () + ' ' + s : string (); + } + + _selected_package_ref:: + _selected_package_ref (const lazy_shared_ptr<selected_package>& p) + : configuration (p.database ().uuid), + prerequisite (p.object_id ()) + { + } + + lazy_shared_ptr<selected_package> _selected_package_ref:: + to_ptr (odb::database& db) && + { + database& pdb (static_cast<database&> (db)); + + // Note that if this points to a different configuration, then it should + // already be pre-attached since it must be explicitly linked. + // + database& ddb (pdb.find_dependency_config (configuration)); + + // Make sure the prerequisite exists in the explicitly linked + // configuration, so that a subsequent load() call will not fail. This, + // for example, can happen in unlikely but possible situation when the + // implicitly linked configuration containing a dependent was temporarily + // renamed before its prerequisite was dropped. + // + // Note that the diagnostics lacks information about the dependent and its + // configuration. However, handling this situation at all the load() + // function call sites where this information is available, for example by + // catching the odb::object_not_persistent exception, feels a bit + // hairy. Given the situation is not common, let's keep it simple for now + // and see how it goes. + // + if (ddb != pdb && ddb.find<selected_package> (prerequisite) == nullptr) + fail << "unable to find prerequisite package " << prerequisite + << " in linked configuration " << ddb.config_orig; + + return lazy_shared_ptr<selected_package> (ddb, move (prerequisite)); + } + + string + to_string (config_source s) + { + switch (s) + { + case config_source::user: return "user"; + case config_source::dependent: return "dependent"; + case config_source::reflect: return "reflect"; + } + + return string (); // Should never reach. + } + + config_source + to_config_source (const string& s) + { + if (s == "user") return config_source::user; + else if (s == "dependent") return config_source::dependent; + else if (s == "reflect") return config_source::reflect; + else throw invalid_argument ("invalid config source '" + s + '\''); + } + + shared_ptr<available_package> + make_available (const common_options& options, + database& db, + const shared_ptr<selected_package>& sp) + { + assert (sp != nullptr && sp->state != package_state::broken); + + if (sp->system ()) + return make_shared<available_package> (sp->name, sp->version); + + // The package is in at least fetched state, which means we should + // be able to get its manifest. + // + // @@ PERF We should probably implement the available package caching not + // to parse the same manifests multiple times during all that build + // plan refinement iterations. What should be the cache key? Feels like + // it should be the archive/directory path. Note that the package + // manifests can potentially differ in different external package + // directories for the same version iteration. Testing showed 6% + // speedup on tests (debug/sanitized). + // + package_manifest m ( + sp->state == package_state::fetched + ? pkg_verify (options, + sp->effective_archive (db.config_orig), + true /* ignore_unknown */, + false /* ignore_toolchain */, + false /* expand_values */, + true /* load_buildfiles */) + : pkg_verify (options, + sp->effective_src_root (db.config_orig), + true /* ignore_unknown */, + false /* ignore_toolchain */, + true /* load_buildfiles */, + // Copy potentially fixed up version from selected package. + [&sp] (version& v) {v = sp->version;})); + + return make_shared<available_package> (move (m)); + } + + pair<shared_ptr<selected_package>, database*> + find_dependency (database& db, const package_name& pn, bool buildtime) + { + pair<shared_ptr<selected_package>, database*> r; + + for (database& ldb: db.dependency_configs (pn, buildtime)) + { + shared_ptr<selected_package> p (ldb.find<selected_package> (pn)); + + if (p != nullptr) + { + if (r.first == nullptr) + { + r.first = move (p); + r.second = &ldb; + } + else + { + fail << "package " << pn << " appears in multiple configurations" << + info << r.first->state << " in " << r.second->config_orig << + info << p->state << " in " << ldb.config_orig; + } + } + } + + return r; } optional<version> package_iteration (const common_options& o, - const dir_path& c, - transaction& t, + database& db, + transaction&, const dir_path& d, const package_name& n, const version& v, + const package_info* pi, bool check_external) { tracer trace ("package_iteration"); - database& db (t.database ()); tracer_guard tg (db, trace); if (check_external) @@ -416,7 +416,7 @@ namespace bpkg { const shared_ptr<repository_fragment>& rf (prf.repository_fragment); - if (rf->location.directory_based ()) + if (!rep_masked_fragment (db, rf) && rf->location.directory_based ()) fail << "external package " << n << '/' << v << " is already available from " << rf->location.canonical_name (); @@ -432,30 +432,54 @@ namespace bpkg false /* iteration */)) return nullopt; - string mc (sha256 (o, d / manifest_file)); + bool changed (!p->external ()); - // The selected package must not be "simulated" (see pkg-build for - // details). + // If the selected package is not external, then increment the iteration + // number to make the external package preferable. Note that for such + // packages the manifest/subprojects and buildfiles checksums are absent. // - assert (p->manifest_checksum); + if (!changed) + { + // The selected package must not be "simulated" (see pkg-build for + // details). + // + assert (p->manifest_checksum); - bool changed (mc != *p->manifest_checksum); + changed = (package_checksum (o, d, pi) != *p->manifest_checksum); - // If the manifest didn't changed but the selected package points to an - // external source directory, then we also check if the directory have - // moved. - // - if (!changed && p->external ()) - { - dir_path src_root (p->effective_src_root (c)); + // If the manifest hasn't changed and the package has buildfile clauses + // in the dependencies, then check if the buildfiles haven't changed + // either. + // + if (!changed && p->buildfiles_checksum) + { + // Always calculate the checksum over the buildfiles since the package + // is external. + // + changed = package_buildfiles_checksum ( + nullopt /* bootstrap_build */, + nullopt /* root_build */, + {} /* buildfiles */, + d) != *p->buildfiles_checksum; + } - // We need to complete and normalize the source directory as it may - // generally be completed against the configuration directory (unlikely - // but possible), that can be relative and/or not normalized. + // If the manifest hasn't changed but the selected package points to an + // external source directory, then we also check if the directory have + // moved. // - normalize (src_root, "package source"); + if (!changed) + { + dir_path src_root (p->effective_src_root (db.config)); + + // We need to complete and normalize the source directory as it may + // generally be completed against the configuration directory + // (unlikely but possible), that can be relative and/or not + // normalized. + // + normalize (src_root, "package source"); - changed = src_root != normalize (d, "package source"); + changed = src_root != normalize (d, "package source"); + } } return !changed @@ -492,7 +516,7 @@ namespace bpkg else if (s == "fetched") return package_state::fetched; else if (s == "unpacked") return package_state::unpacked; else if (s == "configured") return package_state::configured; - else throw invalid_argument ("invalid package state '" + s + "'"); + else throw invalid_argument ("invalid package state '" + s + '\''); } // substate @@ -514,7 +538,7 @@ namespace bpkg { if (s == "none") return package_substate::none; else if (s == "system") return package_substate::system; - else throw invalid_argument ("invalid package substate '" + s + "'"); + else throw invalid_argument ("invalid package substate '" + s + '\''); } // certificate @@ -532,4 +556,113 @@ namespace bpkg return os; } + + // package_dependent + // + odb::result<package_dependent> + query_dependents (database& db, + const package_name& dep, + database& dep_db) + { + // Prepare and cache this query since it's executed a lot. Note that we + // have to cache one per database. + // + using query = query<package_dependent>; + using prep_query = prepared_query<package_dependent>; + + struct params + { + string name; + string config; // Configuration UUID. + string query_name; + }; + + params* qp; + string qn (db.uuid.string () + "-package-dependent-query"); + prep_query pq (db.lookup_query<package_dependent> (qn.c_str (), qp)); + + if (!pq) + { + unique_ptr<params> p (qp = new params ()); + p->query_name = move (qn); + + query q ("prerequisite = " + query::_ref (p->name) + "AND" + + "configuration = " + query::_ref (p->config)); + + pq = db.prepare_query<package_dependent> (p->query_name.c_str (), q); + db.cache_query (pq, move (p)); + } + + qp->name = dep.string (); + qp->config = dep_db.uuid.string (); + + return pq.execute (); + } + + vector<package_dependent> + query_dependents_cache (database& db, + const package_name& dep, + database& dep_db) + { + vector<package_dependent> r; + for (package_dependent& pd: query_dependents (db, dep, dep_db)) + r.push_back (move (pd)); + return r; + } + + bool + toolchain_buildtime_dependency (const common_options& o, + const dependency_alternatives& das, + const package_name* pkg) + { + if (das.buildtime) + { + for (const dependency_alternative& da: das) + { + for (const dependency& d: da) + { + const package_name& dn (d.name); + + if (dn == "build2") + { + if (pkg != nullptr && d.constraint && !satisfy_build2 (o, d)) + { + fail << "unable to satisfy constraint (" << d << ") for " + << "package " << *pkg << + info << "available build2 version is " << build2_version; + } + + return true; + } + else if (dn == "bpkg") + { + if (pkg != nullptr && d.constraint && !satisfy_bpkg (o, d)) + { + fail << "unable to satisfy constraint (" << d << ") for " + << "package " << *pkg << + info << "available bpkg version is " << bpkg_version; + } + + return true; + } + } + } + } + + return false; + } + + bool + has_dependencies (const common_options& o, + const dependencies& deps, + const package_name* pkg) + { + for (const auto& das: deps) + { + if (!toolchain_buildtime_dependency (o, das, pkg)) + return true; + } + + return false; + } } |