diff options
Diffstat (limited to 'bpkg/pkg-build.cxx')
| -rw-r--r-- | bpkg/pkg-build.cxx | 12317 |
1 files changed, 3979 insertions, 8338 deletions
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index 64bb452..fac79c2 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -5,13 +5,9 @@ #include <map> #include <set> -#include <list> -#include <limits> // numeric_limits -#include <cstring> // strlen() +#include <cstring> // strlen() #include <sstream> -#include <iostream> // cout -#include <functional> // ref() -#include <forward_list> +#include <iostream> // cout #include <libbutl/standard-version.hxx> @@ -25,29 +21,33 @@ #include <bpkg/common-options.hxx> #include <bpkg/cfg-link.hxx> +#include <bpkg/rep-mask.hxx> #include <bpkg/pkg-purge.hxx> #include <bpkg/pkg-fetch.hxx> #include <bpkg/rep-fetch.hxx> -#include <bpkg/cfg-create.hxx> #include <bpkg/pkg-unpack.hxx> #include <bpkg/pkg-update.hxx> #include <bpkg/pkg-verify.hxx> #include <bpkg/pkg-checkout.hxx> #include <bpkg/pkg-configure.hxx> #include <bpkg/pkg-disfigure.hxx> +#include <bpkg/package-query.hxx> #include <bpkg/package-skeleton.hxx> + #include <bpkg/system-repository.hxx> -#include <bpkg/package-configuration.hxx> +#include <bpkg/system-package-manager.hxx> + +#include <bpkg/pkg-build-collect.hxx> using namespace std; using namespace butl; namespace bpkg { - // @@ Overall TODO: - // - // - Configuration vars (both passed and preserved) + // System package manager. Resolved lazily if and when needed. Present NULL + // value means no system package manager is available for this host. // + static optional<unique_ptr<system_package_manager>> sys_pkg_mgr; // Current configurations as specified with --directory|-d (or the current // working directory if none specified). @@ -67,47 +67,35 @@ namespace bpkg current_configs.end (); } - // Configurations to use as the repository information sources. - // - // The list contains the current configurations and configurations of the - // specified on the command line build-to-hold packages (ultimate - // dependents). - // - // For ultimate dependents we use configurations in which they are being - // built as a source of the repository information. For dependency packages - // we use configurations of their ultimate dependents. - // - static linked_databases repo_configs; - - // Return the ultimate dependent configurations for packages in this - // configuration. - // - static linked_databases - dependent_repo_configs (database& db) - { - linked_databases r; - for (database& ddb: db.dependent_configs ()) - { - if (find (repo_configs.begin (), repo_configs.end (), ddb) != - repo_configs.end ()) - r.push_back (ddb); - } - - return r; - } - // Retrieve the repository fragments for the specified package from its // ultimate dependent configurations and add them to the respective // configuration-associated fragment lists. // - using config_repo_fragments = - database_map<vector<shared_ptr<repository_fragment>>>; - + // If this package's repository fragment is a root fragment (package is + // fetched/unpacked using the existing archive/directory), then also add + // this repository fragment to the resulting list assuming that this + // package's dependencies can be resolved from this repository fragment or + // its complements (user-added repositories) as well. + // static void add_dependent_repo_fragments (database& db, - const available_package_id& id, + const shared_ptr<selected_package>& p, config_repo_fragments& r) { + available_package_id id (p->name, p->version); + + // Add a repository fragment to the specified list, suppressing duplicates. + // + auto add = [] (shared_ptr<repository_fragment>&& rf, + vector<shared_ptr<repository_fragment>>& rfs) + { + if (find (rfs.begin (), rfs.end (), rf) == rfs.end ()) + rfs.push_back (move (rf)); + }; + + if (p->repository_fragment.empty ()) // Root repository fragment? + add (db.find<repository_fragment> (empty_string), r[db]); + for (database& ddb: dependent_repo_configs (db)) { shared_ptr<available_package> dap (ddb.find<available_package> (id)); @@ -126,8392 +114,2814 @@ namespace bpkg for (const auto& pl: dap->locations) { - shared_ptr<repository_fragment> rf (pl.repository_fragment.load ()); + const lazy_shared_ptr<repository_fragment>& lrf ( + pl.repository_fragment); - if (find (rfs.begin (), rfs.end (), rf) == rfs.end ()) - rfs.push_back (move (rf)); + if (!rep_masked_fragment (lrf)) + add (lrf.load (), rfs); } + + // Erase the entry from the map if it contains no fragments, which may + // happen if all the available package repositories are masked. + // + if (rfs.empty ()) + r.erase (i); } } } - // Try to find an available stub package in the imaginary system repository. - // Such a repository contains stubs corresponding to the system packages - // specified by the user on the command line with version information - // (sys:libfoo/1.0, ?sys:libfoo/* but not ?sys:libfoo; the idea is that a - // real stub won't add any extra information to such a specification so we - // shouldn't insist on its presence). Semantically this imaginary repository - // complements all real repositories. + // Return a patch version constraint for the specified package version if it + // is a standard version (~ shortcut). Otherwise, if requested, issue a + // warning and return nullopt. // - static vector<shared_ptr<available_package>> imaginary_stubs; - - static shared_ptr<available_package> - find_imaginary_stub (const package_name& name) - { - auto i (find_if (imaginary_stubs.begin (), imaginary_stubs.end (), - [&name] (const shared_ptr<available_package>& p) - { - return p->id.name == name; - })); - - return i != imaginary_stubs.end () ? *i : nullptr; - } - - // Sort the available package fragments in the package version descending - // order and suppress duplicate packages. - // - static void - sort_dedup (vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>>& pfs) - { - sort (pfs.begin (), pfs.end (), - [] (const auto& x, const auto& y) - { - return x.first->version > y.first->version; - }); - - pfs.erase (unique (pfs.begin(), pfs.end(), - [] (const auto& x, const auto& y) - { - return x.first->version == y.first->version; - }), - pfs.end ()); - } - - // Try to find packages that optionally satisfy the specified version - // constraint in multiple databases, suppressing duplicates. Return the list - // of packages and repository fragments in which each was found in the - // package version descending or empty list if none were found. Note that a - // stub satisfies any constraint. - // - // Note that we return (loaded) lazy_shared_ptr in order to also convey - // the database to which it belongs. + // Note that the function may also issue a warning and return nullopt if the + // package minor version reached the limit (see standard-version.cxx for + // details). // - static vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>> - find_available (const linked_databases& dbs, - const package_name& name, - const optional<version_constraint>& c) + static optional<version_constraint> + patch_constraint (const package_name& nm, + const version& pv, + bool quiet = false) { - vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>> r; - - for (database& db: dbs) - { - for (shared_ptr<available_package> ap: - pointer_result (query_available (db, name, c))) - { - // An available package should come from at least one fetched - // repository fragment. - // - assert (!ap->locations.empty ()); - - // All repository fragments the package comes from are equally good, so - // we pick the first one. - // - r.emplace_back (move (ap), ap->locations[0].repository_fragment); - } - } - - // If there are multiple databases specified, then sort the result in the - // package version descending order and suppress duplicates. + // Note that we don't pass allow_stub flag so the system wildcard version + // will (naturally) not be patched. // - if (dbs.size () > 1) - sort_dedup (r); + string vs (pv.string ()); + optional<standard_version> v (parse_standard_version (vs)); - // Adding a stub from the imaginary system repository to the non-empty - // results isn't necessary but may end up with a duplicate. That's why we - // only add it if nothing else is found. - // - if (r.empty ()) + if (!v) { - if (shared_ptr<available_package> ap = find_imaginary_stub (name)) - r.emplace_back (move (ap), nullptr); - } - - return r; - } - - // As above but only look for packages from the specified list of repository - // fragments, their prerequisite repositories, and their complements, - // recursively (note: recursivity applies to complements, not prerequisites). - // - static vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>> - find_available (const package_name& name, - const optional<version_constraint>& c, - const config_repo_fragments& rfs, - bool prereq = true) - { - vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>> r; + if (!quiet) + warn << "unable to patch " << package_string (nm, pv) << + info << "package is not using semantic/standard version"; - for (const auto& dfs: rfs) - { - database& db (dfs.first); - for (auto& af: filter (dfs.second, - query_available (db, name, c), - prereq)) - { - r.emplace_back ( - move (af.first), - lazy_shared_ptr<repository_fragment> (db, move (af.second))); - } + return nullopt; } - if (rfs.size () > 1) - sort_dedup (r); - - if (r.empty ()) + try { - if (shared_ptr<available_package> ap = find_imaginary_stub (name)) - r.emplace_back (move (ap), nullptr); + return version_constraint ('~' + vs); } - - return r; - } - - // As above but only look for packages from a single repository fragment, - // its prerequisite repositories, and its complements, recursively (note: - // recursivity applies to complements, not prerequisites). Doesn't provide - // the repository fragments the packages come from. - // - // It is assumed that the repository fragment lazy pointer contains the - // database information. - // - static vector<shared_ptr<available_package>> - find_available (const package_name& name, - const optional<version_constraint>& c, - const lazy_shared_ptr<repository_fragment>& rf, - bool prereq = true) - { - vector<shared_ptr<available_package>> r; - - database& db (rf.database ()); - for (auto& ap: filter (rf.load (), query_available (db, name, c), prereq)) - r.emplace_back (move (ap)); - - if (r.empty ()) + // Note that the only possible reason for invalid_argument exception to be + // thrown is that minor version reached the 99999 limit (see + // standard-version.cxx for details). + // + catch (const invalid_argument&) { - if (shared_ptr<available_package> ap = find_imaginary_stub (name)) - r.emplace_back (move (ap)); - } + if (!quiet) + warn << "unable to patch " << package_string (nm, pv) << + info << "minor version limit reached"; - return r; + return nullopt; + } } - // As above but only look for a single package from the specified repository - // fragment, its prerequisite repositories, and their complements, - // recursively (note: recursivity applies to complements, not - // prerequisites). Return the package and the repository fragment in which - // it was found or NULL for both if not found. - // - // It is assumed that the repository fragment lazy pointer contains the - // database information. - // - static pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> - find_available_one (const package_name& name, - const optional<version_constraint>& c, - const lazy_shared_ptr<repository_fragment>& rf, - bool prereq = true, - bool revision = false) + static inline optional<version_constraint> + patch_constraint (const shared_ptr<selected_package>& sp, bool quiet = false) { - // Filter the result based on the repository fragment to which each - // version belongs. - // - database& db (rf.database ()); - auto r ( - filter_one (rf.load (), - query_available (db, name, c, true /* order */, revision), - prereq)); - - if (r.first == nullptr) - r.first = find_imaginary_stub (name); - - return make_pair (r.first, - (r.second != nullptr - ? lazy_shared_ptr<repository_fragment> (db, - move (r.second)) - : nullptr)); + return patch_constraint (sp->name, sp->version, quiet); } - // As above but look for a single package from a list of repository - // fragments. + // As above but returns a minor version constraint (^ shortcut) instead of + // the patch version constraint (~ shortcut). // - static pair<shared_ptr<available_package>, shared_ptr<repository_fragment>> - find_available_one (database& db, - const package_name& name, - const optional<version_constraint>& c, - const vector<shared_ptr<repository_fragment>>& rfs, - bool prereq = true, - bool revision = false) + static optional<version_constraint> + minor_constraint (const package_name& nm, + const version& pv, + bool quiet = false) { - // Filter the result based on the repository fragments to which each - // version belongs. + // Note that we don't pass allow_stub flag so the system wildcard version + // will (naturally) not be patched. // - auto r ( - filter_one (rfs, - query_available (db, name, c, true /* order */, revision), - prereq)); + string vs (pv.string ()); + optional<standard_version> v (parse_standard_version (vs)); - if (r.first == nullptr) - r.first = find_imaginary_stub (name); + if (!v) + { + if (!quiet) + warn << "unable to upgrade " << package_string (nm, pv) + << " to latest minor version" << + info << "package is not using semantic/standard version"; - return r; - } + return nullopt; + } - // As above but look for a single package in multiple databases from their - // respective root repository fragments. - // - static pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> - find_available_one (const linked_databases& dbs, - const package_name& name, - const optional<version_constraint>& c, - bool prereq = true, - bool revision = false) - { - for (database& db: dbs) + try { - auto r ( - filter_one (db.load<repository_fragment> (""), - query_available (db, name, c, true /* order */, revision), - prereq)); - - if (r.first != nullptr) - return make_pair ( - move (r.first), - lazy_shared_ptr<repository_fragment> (db, move (r.second))); + return version_constraint ('^' + vs); } + // Note that the only possible reason for invalid_argument exception to be + // thrown is that major version reached the 99999 limit (see + // standard-version.cxx for details). + // + catch (const invalid_argument&) + { + if (!quiet) + warn << "unable to upgrade " << package_string (nm, pv) + << " to latest minor version" << + info << "major version limit reached"; - return make_pair (find_imaginary_stub (name), nullptr); + return nullopt; + } } - // Create a transient (or fake, if you prefer) available_package object - // corresponding to the specified selected object. Note that the package - // locations list is left empty and that the returned repository fragment - // could be NULL if the package is an orphan. + // Return true if the selected package is not configured as system and its + // repository fragment is not present in any ultimate dependent + // configuration (see dependent_repo_configs() for details) or this exact + // version is not available from this repository fragment nor from its + // complements. Also return true if the selected package repository fragment + // is a root fragment (package is fetched/unpacked using the existing + // archive/directory). // - // Note also that in our model we assume that make_available_fragment() is - // only called if there is no real available_package. This makes sure that - // if the package moves (e.g., from testing to stable), then we will be - // using stable to resolve its dependencies. + // Note that the orphan definition here is stronger than in the rest of the + // code, since we request the available package to also be present in the + // repository fragment and consider packages built as existing archives or + // directories as orphans. It feels that such a definition aligns better + // with the user expectations about deorphaning. // - static pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> - make_available_fragment (const common_options& options, - database& db, - const shared_ptr<selected_package>& sp) + static bool + orphan_package (database& db, const shared_ptr<selected_package>& sp) { - shared_ptr<available_package> ap (make_available (options, db, sp)); + assert (sp != nullptr); if (sp->system ()) - return make_pair (move (ap), nullptr); - - // First see if we can find its repository fragment. - // - // Note that this is package's "old" repository fragment and there is no - // guarantee that its dependencies are still resolvable from it. But this - // is our best chance (we could go nuclear and point all orphans to the - // root repository fragment but that feels a bit too drastic at the - // moment). - // - // Also note that the repository information for this selected package can - // potentially be in one of the ultimate dependent configurations as - // determined at the time of the run when the package was configured. This - // configurations set may differ from the current one, but let's try - // anyway. - // - lazy_shared_ptr<repository_fragment> rf; + return false; - for (database& ddb: dependent_repo_configs (db)) - { - if (shared_ptr<repository_fragment> f = ddb.find<repository_fragment> ( - sp->repository_fragment.canonical_name ())) - { - rf = lazy_shared_ptr<repository_fragment> (ddb, move (f)); - break; - } - } + const string& cn (sp->repository_fragment.canonical_name ()); - return make_pair (move (ap), move (rf)); - } + if (cn.empty ()) // Root repository fragment? + return true; - // Try to find an available package corresponding to the specified selected - // package and, if not found, return a transient one. - // - static shared_ptr<available_package> - find_available (const common_options& options, - database& db, - const shared_ptr<selected_package>& sp) - { - available_package_id pid (sp->name, sp->version); for (database& ddb: dependent_repo_configs (db)) { - shared_ptr<available_package> ap (ddb.find<available_package> (pid)); + const shared_ptr<repository_fragment> rf ( + ddb.find<repository_fragment> (cn)); - if (ap != nullptr && !ap->stub ()) - return ap; - } + if (rf != nullptr && !rep_masked_fragment (ddb, rf)) + { + auto af ( + find_available_one (sp->name, + version_constraint (sp->version), + lazy_shared_ptr<repository_fragment> (ddb, + move (rf)), + false /* prereq */, + true /* revision */)); - return make_available (options, db, sp); - } + const shared_ptr<available_package>& ap (af.first); - // As above but also pair the available package with the repository fragment - // the available package comes from. Note that the package locations list is - // left empty and that the returned repository fragment could be NULL if the - // package is an orphan. - // - static pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> - find_available_fragment (const common_options& options, - database& db, - const shared_ptr<selected_package>& sp) - { - available_package_id pid (sp->name, sp->version); - for (database& ddb: dependent_repo_configs (db)) - { - shared_ptr<available_package> ap (ddb.find<available_package> (pid)); - - if (ap != nullptr && !ap->stub ()) - { - if (shared_ptr<repository_fragment> f = ddb.find<repository_fragment> ( - sp->repository_fragment.canonical_name ())) - return make_pair (ap, - lazy_shared_ptr<repository_fragment> (ddb, - move (f))); + if (ap != nullptr && !ap->stub ()) + return false; } } - return make_pair (find_available (options, db, sp), nullptr); + return true; } - // Return true if the version constraint represents the wildcard version. + // List of dependency packages (specified with ? on the command line). // - static inline bool - wildcard (const version_constraint& vc) + // If configuration is not specified for a system dependency package (db is + // NULL), then the dependency is assumed to be specified for all current + // configurations and their explicitly linked configurations, recursively, + // including private configurations that can potentially be created during + // this run. + // + // The selected package is not NULL if the database is not NULL and the + // dependency package is present in this database. + // + struct dependency_package { - bool r (vc.min_version && *vc.min_version == wildcard_version); + database* db; // Can only be NULL if system. + package_name name; + optional<version_constraint> constraint; // nullopt if unspecified. - if (r) - assert (vc.max_version == vc.min_version); + // Can only be true if constraint is specified. + // + bool hold_version; - return r; - } + shared_ptr<selected_package> selected; + bool system; + bool existing; // Build as archive or directory. - // The current configurations dependents being "repointed" to prerequisites - // in other configurations, together with their replacement flags. The flag - // is true for the replacement prerequisites ("new") and false for the - // prerequisites being replaced ("old"). The unamended prerequisites have no - // entries. - // - using repointed_dependents = map<package_key, map<package_key, bool>>; + // true -- upgrade, false -- patch. + // + optional<bool> upgrade; // Only for absent constraint. + + bool deorphan; + bool keep_out; + bool disfigure; + optional<dir_path> checkout_root; + bool checkout_purge; + strings config_vars; // Only if not system. + const system_package_status* system_status; // See struct pkg_arg. + }; + using dependency_packages = vector<dependency_package>; - // List of the private configuration paths, relative to the containing - // configuration directories (.bpkg/host/, etc), together with the - // containing configuration databases. + // Evaluate a dependency package and return a new desired version. If the + // result is absent (nullopt), then there are no user expectations regarding + // this dependency. If the result is a NULL available_package, then it is + // either no longer used and can be dropped, or no changes to the dependency + // are necessary. Otherwise, the result is available_package to + // upgrade/downgrade to or replace with the same version (deorphan, rebuild + // as an existing archive or directory, etc) as well as the repository + // fragment it must come from, the system flag, and the database it must be + // configured in. // - using private_configs = vector<pair<database&, dir_path>>; - - // A "dependency-ordered" list of packages and their prerequisites. - // That is, every package on the list only possibly depending on the - // ones after it. In a nutshell, the usage is as follows: we first - // add one or more packages (the "initial selection"; for example, a - // list of packages the user wants built). The list then satisfies all - // the prerequisites of the packages that were added, recursively. At - // the end of this process we have an ordered list of all the packages - // that we have to build, from last to first, in order to build our - // initial selection. + // If the dependency is being rebuilt as an existing archive or directory we + // may end up with the available package version being the same as the + // selected package version. In this case the dependency needs to be + // re-fetched/re-unpacked from this archive/directory. Also note that if the + // dependency needs to be rebuilt as an existing archive or directory the + // caller may need to stash its name/database. This way on the subsequent + // call this function may return the "no change" recommendation rather than + // the "replace" recommendation. // - // This process is split into two phases: satisfaction of all the - // dependencies (the collect_build() function) and ordering of the list - // (the order() function). + // If in the deorphan mode it turns out that the package is not an orphan + // and there is no version constraint specified and upgrade/patch is not + // requested, then assume that no changes are necessary for the dependency. + // Otherwise, if the package version is not constrained and no upgrade/patch + // is requested, then pick the version that matches the dependency version + // best in the following preference order: // - // During the satisfaction phase, we collect all the packages, their - // prerequisites (and so on, recursively) in a map trying to satisfy - // any version constraints. Specifically, during this step, we may - // "upgrade" or "downgrade" a package that is already in a map as a - // result of another package depending on it and, for example, requiring - // a different version. If that happens, we make sure that the replaced - // package version doesn't apply constraints and/or configuration to its - // own dependencies anymore and also that its non-shared dependencies are - // gone from the map, recursively (see replaced_versions for details). - // One notable side-effect of this process is that all the packages in the - // map end up in the list. + // - same version, revision, and iteration + // - latest iteration of same version and revision + // - later revision of same version + // - later patch of same version + // - later minor of same version + // - latest available version, including earlier // - // Note that we don't try to do exhaustive constraint satisfaction (i.e., - // there is no backtracking). Specifically, if we have two candidate - // packages each satisfying a constraint of its dependent package, then if - // neither of them satisfy both constraints, then we give up and ask the - // user to resolve this manually by explicitly specifying the version that - // will satisfy both constraints. + // Otherwise, always upgrade/downgrade the orphan or fail if no satisfactory + // version is available. Note that in the both cases (deorphan and + // upgrade/downgrade+deorphan) we may end up with the available package + // version being the same as the selected package version. In this case the + // dependency needs to be re-fetched from an existing repository. Also note + // that if the dependency needs to be deorphaned the caller may need to + // cache the original orphan version. This way on the subsequent calls this + // function still considers this package as an orphan and uses its original + // version to deduce the best match, which may change due, for example, to a + // change of the constraining dependents set. // - // Also note that we rule out dependency alternatives with enable constraint - // that evaluates to false and try to select one satisfactory alternative if - // there are multiple of them. In the latter case we pick the first - // alternative with packages that are already used (as a result of being - // dependencies of other package, requested by the user, or already being - // present in the configuration) and fail if such an alternative doesn't - // exist. + // If the package version that satisfies explicitly specified dependency + // version constraint can not be found in the dependents repositories, then + // return the "no changes are necessary" result if ignore_unsatisfiable + // argument is true and fail otherwise. The common approach is to pass true + // for this argument until the execution plan is finalized, assuming that + // the problematic dependency might be dropped. // - struct build_package + struct evaluate_result { - enum action_type - { - // Available package is not NULL. - // - build, - - // Selected package is not NULL, available package is NULL. - // - drop, - - // Selected package is not NULL, available package is NULL. - // - // This is the "only adjustments" action for a selected package. - // Adjustment flags (see below) are unhold (the package should be - // treated as a dependency) and reconfigure (dependent package that - // needs to be reconfigured because its prerequisite is being - // up/down-graded or reconfigured). - // - // Note that this action is "replaceable" with either drop or build - // action but in the latter case the adjustments must be copied over. - // - adjust - }; - - // An object with an absent action is there to "pre-enter" information - // about a package (constraints and flags) in case it is used. - // - optional<action_type> action; - - reference_wrapper<database> db; // Needs to be move-assignable. - - shared_ptr<selected_package> selected; // NULL if not selected. - shared_ptr<available_package> available; // Can be NULL, fake/transient. - - // Can be NULL (orphan) or root. If not NULL, then loaded from the - // repository configuration database, which may differ from the - // configuration the package is being built in. + // The system, existing, upgrade, and orphan members are meaningless if + // the unused flag is true. // + reference_wrapper<database> db; + shared_ptr<available_package> available; lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; + bool unused; + bool system; + bool existing; + optional<bool> upgrade; - const package_name& - name () const - { - return selected != nullptr ? selected->name : available->id.name; - } - - // If we end up collecting the prerequisite builds for this package, then - // this member stores copies of the selected dependency alternatives. The - // dependency alternatives for toolchain build-time dependencies and for - // dependencies which have all the alternatives disabled are represented - // as empty dependency alternatives lists. If present, it is parallel to - // the available package's dependencies member. - // - // Initially nullopt. Can be filled partially if the package prerequisite - // builds collection is postponed for any reason (see postponed_packages - // and postponed_configurations for possible reasons). - // - optional<bpkg::dependencies> dependencies; - - // Indexes of the selected dependency alternatives stored in the above - // dependencies member. - // - optional<vector<size_t>> alternatives; - - // If we end up collecting the prerequisite builds for this package, then - // this member stores the skeleton of the package being built. - // - // Initially nullopt. Can potentially be loaded but with the reflection - // configuration variables collected only partially if the package - // prerequisite builds collection is postponed for any reason. Can also be - // unloaded if the package has no conditional dependencies. - // - optional<package_skeleton> skeleton; - - // If the package prerequisite builds collection is postponed, then this - // member stores the references to the enabled alternatives (in available - // package) of a dependency being the cause of the postponement together - // with their original indexes in the respective dependency alternatives - // list. This, in particular, allows us not to re-evaluate conditions - // multiple times on the re-collection attempts. - // - // Note: it shouldn't be very common for a dependency to contain more than - // two true alternatives. - // - using dependency_alternatives_refs = - small_vector<pair<reference_wrapper<const dependency_alternative>, - size_t>, - 2>; - - optional<dependency_alternatives_refs> postponed_dependency_alternatives; - - // True if the recursive collection of the package has been started or - // performed. - // - // Used by the dependency configuration negotiation machinery which makes - // sure that its configuration is negotiated between dependents before its - // recursive collection is started (see postponed_configurations for - // details). - // - // Note that the dependencies member cannot be used for that purpose since - // it is not always created (think of a system dependency or an existing - // dependency that doesn't need its prerequisites re-collection). In a - // sense the recursive collection flag is a barrier for the dependency - // configuration negotiation. - // - bool recursive_collection; - - // Hold flags. Note that we only "increase" the hold_package value that is - // already in the selected package. - // - optional<bool> hold_package; - optional<bool> hold_version; - - // Constraint value plus, normally, the dependent package name that placed - // this constraint but can also be some other name for the initial - // selection (e.g., package version specified by the user on the command - // line). This why we use the string type, rather than package_name. - // - struct constraint_type - { - reference_wrapper<database> db; // Main database for non-packages. - string dependent; - version_constraint value; - - constraint_type (database& d, string dp, version_constraint v) - : db (d), dependent (move (dp)), value (move (v)) {} - }; - - vector<constraint_type> constraints; - - // System package indicator. See also a note in the merge() function. - // - bool system; - - // If this flag is set and the external package is being replaced with an - // external one, then keep its output directory between upgrades and - // downgrades. - // - bool keep_out; - - // If this flag is set then disfigure the package between upgrades and - // downgrades effectively causing a from-scratch reconfiguration. - // - bool disfigure; - - // If this flag is set, then don't build this package, only configure. - // - // Note: use configure_only() to query. - // - bool configure_only_; - - // If present, then check out the package into the specified directory - // rather than into the configuration directory, if it comes from a - // version control-based repository. Optionally, remove this directory - // when the package is purged. - // - optional<dir_path> checkout_root; - bool checkout_purge; - - // Command line configuration variables. Only meaningful for non-system - // packages. - // - strings config_vars; - - // Set of packages (dependents or dependencies but not a mix) that caused - // this package to be built or adjusted. Empty name signifies user - // selection and can be present regardless of the required_by_dependents - // flag value. - // - set<package_key> required_by; - - // If this flags is true, then required_by contains dependents. - // - // We need this because required_by packages have different semantics for - // different actions: the dependent for regular builds and dependency for - // adjustments and repointed dependent reconfiguration builds. Mixing them - // would break prompts/diagnostics. + // Original orphan version which needs to be deorphaned. May only be + // present for the deorphan mode. // - bool required_by_dependents; + optional<version> orphan; + }; - // Consider a package as user-selected if it is specified on the command - // line, is a held package being upgraded via the `pkg-build -u|-p` - // command form, or is a dependency being upgraded via the recursively - // upgraded dependent. - // - bool - user_selection () const - { - return required_by.find (package_key {db.get ().main_database (), - ""}) != required_by.end (); - } + struct dependent_constraint + { + database& db; + shared_ptr<selected_package> package; + optional<version_constraint> constraint; - // Consider a package as user-selected only if it is specified on the - // command line as build to hold. - // - bool - user_selection (const vector<build_package>& hold_pkgs) const - { - return find_if (hold_pkgs.begin (), hold_pkgs.end (), - [this] (const build_package& p) - { - return p.db == db && p.name () == name (); - }) != hold_pkgs.end (); - } + dependent_constraint (database& d, + shared_ptr<selected_package> p, + optional<version_constraint> c) + : db (d), package (move (p)), constraint (move (c)) {} + }; - // Return true if the configured package needs to be recollected - // recursively. - // - // This is required if it is being built as a source package and needs to - // be up/down-graded and/or reconfigured and has some buildfile clauses, - // it is a repointed dependent, or it is already in the process of being - // collected. - // - bool - recollect_recursively (const repointed_dependents& rpt_depts) const - { - assert (action && - *action == build_package::build && - available != nullptr && - selected != nullptr && - selected->state == package_state::configured && - selected->substate != package_substate::system); - - // Note that if the skeleton is present then the package is either being - // already collected or its configuration has been negotiated between - // the dependents. - // - return !system && - (dependencies || - selected->version != available_version () || - ((!config_vars.empty () || skeleton) && - has_buildfile_clause (available->dependencies)) || - rpt_depts.find (package_key (db, name ())) != rpt_depts.end ()); - } + using dependent_constraints = vector<dependent_constraint>; + using deorphaned_dependencies = map<package_key, version>; + using existing_dependencies = vector<package_key>; - // State flags. - // - uint16_t flags; + static evaluate_result + evaluate_dependency (const common_options&, + database&, + const shared_ptr<selected_package>&, + const optional<version_constraint>& desired, + bool desired_sys, + bool existing, + database& desired_db, + const shared_ptr<selected_package>& desired_db_sp, + optional<bool> upgrade, + bool deorphan, + bool explicitly, + const config_repo_fragments&, + const dependent_constraints&, + const existing_dependencies&, + const deorphaned_dependencies&, + const build_packages&, + bool ignore_unsatisfiable); - // Set if we also need to clear the hold package flag. - // - static const uint16_t adjust_unhold = 0x0001; + // If there are no user expectations regarding this dependency, then we give + // no up/down-grade/replace recommendation, unless there are no dependents + // in which case we recommend to drop the dependency. + // + // Note that the user expectations are only applied for dependencies that + // have dependents in the current configurations. + // + static optional<evaluate_result> + evaluate_dependency (const common_options& o, + database& db, + const shared_ptr<selected_package>& sp, + const dependency_packages& deps, + bool no_move, + const existing_dependencies& existing_deps, + const deorphaned_dependencies& deorphaned_deps, + const build_packages& pkgs, + bool ignore_unsatisfiable) + { + tracer trace ("evaluate_dependency"); - bool - unhold () const - { - return (flags & adjust_unhold) != 0; - } + assert (sp != nullptr && !sp->hold_package); - // Set if we also need to reconfigure this package. Note that in some - // cases reconfigure is naturally implied. For example, if an already - // configured package is being up/down-graded. For such cases we don't - // guarantee that the reconfigure flag is set. We only make sure to set it - // for cases that would otherwise miss the need for reconfiguration. As a - // result, use the reconfigure() predicate which detects both explicit and - // implied cases. - // - // At first, it may seem that this flag is redundant and having the - // available package set to NULL is sufficient. But consider the case - // where the user asked us to build a package that is already in the - // configured state (so all we have to do is pkg-update). Next, add to - // this a prerequisite package that is being upgraded. Now our original - // package has to be reconfigured. But without this flag we won't know - // (available for our package won't be NULL). - // - static const uint16_t adjust_reconfigure = 0x0002; + const package_name& nm (sp->name); - bool - reconfigure () const + auto no_change = [&db] () { - assert (action && *action != drop); - - return selected != nullptr && - selected->state == package_state::configured && - ((flags & adjust_reconfigure) != 0 || - (*action == build && - (selected->system () != system || - selected->version != available_version () || - (!system && (!config_vars.empty () || disfigure))))); - } + return evaluate_result {db, + nullptr /* available */, + nullptr /* repository_fragment */, + false /* unused */, + false /* system */, + false /* existing */, + nullopt /* upgrade */, + nullopt /* orphan */}; + }; - // Set if this build action is for repointing of prerequisite. + // Only search for the user expectations regarding this dependency if it + // has dependents in the current configurations, unless --no-move is + // specified. // - static const uint16_t build_repoint = 0x0004; - - // Set if this build action is for re-evaluating of an existing dependent. + // In the no-move mode consider the user-specified configurations not as a + // dependency new location, but as the current location of the dependency + // to which the expectations are applied. Note that multiple package specs + // for the same dependency in different configurations can be specified on + // the command line. // - static const uint16_t build_reevaluate = 0x0008; - - bool - configure_only () const - { - assert (action); - - return configure_only_ || - (*action == build && (flags & (build_repoint | build_reevaluate)) != 0); - } + linked_databases cur_dbs; + dependency_packages::const_iterator i (deps.end ()); - // Return true if the resulting package will be configured as external. - // Optionally, if the package is external, return its absolute and - // normalized source root directory path. - // - bool - external (dir_path* d = nullptr) const + if (!no_move) { - assert (action); - - if (*action == build_package::drop) - return false; - - // If adjustment or orphan, then new and old are the same. + // Collect the current configurations which contain dependents for this + // dependency and assume no expectations if there is none. // - if (available == nullptr || available->locations.empty ()) + for (database& cdb: current_configs) { - assert (selected != nullptr); - - if (selected->external ()) - { - assert (selected->src_root); - - if (d != nullptr) - *d = *selected->src_root; - - return true; - } + if (!query_dependents (cdb, nm, db).empty ()) + cur_dbs.push_back (cdb); } - else - { - const package_location& pl (available->locations[0]); - - if (pl.repository_fragment.object_id () == "") // Special root? - { - if (!exists (pl.location)) // Directory case? - { - if (d != nullptr) - *d = normalize (path_cast<dir_path> (pl.location), "package"); - return true; - } - } - else + // Search for the user expectations regarding this dependency by + // matching the package name and configuration type, if configuration is + // specified, preferring entries with configuration specified and fail + // if there are multiple candidates. + // + if (!cur_dbs.empty ()) + { + for (dependency_packages::const_iterator j (deps.begin ()); + j != deps.end (); + ++j) { - // See if the package comes from the directory-based repository, and - // so is external. - // - // Note that such repository fragments are always preferred over - // others (see below). - // - for (const package_location& pl: available->locations) + if (j->name == nm && (j->db == nullptr || j->db->type == db.type)) { - const repository_location& rl ( - pl.repository_fragment.load ()->location); - - if (rl.directory_based ()) + if (i == deps.end () || i->db == nullptr) { - // Note that the repository location path is always absolute for - // the directory-based repositories but the package location may - // potentially not be normalized. Thus, we normalize the - // resulting path, if requested. - // - if (d != nullptr) - *d = normalize (path_cast<dir_path> (rl.path () / pl.location), - "package"); - - return true; + i = j; + } + else if (j->db != nullptr) + { + fail << "multiple " << db.type << " configurations specified " + << "for dependency package " << nm << + info << i->db->config_orig << + info << j->db->config_orig << + info << "consider using the --no-move option"; } } } } - - return false; - } - - // If the resulting package will be configured as external, then return - // its absolute and normalized source root directory path and nullopt - // otherwise. - // - optional<dir_path> - external_dir () const - { - dir_path r; - return external (&r) ? optional<dir_path> (move (r)) : nullopt; } - - const version& - available_version () const + else { - // This should have been diagnosed before creating build_package object. - // - assert (available != nullptr && - (system - ? available->system_version (db) != nullptr - : !available->stub ())); - - return system ? *available->system_version (db) : available->version; - } + for (dependency_packages::const_iterator j (deps.begin ()); + j != deps.end (); + ++j) + { + if (j->name == nm && (j->db == nullptr || *j->db == db)) + { + if (i == deps.end () || i->db == nullptr) + i = j; - string - available_name_version () const - { - assert (available != nullptr); - return package_string (available->id.name, - available_version (), - system); + if (i->db != nullptr) + break; + } + } } - string - available_name_version_db () const - { - const string& s (db.get ().string); - return !s.empty () - ? available_name_version () + ' ' + s - : available_name_version (); - } + bool user_exp (i != deps.end ()); + bool copy_dep (user_exp && i->db != nullptr && *i->db != db); - // Merge constraints, required-by package names, hold_* flags, state - // flags, and user-specified options/variables. + // Collect the dependents for checking the version constraints, using + // their repository fragments for discovering available dependency package + // versions, etc. // - void - merge (build_package&& p) - { - // We don't merge objects from different configurations. - // - assert (db == p.db); - - // We don't merge into pre-entered objects, and from/into drops. - // - assert (action && *action != drop && (!p.action || *p.action != drop)); - - // We never merge two repointed dependent reconfigurations. - // - assert ((flags & build_repoint) == 0 || - (p.flags & build_repoint) == 0); - - // We never merge two existing dependent re-evaluations. - // - assert ((flags & build_reevaluate) == 0 || - (p.flags & build_reevaluate) == 0); + // Note that if dependency needs to be copied, then we only consider its + // dependents in the current configurations which potentially can be + // repointed to it. Note that configurations of such dependents must + // contain the new dependency configuration in their dependency tree. + // + linked_databases dep_dbs; - // Copy the user-specified options/variables. - // - if (p.user_selection ()) + if (copy_dep) + { + for (database& db: i->db->dependent_configs ()) { - // We don't allow a package specified on the command line multiple - // times to have different sets of options/variables. Given that, it's - // tempting to assert that the options/variables don't change if we - // merge into a user selection. That's, however, not the case due to - // the iterative plan refinement implementation details (--checkout-* - // options and variables are only saved into the pre-entered - // dependencies, etc.). - // - // Note that configuration can only be specified for packages on the - // command line and such packages get collected/pre-entered early, - // before any prerequisites get collected. Thus, it doesn't seem - // possible that a package configuration/options may change after we - // have created the package skeleton. - // - // Also note that if it wouldn't be true, we would potentially need to - // re-collect the package prerequisites, since configuration change - // could affect the enable condition evaluation and, as a result, the - // dependency alternative choice. - // - assert (!skeleton || - ((p.config_vars.empty () || p.config_vars == config_vars) && - p.disfigure == disfigure)); - - if (p.keep_out) - keep_out = p.keep_out; - - if (p.disfigure) - disfigure = p.disfigure; - - if (p.configure_only_) - configure_only_ = p.configure_only_; - - if (p.checkout_root) - checkout_root = move (p.checkout_root); - - if (p.checkout_purge) - checkout_purge = p.checkout_purge; - - if (!p.config_vars.empty ()) - config_vars = move (p.config_vars); - - // Propagate the user-selection tag. - // - required_by.emplace (db.get ().main_database (), ""); + if (find (cur_dbs.begin (), cur_dbs.end (), db) != cur_dbs.end ()) + dep_dbs.push_back (db); } - // Copy the required-by package names only if semantics matches. - // - if (p.required_by_dependents == required_by_dependents) - required_by.insert (p.required_by.begin (), p.required_by.end ()); - - // Copy constraints. - // - // Note that we may duplicate them, but this is harmless. - // - constraints.insert (constraints.end (), - make_move_iterator (p.constraints.begin ()), - make_move_iterator (p.constraints.end ())); - - // Copy hold_* flags if they are "stronger". - // - if (!hold_package || (p.hold_package && *p.hold_package > *hold_package)) - hold_package = p.hold_package; - - if (!hold_version || (p.hold_version && *p.hold_version > *hold_version)) - hold_version = p.hold_version; - - // Copy state flags. - // - flags |= p.flags; - - // Upgrade dependent repointments and re-evaluations to the full builds. - // - if (*action == build) - flags &= ~(build_repoint | build_reevaluate); - - // Note that we don't copy the build_package::system flag. If it was - // set from the command line ("strong system") then we will also have - // the '==' constraint which means that this build_package object will - // never be replaced. + // Bail out if no dependents can be repointed to the dependency. // - // For other cases ("weak system") we don't want to copy system over in - // order not prevent, for example, system to non-system upgrade. + if (dep_dbs.empty ()) + { + l5 ([&]{trace << *sp << db << ": can't repoint";}); + return no_change (); + } } + else + dep_dbs = db.dependent_configs (); - // Initialize the skeleton of a being built package. + // Collect the dependents but bail out if the dependency is used but there + // are no user expectations regarding it. // - package_skeleton& - init_skeleton (const common_options& options, - const shared_ptr<available_package>& override = nullptr) - { - shared_ptr<available_package> ap (override != nullptr - ? override - : available); - - assert (!skeleton && ap != nullptr); + vector<pair<database&, package_dependent>> pds; - package_key pk (db, ap->id.name); + for (database& ddb: dep_dbs) + { + auto ds (query_dependents (ddb, nm, db)); - if (system) + if (!ds.empty ()) { - // Keep the available package if its version is "close enough" to the - // system package version. For now we will require the exact match - // but in the future we could relax this (e.g., allow the user to - // specify something like libfoo/^1.2.0 or some such). - // - const version* v (!ap->stub () ? ap->system_version (db) : nullptr); - - if (v == nullptr || *v != ap->version) - ap = nullptr; - } - - optional<dir_path> src_root, out_root; + if (!user_exp) + return nullopt; - if (ap != nullptr) - { - src_root = external_dir (); - out_root = (src_root && !disfigure - ? dir_path (db.get ().config) /= name ().string () - : optional<dir_path> ()); + for (auto& d: ds) + pds.emplace_back (ddb, move (d)); } - - skeleton = package_skeleton ( - options, - move (pk), - system, - move (ap), - config_vars, // @@ Maybe make optional<strings> and move? - disfigure, - (selected != nullptr ? &selected->config_variables : nullptr), - move (src_root), - move (out_root)); - - return *skeleton; } - }; - - using build_package_list = list<reference_wrapper<build_package>>; - using build_package_refs = - small_vector<reference_wrapper<const build_package>, 16>; - - using add_priv_cfg_function = void (database&, dir_path&&); - - // Base for exception types that indicate an inability to collect a package - // build because it was collected prematurely (version needs to be replaced, - // configuration requires further negotiation, etc). - // - struct scratch_collection - { - // Only used for tracing. + // Bail out if the dependency is unused. // - const char* description; - const package_key* package = nullptr; // Could be NULL. + if (pds.empty ()) + { + l5 ([&]{trace << *sp << db << ": unused";}); - explicit - scratch_collection (const char* d): description (d) {} - }; + return evaluate_result {db, + nullptr /* available */, + nullptr /* repository_fragment */, + true /* unused */, + false /* system */, + false /* existing */, + nullopt /* upgrade */, + nullopt /* orphan */}; + } - // Map of packages which need to be re-collected with the different version - // and/or system flag or dropped. - // - // Note that the initial package version may be adjusted to satisfy - // constraints of dependents discovered during the packages collection. It - // may also be dropped if this is a dependency which turns out to be unused. - // However, it may not always be possible to perform such an adjustment - // in-place since the intermediate package version could already apply some - // constraints and/or configuration to its own dependencies. Thus, we may - // need to note the desired package version information and re-collect from - // scratch. - // - // Also note that during re-collection such a desired version may turn out - // to not be a final version and the adjustment/re-collection can repeat. - // - // And yet, it doesn't seem plausible to ever create a replacement for the - // drop: replacing one drop with another is meaningless (all drops are the - // same) and replacing the package drop with a package version build can - // always been handled in-place. - // - // On the first glance, the map entries which have not been used for - // replacement during the package collection (bogus entries) are harmless - // and can be ignored. However, the dependency configuration negotiation - // machinery refers to this map and skips existing dependents with - // configuration clause which belong to it (see query_existing_dependents() - // for details). Thus, if after collection of packages some bogus entries - // are present in the map, then it means that we could have erroneously - // skipped some existing dependents because of them and so need to erase - // these entries and re-collect. - // - struct replaced_version - { - // Desired package version, repository fragment, and system flag. - // - // Both are NULL for the replacement with the drop. + // The requested dependency database, version constraint, and system flag. // - shared_ptr<available_package> available; - lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; - bool system; // Meaningless for the drop. + assert (i != deps.end ()); - // True if the entry has been inserted or used for the replacement during - // the current (re-)collection iteration. Used to keep track of "bogus" - // (no longer relevant) entries. - // - bool replaced; + database& ddb (i->db != nullptr ? *i->db : db); + const optional<version_constraint>& dvc (i->constraint); // May be nullopt. + bool dsys (i->system); + bool existing (i->existing); + bool deorphan (i->deorphan); - // Create replacement with the different version. + // The selected package in the desired database which we copy over. // - replaced_version (shared_ptr<available_package> a, - lazy_shared_ptr<bpkg::repository_fragment> f, - bool s) - : available (move (a)), - repository_fragment (move (f)), - system (s), - replaced (true) {} - - // Create replacement with the drop. + // It is the current dependency package, if we don't copy, and may or may + // not exist otherwise. // - replaced_version (): system (false), replaced (true) {} - }; + shared_ptr<selected_package> dsp (db == ddb + ? sp + : ddb.find<selected_package> (nm)); - class replaced_versions: public map<package_key, replaced_version> - { - public: - // Erase the bogus replacements and, if any, throw cancel_replacement, if - // requested. + // If a package in the desired database is already selected and matches + // the user expectations then no package change is required, unless the + // package is also being built as an existing archive or directory or + // needs to be deorphaned. // - struct cancel_replacement: scratch_collection + if (dsp != nullptr && dvc) { - cancel_replacement () - : scratch_collection ("bogus version replacement cancellation") {} - }; + const version& sv (dsp->version); + bool ssys (dsp->system ()); - void - cancel_bogus (tracer& trace, bool scratch) - { - bool bogus (false); - for (auto i (begin ()); i != end (); ) + if (!existing && + !deorphan && + ssys == dsys && + (ssys ? sv == *dvc->min_version : satisfies (sv, dvc))) { - const replaced_version& v (i->second); + l5 ([&]{trace << *dsp << ddb << ": unchanged";}); + return no_change (); + } + } - if (!v.replaced) - { - bogus = true; + // Build a set of repository fragments the dependent packages come from. + // Also cache the dependents and the constraints they apply to this + // dependency. + // + config_repo_fragments repo_frags; + dependent_constraints dpt_constrs; - l5 ([&]{trace << "erase bogus version replacement " - << i->first;}); + for (auto& pd: pds) + { + database& ddb (pd.first); + package_dependent& dep (pd.second); - i = erase (i); - } - else - ++i; - } + shared_ptr<selected_package> p (ddb.load<selected_package> (dep.name)); + add_dependent_repo_fragments (ddb, p, repo_frags); - if (bogus && scratch) - { - l5 ([&]{trace << "bogus version replacement erased, throwing";}); - throw cancel_replacement (); - } + dpt_constrs.emplace_back (ddb, move (p), move (dep.constraint)); } - }; - // List of dependency groups whose recursive processing should be postponed - // due to dependents with configuration clauses, together with these - // dependents (we will call them package clusters). - // - // The idea is that configuration for the dependencies in the cluster needs - // to be negotiated between the dependents in the cluster. Note that at any - // given time during collection a dependency can only belong to a single - // cluster. For example, the following dependent/dependencies with - // configuration clauses: - // - // foo: depends: libfoo - // bar: depends: libfoo - // depends: libbar - // baz: depends: libbaz - // - // End up in the following clusters (see string() below for the cluster - // representation): - // - // {foo bar | libfoo->{foo/1,1 bar/1,1}} - // {bar | libbar->{bar/2,1}} - // {baz | libbaz->{baz/1,1}} - // - // Or, another example: - // - // foo: depends: libfoo - // bar: depends: libfoo libbar - // baz: depends: libbaz - // - // {foo bar | libfoo->{foo/1,1 bar/1,1} libbar->{bar/1,1}} - // {baz | libbaz->{baz/1,1}} - // - // Note that a dependent can belong to any given non-negotiated cluster with - // only one `depends` position. However, if some dependency configuration is - // up-negotiated for a dependent, then multiple `depends` positions will - // correspond to this dependent in the same cluster. Naturally, such - // clusters are always (being) negotiated. - // - // Note that adding new dependent/dependencies to the postponed - // configurations can result in merging some of the existing clusters if the - // dependencies being added intersect with multiple clusters. For example, - // adding: - // - // fox: depends: libbar libbaz - // - // to the clusters in the second example will merge them into a single - // cluster: - // - // {foo bar baz fox | libfoo->{foo/1,1 bar/1,1} libbar->{bar/1,1 fox/1,1} - // libbaz->{baz/1,1 fox/1,1}} - // - // Also note that we keep track of packages which turn out to be - // dependencies of existing (configured) dependents with configuration - // clauses. The recursive processing of such packages should be postponed - // until negotiation between all the existing and new dependents which may - // or may not be present. - // - class postponed_configuration; - - static ostream& - operator<< (ostream&, const postponed_configuration&); + return evaluate_dependency (o, + db, + sp, + dvc, + dsys, + existing, + ddb, + dsp, + i->upgrade, + deorphan, + true /* explicitly */, + repo_frags, + dpt_constrs, + existing_deps, + deorphaned_deps, + pkgs, + ignore_unsatisfiable); + } - class postponed_configuration + struct config_selected_package { - public: - // The id of the cluster plus the ids of all the clusters that have been - // merged into it. - // - size_t id; - small_vector<size_t, 1> merged_ids; + database& db; + const shared_ptr<selected_package>& package; - using packages = small_vector<package_key, 1>; + config_selected_package (database& d, + const shared_ptr<selected_package>& p) + : db (d), package (p) {} - class dependency: public packages + bool + operator== (const config_selected_package& v) const { - public: - pair<size_t, size_t> position; // depends + alternative - - dependency (const pair<size_t, size_t>& pos, packages deps) - : packages (move (deps)), position (pos) {} - }; + return package->name == v.package->name && db == v.db; + } - class dependent_info + bool + operator< (const config_selected_package& v) const { - public: - bool existing; - small_vector<dependency, 1> dependencies; + int r (package->name.compare (v.package->name)); + return r != 0 ? (r < 0) : (db < v.db); + } + }; - dependency* - find_dependency (pair<size_t, size_t> pos) - { - auto i (find_if (dependencies.begin (), - dependencies.end (), - [&pos] (const dependency& d) - { - return d.position == pos; - })); - return i != dependencies.end () ? &*i : nullptr; - } + static evaluate_result + evaluate_dependency (const common_options& o, + database& db, + const shared_ptr<selected_package>& sp, + const optional<version_constraint>& dvc, + bool dsys, + bool existing, + database& ddb, + const shared_ptr<selected_package>& dsp, + optional<bool> upgrade, + bool deorphan, + bool explicitly, + const config_repo_fragments& rfs, + const dependent_constraints& dpt_constrs, + const existing_dependencies& existing_deps, + const deorphaned_dependencies& deorphaned_deps, + const build_packages& pkgs, + bool ignore_unsatisfiable) + { + tracer trace ("evaluate_dependency"); - void - add (dependency&& dep) - { - if (dependency* d = find_dependency (dep.position)) - { - // Feels like we can accumulate dependencies into an existing - // position only for an existing dependent. - // - assert (existing); + const package_name& nm (sp->name); - for (package_key& p: dep) - { - // Add the dependency unless it's already there. - // - if (find (d->begin (), d->end (), p) == d->end ()) - d->push_back (move (p)); - } - } - else - dependencies.push_back (move (dep)); - } + auto no_change = [&db] () + { + return evaluate_result {db, + nullptr /* available */, + nullptr /* repository_fragment */, + false /* unused */, + false /* system */, + false /* existing */, + nullopt /* upgrade */, + nullopt /* orphan */}; }; - using dependents_map = map<package_key, dependent_info>; - - dependents_map dependents; - packages dependencies; - - // Dependency configuration. - // - // Note that this container may not yet contain some entries that are - // already in the dependencies member above. And it may already contain - // entries that are not yet in dependencies due to the retry_configuration - // logic. - // - package_configurations dependency_configurations; - - // Shadow clusters. + // Build the list of available packages for the potential up/down-grade + // to, in the version-descending order. If patching, then we constrain the + // choice with the latest patch version and place no constraints if + // upgrading. For a system package we will try to find the available + // package that matches the user-specified system version (preferable for + // the configuration negotiation machinery) and, if fail, fallback to + // picking the latest one just to make sure the package is recognized. // - // See the collect lambda in collect_build_prerequisites() for details. + // But first check if this package is specified as an existing archive or + // directory. If that's the case, then only consider its (transient) + // available package instead of the above. // - using positions = small_vector<pair<size_t, size_t>, 1>; - using shadow_dependents_map = map<package_key, positions>; - - shadow_dependents_map shadow_cluster; + bool patch (false); + available_packages afs; - // Absent -- not negotiated yet, false -- being negotiated, true -- has - // been negotiated. - // - optional<bool> negotiated; + if (existing) + { + // By definition such a dependency has a version specified and may not + // be system. + // + assert (dvc && !dsys); - // The depth of the negotiating recursion (see collect_build_postponed() - // for details). - // - size_t depth = 0; + pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>> rp ( + find_existing (ddb, nm, *dvc)); - // Add dependencies of a new dependent. - // - postponed_configuration (size_t i, - package_key&& dependent, - bool existing, - pair<size_t, size_t> position, - packages&& deps) - : id (i) - { - add (move (dependent), existing, position, move (deps)); - } + // Must have been added to the existing packages registry. + // + assert (rp.first != nullptr); - // Add dependency of an existing dependent. - // - postponed_configuration (size_t i, - package_key&& dependent, - pair<size_t, size_t> position, - package_key&& dep) - : id (i) - { - add (move (dependent), - true /* existing */, - position, - packages ({move (dep)})); + afs.push_back (move (rp)); } - - // Add dependencies of a dependent. - // - // Note: adds the specified dependencies to the end of the configuration - // dependencies list suppressing duplicates. - // - void - add (package_key&& dependent, - bool existing, - pair<size_t, size_t> position, - packages&& deps) + else { - assert (position.first != 0 && position.second != 0); + optional<version_constraint> c; - add_dependencies (deps); // Don't move from since will be used later. - - auto i (dependents.find (dependent)); - - if (i != dependents.end ()) + if (!dvc) { - dependent_info& ddi (i->second); + assert (!dsys); // The version can't be empty for the system package. - ddi.add (dependency (position, move (deps))); + patch = (upgrade && !*upgrade); - // Conceptually, on the first glance, we can only move from existing - // to non-existing (e.g., due to a upgrade/downgrade later) and that - // case is handled via the version replacement rollback. However, - // after re-evaluation the existing dependent is handled similar to - // the new dependent and we can potentially up-negotiate the - // dependency configuration for it. - // - assert (ddi.existing || !existing); - } - else - { - small_vector<dependency, 1> ds ({dependency (position, move (deps))}); + if (patch) + { + c = patch_constraint (sp, ignore_unsatisfiable); - dependents.emplace (move (dependent), - dependent_info {existing, move (ds)}); + if (!c) + { + l5 ([&]{trace << *sp << db << ": non-patchable";}); + return no_change (); + } + } } - } + else if (!dsys || !wildcard (*dvc)) + c = dvc; - // Return true if any of the configuration's dependents depend on the - // specified package. - // - bool - contains_dependency (const package_key& d) const - { - return find (dependencies.begin (), dependencies.end (), d) != - dependencies.end (); - } - - // Return true if this configuration contains any of the specified - // dependencies. - // - bool - contains_dependency (const packages& ds) const - { - for (const package_key& d: ds) - { - if (contains_dependency (d)) - return true; - } + afs = find_available (nm, c, rfs); - return false; + if (afs.empty () && dsys && c) + afs = find_available (nm, nullopt, rfs); } - // Return true if this and specified configurations contain any common - // dependencies. + // In the deorphan mode check that the dependency is an orphan or was + // deorphaned on some previous refinement iteration. If that's not the + // case, then just disable the deorphan mode for this dependency and, if + // the version is not constrained and upgrade/patch is not requested, bail + // out indicating that no change is required. // - bool - contains_dependency (const postponed_configuration& c) const + // Note that in the move mode (dsp != sp) we deorphan the dependency in + // its destination configuration, if present. In the worst case scenario + // both the source and destination selected packages may need to be + // deorphaned since the source selected package may also stay if some + // dependents were not repointed to the new dependency (remember that the + // move mode is actually a copy mode). We, however, have no easy way to + // issue recommendations for both the old and the new dependencies at the + // moment. Given that in the common case the old dependency get dropped, + // let's keep it simple and do nothing about the old dependency and see + // how it goes. + // + const version* deorphaned (nullptr); + + if (deorphan) { - for (const auto& d: c.dependencies) - { - if (contains_dependency (d)) - return true; - } + bool orphan (dsp != nullptr && !dsp->system () && !dsys); - return false; - } + if (orphan) + { + auto i (deorphaned_deps.find (package_key (ddb, nm))); - // If the configuration contains the specified existing dependent, then - // return the earliest dependency position. Otherwise return NULL. - // - const pair<size_t, size_t>* - existing_dependent_position (const package_key& p) const - { - const pair<size_t, size_t>* r (nullptr); + if (i == deorphaned_deps.end ()) + orphan = orphan_package (ddb, dsp); + else + deorphaned = &i->second; + } - auto i (dependents.find (p)); - if (i != dependents.end () && i->second.existing) + if (!orphan) { - for (const dependency& d: i->second.dependencies) + if (!dvc && !upgrade) { - if (r == nullptr || d.position < *r) - r = &d.position; + l5 ([&]{trace << *sp << db << ": non-orphan";}); + return no_change (); } - assert (r != nullptr); + deorphan = false; } - - return r; } - // Notes: - // - // - Adds dependencies of the being merged from configuration to the end - // of the current configuration dependencies list suppressing - // duplicates. + // Go through up/down-grade candidates and pick the first one that + // satisfies all the dependents. In the deorphan mode if the package + // version is not constrained and upgrade/patch is not requested, then + // pick the version that matches the dependency version best (see the + // function description for details). Collect (and sort) unsatisfied + // dependents per the unsatisfiable version in case we need to print them. // - // - Doesn't change the negotiate member of this configuration. + // NOTE: don't forget to update the find_orphan_match() lambda and the + // try_replace_dependency() function if changing anything deorphan-related + // here. // - void - merge (postponed_configuration&& c) - { - assert (c.id != id); // Can't merge to itself. - - merged_ids.push_back (c.id); + using sp_set = set<config_selected_package>; - // Merge dependents. - // - for (auto& d: c.dependents) - { - auto i (dependents.find (d.first)); + vector<pair<version, sp_set>> unsatisfiable; - if (i != dependents.end ()) - { - dependent_info& ddi (i->second); // Destination dependent info. - dependent_info& sdi (d.second); // Source dependent info. + bool stub (false); - for (dependency& sd: sdi.dependencies) - ddi.add (move (sd)); + assert (!dsys || + (ddb.system_repository && + ddb.system_repository->find (nm) != nullptr)); - // As in add() above. - // - assert (ddi.existing || !sdi.existing); - } - else - dependents.emplace (d.first, move (d.second)); - } + // Version to deorphan (original orphan version). + // + const version* dov (deorphaned != nullptr ? deorphaned : + deorphan ? &dsp->version : + nullptr); - // Merge dependencies. - // - add_dependencies (move (c.dependencies)); + optional<version_constraint> dopc; // Patch constraint for the above. + optional<version_constraint> domc; // Minor constraint for the above. - // Pick the depth of the outermost negotiated configuration (minimum - // non-zero depth) between the two. - // - if (depth != 0) - { - if (c.depth != 0 && depth > c.depth) - depth = c.depth; - } - else - depth = c.depth; - } + bool orphan_best_match (deorphan && !dvc && !upgrade); - void - set_shadow_cluster (postponed_configuration&& c) + if (orphan_best_match) { - shadow_cluster.clear (); - - for (auto& dt: c.dependents) - { - positions ps; - for (auto& d: dt.second.dependencies) - ps.push_back (d.position); + // Note that non-zero iteration makes a version non-standard, so we + // reset it to 0 to produce the patch/minor constraints. + // + version v (dov->epoch, + dov->upstream, + dov->release, + dov->revision, + 0 /* iteration */); - shadow_cluster.emplace (dt.first, move (ps)); - } + dopc = patch_constraint (nm, v, true /* quiet */); + domc = minor_constraint (nm, v, true /* quiet */); } - bool - contains_in_shadow_cluster (package_key dependent, - pair<size_t, size_t> pos) const - { - auto i (shadow_cluster.find (dependent)); + using available = pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>>; - if (i != shadow_cluster.end ()) - { - const positions& ps (i->second); - return find (ps.begin (), ps.end (), pos) != ps.end (); - } - else - return false; - } + available deorphan_latest_iteration; + available deorphan_later_revision; + available deorphan_later_patch; + available deorphan_later_minor; + available deorphan_latest_available; - // Return the postponed configuration string representation in the form: - // - // {<dependent>[ <dependent>]* | <dependency>[ <dependency>]*}['!'|'?'] - // - // <dependent> = <package>['^'] - // <dependency> = <package>->{<dependent>/<position>[ <dependent>/<position>]*} - // - // The potential trailing '!' or '?' of the configuration representation - // indicates that the configuration is negotiated or is being negotiated, - // respectively. + // If the dependency is deorphaned to the same version as on the previous + // call, then return the "no change" result. Otherwise, return the + // deorphan result. // - // '^' character that may follow a dependent indicates that this is an - // existing dependent. - // - // <position> = <depends-index>','<alternative-index> - // - // <depends-index> and <alternative-index> are the 1-based serial numbers - // of the respective depends value and the dependency alternative in the - // dependent's manifest. - // - // See package_key for details on <package>. - // - // For example: - // - // {foo^ bar | libfoo->{foo/2,3 bar/1,1} libbar->{bar/1,1}}! - // - std::string - string () const + auto deorphan_result = [&sp, &db, + &ddb, &dsp, + dsys, + deorphaned, dov, + existing, + upgrade, + &no_change, + &trace] (available&& a, const char* what) { - std::string r; - - for (const auto& d: dependents) - { - r += r.empty () ? '{' : ' '; - r += d.first.string (); - - if (d.second.existing) - r += '^'; - } - - if (r.empty ()) - r += '{'; - - r += " |"; - - for (const package_key& d: dependencies) + if (deorphaned != nullptr && dsp->version == a.first->version) { - r += ' '; - r += d.string (); - r += "->{"; - - bool first (true); - for (const auto& dt: dependents) - { - for (const dependency& dp: dt.second.dependencies) - { - if (find (dp.begin (), dp.end (), d) != dp.end ()) - { - if (!first) - r += ' '; - else - first = false; - - r += dt.first.string (); - r += '/'; - r += to_string (dp.position.first); - r += ','; - r += to_string (dp.position.second); - } - } - } - - r += '}'; + l5 ([&]{trace << *sp << db << ": already deorphaned";}); + return no_change (); } - r += '}'; + l5 ([&]{trace << *sp << db << ": deorphan to " << what << ' ' + << package_string (sp->name, a.first->version) + << ddb;}); - if (negotiated) - r += *negotiated ? '!' : '?'; + return evaluate_result { + ddb, move (a.first), move (a.second), + false /* unused */, + dsys, + existing, + upgrade, + *dov}; + }; - return r; - } + auto build_result = [&ddb, dsys, existing, upgrade] (available&& a) + { + return evaluate_result { + ddb, move (a.first), move (a.second), + false /* unused */, + dsys, + existing, + upgrade, + nullopt /* orphan */}; + }; - private: - // Add the specified packages to the end of the dependencies list - // suppressing duplicates. + // Note that if the selected dependency is the best that we can get, we + // normally issue the "no change" recommendation. However, if the + // configuration variables are specified for this dependency on the + // command line, then we issue the "reconfigure" recommendation instead. // - void - add_dependencies (packages&& deps) + // Return true, if the already selected dependency has been specified on + // the command line with the configuration variables, but has not yet been + // built on this pkg-build run. + // + auto reconfigure = [&ddb, &dsp, &nm, dsys, &pkgs] () { - for (auto& d: deps) - { - if (find (dependencies.begin (), dependencies.end (), d) == - dependencies.end ()) - dependencies.push_back (move (d)); - } - } + assert (dsp != nullptr); - void - add_dependencies (const packages& deps) - { - for (const auto& d: deps) + if (!dsys) { - if (find (dependencies.begin (), dependencies.end (), d) == - dependencies.end ()) - dependencies.push_back (d); + const build_package* p (pkgs.entered_build (ddb, nm)); + return p != nullptr && !p->action && !p->config_vars.empty (); } - } - }; - - static ostream& - operator<< (ostream& os, const postponed_configuration& c) - { - return os << c.string (); - } + else + return false; + }; - // Note that we could be adding new/merging existing entries while - // processing an entry. Thus we use a list. - // - class postponed_configurations: public forward_list<postponed_configuration> - { - public: - // Return the configuration the dependent is added to (after all the - // potential configuration merges, etc). - // - // Also return in second absent if the merge happened due to the shadow - // cluster logic (in which case the cluster was/is being negotiated), - // false if any non-negotiated or being negotiated clusters has been - // merged in, and true otherwise. - // - // If some configurations needs to be merged and this involves the (being) - // negotiated configurations, then merge into the outermost-depth - // negotiated configuration (with minimum non-zero depth). - // - pair<postponed_configuration&, optional<bool>> - add (package_key dependent, - bool existing, - pair<size_t, size_t> position, - postponed_configuration::packages dependencies) + for (available& af: afs) { - tracer trace ("postponed_configurations::add"); - - assert (!dependencies.empty ()); - - // The plan is to first go through the existing clusters and check if - // any of them contain this dependent/dependencies in their shadow - // clusters. If such a cluster is found, then force-add them to - // it. Otherwise, if any dependency-intersecting clusters are present, - // then add the specified dependent/dependencies to the one with the - // minimum non-zero depth, if any, and to the first one otherwise. - // Otherwise, add the new cluster. Afterwards, merge into the resulting - // cluster other dependency-intersecting clusters. Note that in case of - // shadow, this should normally not happen because such a cluster should - // have been either pre-merged or its dependents should be in the - // cluster. But it feels like it may still happen if things change, in - // which case we will throw again (admittedly a bit fuzzy). - // - iterator ri; - bool rb (true); - - // Note that if a single dependency is added, then it can only belong to - // a single existing cluster and so no clusters merge can happen, unless - // we are force-adding. In the later case we can only merge once for a - // single dependency. - // - // Let's optimize for the common case based on these facts. - // - bool single (dependencies.size () == 1); + shared_ptr<available_package>& ap (af.first); + const version& av (!dsys ? ap->version : *ap->system_version (ddb)); - // Merge dependency-intersecting clusters in the specified range into - // the resulting cluster and reset change rb to false if any of the - // merged in clusters is non-negotiated or is being negotiated. + // If we aim to upgrade to the latest version and it tends to be less + // then the selected one, then what we currently have is the best that + // we can get, and so we return the "no change" result, unless we are + // deorphaning. // - // The iterator arguments refer to entries before and after the range - // endpoints, respectively. + // Note that we also handle a package stub here. // - auto merge = [&trace, &ri, &rb, single, this] (iterator i, - iterator e, - bool shadow_based) + if (!dvc && dsp != nullptr && av < dsp->version) { - postponed_configuration& rc (*ri); - - iterator j (i); + assert (!dsys); // Version can't be empty for the system package. - // Merge the intersecting configurations. + // For the selected system package we still need to pick a source + // package version to downgrade to. // - bool merged (false); - for (++i; i != e; ++i) + if (!dsp->system () && !deorphan) { - postponed_configuration& c (*i); - - if (c.contains_dependency (rc)) + if (reconfigure ()) { - if (!c.negotiated || !*c.negotiated) - rb = false; - - l5 ([&]{trace << "merge " << c << " into " << rc;}); - - assert (!shadow_based || (c.negotiated && *c.negotiated)); - - rc.merge (move (c)); - c.dependencies.clear (); // Mark as merged from (see above). - - merged = true; - - if (single) - break; + l5 ([&]{trace << *dsp << ddb << ": reconfigure (best)";}); + return build_result (find_available_fragment (o, ddb, dsp)); + } + else + { + l5 ([&]{trace << *dsp << ddb << ": best";}); + return no_change (); } } - // Erase configurations which we have merged from. + // We can not upgrade the package to a stub version, so just skip it. // - if (merged) + if (ap->stub ()) { - i = j; - - for (++i; i != e; ) - { - if (!i->dependencies.empty ()) - { - ++i; - ++j; - } - else - i = erase_after (j); - } + stub = true; + continue; } - }; + } + + // Check if the version satisfies all the dependents and collect + // unsatisfied ones. + // + bool satisfactory (true); + sp_set unsatisfied_dependents; - auto trace_add = [&trace, &dependent, existing, position, &dependencies] - (const postponed_configuration& c, bool shadow) + for (const auto& dp: dpt_constrs) { - if (verb >= 5) + if (!satisfies (av, dp.constraint)) { - diag_record dr (trace); - dr << "add {" << dependent; - - if (existing) - dr << '^'; - - dr << ' ' << position.first << ',' << position.second << ':'; - - for (const auto& d: dependencies) - dr << ' ' << d; + satisfactory = false; - dr << "} to " << c; + // Continue to collect dependents of the unsatisfiable version if + // we need to print them before failing. + // + if (ignore_unsatisfiable) + break; - if (shadow) - dr << " (shadow cluster-based)"; + unsatisfied_dependents.emplace (dp.db, dp.package); } - }; + } - // Try to add based on the shadow cluster. - // + if (!satisfactory) { - auto i (begin ()); - for (; i != end (); ++i) - { - postponed_configuration& c (*i); + if (!ignore_unsatisfiable) + unsatisfiable.emplace_back (av, move (unsatisfied_dependents)); - if (c.contains_in_shadow_cluster (dependent, position)) - { - trace_add (c, true /* shadow */); + // If the dependency is expected to be configured as system, then bail + // out, as an available package version will always resolve to the + // system one (see above). + // + if (dsys) + break; - c.add (move (dependent), existing, position, move (dependencies)); - break; - } - } + continue; + } - if (i != end ()) - { - // Note that the cluster with a shadow cluster is by definition - // either being negotiated or has been negotiated. Actually, there - // is also a special case when we didn't negotiate the configuration - // yet and are in the process of re-evaluating existing dependents. - // Note though, that in this case we have already got the try/catch - // frame corresponding to the cluster negotiation (see - // collect_build_postponed() for details). - // - assert (i->depth != 0); + if (orphan_best_match) + { + // If the exact orphan version is encountered, then we are done. + // + if (av == *dov) + return deorphan_result (move (af), "exactly same version"); - ri = i; + // If the available package is of the same revision as orphan but a + // different iteration, then save it as the latest iteration of same + // orphan version and revision. + // + if (deorphan_latest_iteration.first == nullptr && + av.compare (*dov, false /* revision */, true /* iteration */) == 0) + deorphan_latest_iteration = af; - merge (before_begin (), ri, true /* shadow_based */); - merge (ri, end (), true /* shadow_based */); + // If the available package is of the same version as orphan and its + // revision is greater, then save it as the later revision of same + // version. + // + if (deorphan_later_revision.first == nullptr && + av.compare (*dov, true /* revision */) == 0 && + av.compare (*dov, false /* revision */, true /* iteration */) > 0) + deorphan_later_revision = af; - return make_pair (ref (*ri), optional<bool> ()); - } - } + // If the available package is of the same minor version as orphan but + // of the greater patch version, then save it as the later patch of + // same version. + // + if (deorphan_later_patch.first == nullptr && + dopc && satisfies (av, *dopc) && + av.compare (*dov, true /* revision */) > 0) // Patch is greater? + deorphan_later_patch = af; - // Find the cluster to add the dependent/dependencies to. - // - optional<size_t> depth; + // If the available package is of the same major version as orphan but + // of the greater minor version, then save it as the later minor of + // same version. + // + // Note that we can't easily compare minor versions here since these + // are bpkg version objects. Thus, we consider that this is a greater + // minor version if the version is greater (ignoring revisions) and + // the latest patch is not yet saved. + // + if (deorphan_later_minor.first == nullptr && + domc && satisfies (av, *domc) && + av.compare (*dov, true /* revision */) > 0 && + deorphan_later_patch.first == nullptr) + deorphan_later_minor = af; - auto j (before_begin ()); // Precedes iterator i. - for (auto i (begin ()); i != end (); ++i, ++j) - { - postponed_configuration& c (*i); + // Save the latest available package version. + // + if (deorphan_latest_available.first == nullptr) + deorphan_latest_available = move (af); - if (c.contains_dependency (dependencies) && - (!depth || (c.depth != 0 && (*depth == 0 || *depth > c.depth)))) + // If the available package version is less then the orphan revision + // then we can bail out from the loop, since all the versions from the + // preference list have already been encountered, if present. + // + if (av.compare (*dov, false /* revision */, true /* iteration */) < 0) { - ri = i; - depth = c.depth; + assert (deorphan_latest_iteration.first != nullptr || + deorphan_later_revision.first != nullptr || + deorphan_later_patch.first != nullptr || + deorphan_later_minor.first != nullptr || + deorphan_latest_available.first != nullptr); + break; } } - - if (!depth) // No intersecting cluster? - { - // New cluster. Insert after the last element. - // - ri = insert_after (j, - postponed_configuration ( - next_id_++, - move (dependent), - existing, - position, - move (dependencies))); - - l5 ([&]{trace << "create " << *ri;}); - } else { - // Add the dependent/dependencies into an existing cluster. + // In the up/downgrade+deorphan mode always replace the dependency, + // re-fetching it from an existing repository if the version stays the + // same. // - postponed_configuration& c (*ri); - - trace_add (c, false /* shadow */); - - c.add (move (dependent), existing, position, move (dependencies)); + if (deorphan) + return deorphan_result (move (af), "constrained version"); - // Try to merge other clusters into this cluster. + // For the regular up/downgrade if the best satisfactory version and + // the desired system flag perfectly match the ones of the selected + // package, then no package change is required. Otherwise, recommend + // an upgrade/downgrade/replacement. // - merge (before_begin (), ri, false /* shadow_based */); - merge (ri, end (), false /* shadow_based */); - } + // Note: we need to be careful here not to start yo-yo'ing for a + // dependency being built as an existing archive or directory. For + // such a dependency we need to return the "no change" recommendation + // if any version recommendation (which may not change) has already + // been returned. + // + if (dsp != nullptr && + av == dsp->version && + dsp->system () == dsys && + (!existing || + find (existing_deps.begin (), existing_deps.end (), + package_key (ddb, nm)) != existing_deps.end ())) + { + if (reconfigure ()) + { + l5 ([&]{trace << *dsp << ddb << ": reconfigure";}); + return build_result (move (af)); + } + else + { + l5 ([&]{trace << *dsp << ddb << ": unchanged";}); + return no_change (); + } + } + else + { + l5 ([&]{trace << *sp << db << ": update to " + << package_string (nm, av, dsys) << ddb;}); - return make_pair (ref (*ri), optional<bool> (rb)); + return build_result (move (af)); + } + } } - // Add new postponed configuration cluster with a single dependency of an - // existing dependent. - // - // Note that it's the caller's responsibility to make sure that the - // dependency doesn't already belong to any existing cluster. - // - void - add (package_key dependent, - pair<size_t, size_t> position, - package_key dependency) + if (orphan_best_match) { - tracer trace ("postponed_configurations::add"); + if (deorphan_latest_iteration.first != nullptr) + return deorphan_result (move (deorphan_latest_iteration), + "latest iteration"); - // Add the new cluster to the end of the list which we can only find by - // traversing the list. While at it, make sure that the dependency - // doesn't belong to any existing cluster. - // - auto i (before_begin ()); // Insert after this element. - - for (auto j (begin ()); j != end (); ++i, ++j) - assert (!j->contains_dependency (dependency)); + if (deorphan_later_revision.first != nullptr) + return deorphan_result (move (deorphan_later_revision), + "later revision"); - i = insert_after (i, - postponed_configuration (next_id_++, - move (dependent), - position, - move (dependency))); - - l5 ([&]{trace << "create " << *i;}); - } + if (deorphan_later_patch.first != nullptr) + return deorphan_result (move (deorphan_later_patch), "later patch"); - postponed_configuration* - find (size_t id) - { - for (postponed_configuration& cfg: *this) - { - if (cfg.id == id) - return &cfg; - } + if (deorphan_later_minor.first != nullptr) + return deorphan_result (move (deorphan_later_minor), "later minor"); - return nullptr; + if (deorphan_latest_available.first != nullptr) + return deorphan_result (move (deorphan_latest_available), + "latest available"); } - // Return address of the cluster the dependency belongs to and NULL if it - // doesn't belong to any cluster. + // If we aim to upgrade to the latest version, then what we currently have + // is the only thing that we can get, and so returning the "no change" + // result, unless we need to upgrade a package configured as system or to + // deorphan. // - const postponed_configuration* - find_dependency (const package_key& d) const + if (!dvc && dsp != nullptr && !dsp->system () && !deorphan) { - for (const postponed_configuration& cfg: *this) + assert (!dsys); // Version cannot be empty for the system package. + + if (reconfigure ()) { - if (cfg.contains_dependency (d)) - return &cfg; + l5 ([&]{trace << *dsp << ddb << ": reconfigure (only)";}); + return build_result (find_available_fragment (o, ddb, dsp)); } - - return nullptr; - } - - // Return true if all the configurations have been negotiated. - // - bool - negotiated () const - { - for (const postponed_configuration& cfg: *this) + else { - if (!cfg.negotiated || !*cfg.negotiated) - return false; + l5 ([&]{trace << *dsp << ddb << ": only";}); + return no_change (); } - - return true; } - // Translate index to iterator and return the referenced configuration. + // If the version satisfying the desired dependency version constraint is + // unavailable or unsatisfiable for some dependents then we fail, unless + // requested not to do so. In the latter case we return the "no change" + // result. // - postponed_configuration& - operator[] (size_t index) + if (ignore_unsatisfiable) { - auto i (begin ()); - for (size_t j (0); j != index; ++j, ++i) assert (i != end ()); + l5 ([&]{trace << package_string (nm, dvc, dsys) << ddb + << (unsatisfiable.empty () + ? ": no source" + : ": unsatisfiable");}); - assert (i != end ()); - return *i; + return no_change (); } - size_t - size () const + // If there are no unsatisfiable versions then the package is not present + // (or is not available in source) in its dependents' repositories. + // + if (unsatisfiable.empty ()) { - size_t r (0); - for (auto i (begin ()); i != end (); ++i, ++r) ; - return r; - } - - private: - size_t next_id_ = 1; - }; - - // The following exceptions are used by the dependency configuration - // up-negotiation/refinement machinery. See the collect lambda in - // collect_build_prerequisites() for details. - // - struct retry_configuration - { - size_t depth; - package_key dependent; - }; - - struct merge_configuration - { - size_t depth; - }; - - // Packages with postponed prerequisites collection, for one of the - // following reasons: - // - // - Postponed due to the inability to find a version satisfying the pre- - // entered constraint from repositories available to this package. The - // idea is that this constraint could still be satisfied from a repository - // fragment of some other package (that we haven't processed yet) that - // also depends on this prerequisite. - // - // - Postponed due to the inability to choose between two dependency - // alternatives, both having dependency packages which are not yet - // selected in the configuration nor being built. The idea is that this - // ambiguity could still be resolved after some of those dependency - // packages get built via some other dependents. - // - using postponed_packages = set<build_package*>; - - // Map of dependency packages whose recursive processing should be postponed - // because they have dependents with configuration clauses. - // - // Note that dependents of such a package that don't have any configuration - // clauses are processed right away (since the negotiated configuration may - // not affect them) while those that do are postponed in the same way as - // those with dependency alternatives (see above). - // - // Note that the latter kind of dependent is what eventually causes - // recursive processing of the dependency packages. Which means we must - // watch out for bogus entries in this map which we may still end up with - // (e.g., because postponement caused cross-talk between dependency - // alternatives). Thus we keep flags that indicate whether we have seen each - // type of dependent and then just process dependencies that have the first - // (without config) but not the second (with config). We also need to track - // at which phase of collection an entry has been added to process the bogus - // entries accordingly. - // - struct postponed_dependency - { - bool wout_config; // Has dependent without config. - bool with_config; // Has dependent with config. - bool initial_collection; + diag_record dr (fail); - postponed_dependency (bool woc, bool wic, bool ic) - : wout_config (woc), - with_config (wic), - initial_collection (ic) {} + if (patch) + { + // Otherwise, we should have bailed out earlier returning "no change" + // (see above). + // + assert (dsp != nullptr && (dsp->system () || deorphan)); - bool - bogus () const {return wout_config && !with_config;} - }; + // Patch (as any upgrade) of a system package is always explicit, so + // we always fail and never treat the package as being up to date. + // + assert (explicitly); - class postponed_dependencies: public map<package_key, postponed_dependency> - { - public: - bool - has_bogus () const - { - for (const auto& pd: *this) + fail << "patch version for " << *sp << db << " is not available " + << "from its dependents' repositories"; + } + else if (!stub) + fail << package_string (nm, dsys ? nullopt : dvc) << ddb + << " is not available from its dependents' repositories"; + else // The only available package is a stub. { - if (pd.second.bogus ()) - return true; + // Otherwise, we should have bailed out earlier, returning "no change" + // rather then setting the stub flag to true (see above). + // + assert (!dvc && !dsys && dsp != nullptr && (dsp->system () || deorphan)); + + fail << package_string (nm, dvc) << ddb << " is not available in " + << "source from its dependents' repositories"; } - return false; } - // Erase the bogus postponements and throw cancel_postponement, if any. + // Issue the diagnostics and fail. // - struct cancel_postponement: scratch_collection - { - cancel_postponement () - : scratch_collection ( - "bogus dependency collection postponement cancellation") {} - }; + diag_record dr (fail); + dr << "package " << nm << ddb << " doesn't satisfy its dependents"; - void - cancel_bogus (tracer& trace, bool initial_collection) + // Print the list of unsatisfiable versions together with dependents they + // don't satisfy: up to three latest versions with no more than five + // dependents each. + // + size_t nv (0); + for (const auto& u: unsatisfiable) { - bool bogus (false); - for (auto i (begin ()); i != end (); ) - { - const postponed_dependency& d (i->second); + dr << info << package_string (nm, u.first) << " doesn't satisfy"; - if (d.bogus () && (!initial_collection || d.initial_collection)) - { - bogus = true; + const sp_set& ps (u.second); - l5 ([&]{trace << "erase bogus postponement " << i->first;}); + size_t i (0), n (ps.size ()); + for (auto p (ps.begin ()); i != n; ++p) + { + // It would probably be nice to also print the unsatisfied constraint + // here, but let's keep it simple for now. + // + dr << (i == 0 ? " " : ", ") << *p->package << p->db; - i = erase (i); - } - else - ++i; + if (++i == 5 && n != 6) // Printing 'and 1 more' looks stupid. + break; } - if (bogus) - { - l5 ([&]{trace << "bogus postponements erased, throwing";}); - throw cancel_postponement (); - } + if (i != n) + dr << " and " << n - i << " more"; + + if (++nv == 3 && unsatisfiable.size () != 4) + break; } - }; - // Map of existing dependents which may not be re-evaluated to a position - // with the dependency index greater than the specified one. - // - // This mechanism applies when we re-evaluate an existing dependent to a - // certain position but later realize we've gone too far. In this case we - // note the earlier position information and re-collect from scratch. On the - // re-collection any re-evaluation of the dependent to a greater position - // will be either skipped or performed but to this earlier position (see the - // replace member for details). - // - // We consider the postponement bogus if some dependent re-evaluation was - // skipped due to its presence but no re-evaluation to this (or earlier) - // dependency index was performed. Thus, if after the collection of packages - // some bogus entries are present in the map, then it means that we have - // skipped the respective re-evaluations erroneously and so need to erase - // these entries and re-collect. - // - // Note that if no re-evaluation is skipped due to a postponement then it - // is harmless and we don't consider it bogus. + if (nv != unsatisfiable.size ()) + dr << info << "and " << unsatisfiable.size () - nv << " more"; + + dr << endf; + } + + // List of dependent packages whose immediate/recursive dependencies must be + // upgraded and/or deorphaned (specified with -i/-r on the command line). // - struct postponed_position: pair<size_t, size_t> + struct recursive_package { - // True if the "later" position should be replaced rather than merely - // skipped. The replacement deals with the case where the "earlier" - // position is encountered while processing the same cluster as what - // contains the later position. In this case, if we merely skip, then we - // will never naturally encounter the earlier position. So we have to - // force the issue (even if things change enough for us to never see the - // later position again). - // - bool replace; + database& db; + package_name name; - // Re-evaluation was skipped due to this postponement. + // Recursive/immediate upgrade/patch. Note the upgrade member is only + // meaningful if recursive is present. // - bool skipped = false; + optional<bool> recursive; // true -- recursive, false -- immediate. + bool upgrade; // true -- upgrade, false -- patch. - // The dependent was re-evaluated. Note that it can be only re-evaluated - // to this or earlier dependency index. + // Recursive/immediate deorphaning. // - bool reevaluated = false; - - postponed_position (pair<size_t, size_t> p, bool r) - : pair<size_t, size_t> (p), replace (r) {} + optional<bool> deorphan; // true -- recursive, false -- immediate. }; + using recursive_packages = vector<recursive_package>; - class postponed_positions: public map<package_key, postponed_position> + // Recursively check if immediate dependencies of this dependent must be + // upgraded or patched and/or deorphaned. + // + // Cache the results of this function calls to avoid multiple traversals of + // the same dependency graphs. + // + struct upgrade_dependencies_key { - public: - // If true, override the first encountered non-replace position to replace - // and clear this flag. See collect_build_postponed() for details. - // - bool replace = false; - - // Erase the bogus postponements and throw cancel_postponement, if any. - // - struct cancel_postponement: scratch_collection - { - cancel_postponement () - : scratch_collection ("bogus existing dependent re-evaluation " - "postponement cancellation") {} - }; + package_key dependent; + bool recursion; - void - cancel_bogus (tracer& trace) + bool + operator< (const upgrade_dependencies_key& v) const { - bool bogus (false); - for (auto i (begin ()); i != end (); ) - { - const postponed_position& p (i->second); + if (recursion != v.recursion) + return recursion < v.recursion; - if (p.skipped && !p.reevaluated) - { - bogus = true; - - l5 ([&]{trace << "erase bogus existing dependent " << i->first - << " re-evaluation postponement with dependency index " - << i->second.first;}); - - // It seems that the replacement may never be bogus. - // - assert (!p.replace); - - i = erase (i); - } - else - ++i; - } - - if (bogus) - { - l5 ([&]{trace << "bogus re-evaluation postponement erased, throwing";}); - throw cancel_postponement (); - } + return dependent < v.dependent; } }; - struct build_packages: build_package_list + struct upgrade_deorphan { - build_packages () = default; + optional<bool> upgrade; // true -- upgrade, false -- patch. + bool deorphan; + }; - // Copy-constructible and move-assignable (used for snapshoting). + using upgrade_dependencies_cache = map<upgrade_dependencies_key, + upgrade_deorphan>; + + static upgrade_deorphan + upgrade_dependencies (database& db, + const package_name& nm, + const recursive_packages& rs, + upgrade_dependencies_cache& cache, + bool recursion = false) + { + // If the result of the upgrade_dependencies() call for these dependent + // and recursion flag value is cached, then return that. Otherwise, cache + // the calculated result prior to returning it to the caller. // - build_packages (const build_packages& v) - : build_package_list () + upgrade_dependencies_key k {package_key (db, nm), recursion}; { - // Copy the map. - // - for (const auto& p: v.map_) - map_.emplace (p.first, data_type {end (), p.second.package}); + auto i (cache.find (k)); - // Copy the list. - // - for (const auto& p: v) - { - auto i (map_.find (p.get ().db, p.get ().name ())); - assert (i != map_.end ()); - i->second.position = insert (end (), i->second.package); - } + if (i != cache.end ()) + return i->second; } - build_packages (build_packages&&) = delete; + auto i (find_if (rs.begin (), rs.end (), + [&nm, &db] (const recursive_package& i) -> bool + { + return i.name == nm && i.db == db; + })); - build_packages& operator= (const build_packages&) = delete; + upgrade_deorphan r {nullopt /* upgrade */, false /* deorphan */}; - build_packages& - operator= (build_packages&& v) + if (i != rs.end ()) { - clear (); - - // Move the map. - // - // Similar to what we do in the copy-constructor, but here we also need - // to restore the database reference and the package shared pointers in - // the source entry after the move. This way we can obtain the source - // packages databases and names later while copying the list. - // - for (auto& p: v.map_) - { - build_package& bp (p.second.package); - - database& db (bp.db); - shared_ptr<selected_package> sp (bp.selected); - shared_ptr<available_package> ap (bp.available); + if (i->recursive && *i->recursive >= recursion) + r.upgrade = i->upgrade; - map_.emplace (p.first, data_type {end (), move (bp)}); + if (i->deorphan && *i->deorphan >= recursion) + r.deorphan = true; - bp.db = db; - bp.selected = move (sp); - bp.available = move (ap); - } - - // Copy the list. + // If we both upgrade and deorphan, then we can bail out since the value + // may not change any further (upgrade wins over patch and deorphaning + // can't be canceled). // - for (const auto& p: v) + if (r.upgrade && *r.upgrade && r.deorphan) { - auto i (map_.find (p.get ().db, p.get ().name ())); - assert (i != map_.end ()); - i->second.position = insert (end (), i->second.package); + cache[move (k)] = r; + return r; } - - return *this; - } - - // Pre-enter a build_package without an action. No entry for this package - // may already exists. - // - void - enter (package_name name, build_package pkg) - { - assert (!pkg.action); - - database& db (pkg.db); // Save before the move() call. - auto p (map_.emplace (package_key {db, move (name)}, - data_type {end (), move (pkg)})); - - assert (p.second); - } - - // Return the package pointer if it is already in the map and NULL - // otherwise (so can be used as bool). - // - build_package* - entered_build (database& db, const package_name& name) - { - auto i (map_.find (db, name)); - return i != map_.end () ? &i->second.package : nullptr; - } - - build_package* - entered_build (const package_key& p) - { - return entered_build (p.db, p.name); } - // Collect the package being built. Return its pointer if this package - // version was, in fact, added to the map and NULL if it was already there - // and the existing version was preferred or if the package build has been - // replaced with the drop. So can be used as bool. - // - // Consult replaced_vers for an existing version replacement entry and - // follow it, if present, potentially collecting the package drop - // instead. Add entry to replaced_vers and throw replace_version if the - // existing version needs to be replaced but the new version cannot be - // re-collected recursively in-place (see replaced_versions for details). - // Also add an entry and throw if the existing dependent needs to be - // replaced. - // - // Optionally, pass the function which verifies the chosen package - // version. It is called before replace_version is potentially thrown or - // the recursive collection is performed. The scratch argument is true if - // the package version needs to be replaced but in-place replacement is - // not possible (see replaced_versions for details). - // - // Also, in the recursive mode (dep_chain is not NULL): - // - // - Use the custom search function to find the package dependency - // databases. - // - // - For the repointed dependents collect the prerequisite replacements - // rather than prerequisites being replaced. - // - // - Add paths of the created private configurations, together with the - // containing configuration databases, into the specified list (see - // private_configs for details). - // - // Note that postponed_* and dep_chain arguments must all be either - // specified or not. - // - struct replace_version: scratch_collection - { - replace_version () - : scratch_collection ("package version replacement") {} - }; - - using verify_package_build_function = void (const build_package&, - bool scratch); - - build_package* - collect_build (const pkg_build_options& options, - build_package pkg, - const function<find_database_function>& fdb, - const repointed_dependents& rpt_depts, - const function<add_priv_cfg_function>& apc, - bool initial_collection, - replaced_versions& replaced_vers, - postponed_configurations& postponed_cfgs, - build_package_refs* dep_chain = nullptr, - postponed_packages* postponed_repo = nullptr, - postponed_packages* postponed_alts = nullptr, - postponed_dependencies* postponed_deps = nullptr, - postponed_positions* postponed_poss = nullptr, - const function<verify_package_build_function>& vpb = nullptr) + for (database& ddb: db.dependent_configs ()) { - using std::swap; // ...and not list::swap(). - - tracer trace ("collect_build"); - - // See the above notes. - // - bool recursive (dep_chain != nullptr); - assert ((postponed_repo != nullptr) == recursive && - (postponed_alts != nullptr) == recursive && - (postponed_deps != nullptr) == recursive && - (postponed_poss != nullptr) == recursive); - - // Only builds are allowed here. - // - assert (pkg.action && *pkg.action == build_package::build && - pkg.available != nullptr); - - package_key pk (pkg.db, pkg.available->id.name); - - // Apply the version replacement, if requested, and indicate that it was - // applied. - // - auto vi (replaced_vers.find (pk)); - - if (vi != replaced_vers.end () && !vi->second.replaced) - { - l5 ([&]{trace << "apply version replacement for " - << pkg.available_name_version_db ();}); - - replaced_version& v (vi->second); - - v.replaced = true; - - if (v.available != nullptr) - { - pkg.available = v.available; - pkg.repository_fragment = v.repository_fragment; - pkg.system = v.system; - - l5 ([&]{trace << "replacement: " - << pkg.available_name_version_db ();}); - } - else - { - l5 ([&]{trace << "replacement: drop";}); - - assert (pkg.selected != nullptr); - - collect_drop (options, pkg.db, pkg.selected, replaced_vers); - return nullptr; - } - } - - // Add the version replacement entry, call the verification function if - // specified, and throw replace_version. - // - auto replace_ver = [&pk, &vpb, &vi, &replaced_vers] - (const build_package& p) - { - replaced_version rv (p.available, p.repository_fragment, p.system); - - if (vi != replaced_vers.end ()) - vi->second = move (rv); - else - replaced_vers.emplace (move (pk), move (rv)); - - if (vpb) - vpb (p, true /* scratch */); - - throw replace_version (); - }; - - auto i (map_.find (pk)); - - // If we already have an entry for this package name, then we have to - // pick one over the other. - // - // If the existing entry is a drop, then we override it. If the existing - // entry is a pre-entered or is non-build one, then we merge it into the - // new build entry. Otherwise (both are builds), we pick one and merge - // the other into it. - // - if (i != map_.end ()) + for (auto& pd: query_dependents_cache (ddb, nm, db)) { - build_package& bp (i->second.package); - - // Note that we used to think that the scenario when the build could - // replace drop could never happen since we would start collecting - // from scratch. This has changed when we introduced replaced_versions - // for collecting drops. + // Note that we cannot end up with an infinite recursion for + // configured packages due to a dependency cycle (see order() for + // details). // - if (bp.action && *bp.action == build_package::drop) // Drop. - { - bp = move (pkg); - } - else if (!bp.action || *bp.action != build_package::build) // Non-build. - { - pkg.merge (move (bp)); - bp = move (pkg); - } - else // Build. - { - // At the end we want p1 to point to the object that we keep - // and p2 to the object that we merge from. - // - build_package* p1 (&bp); - build_package* p2 (&pkg); - - // Pick with the following preference order: user selection over - // implicit one, source package over a system one, newer version - // over an older one. So get the preferred into p1 and the other - // into p2. - // - { - int us (p1->user_selection () - p2->user_selection ()); - int sf (p1->system - p2->system); - - if (us < 0 || - (us == 0 && sf > 0) || - (us == 0 && - sf == 0 && - p2->available_version () > p1->available_version ())) - swap (p1, p2); - } + upgrade_deorphan ud ( + upgrade_dependencies (ddb, pd.name, rs, cache, true /* recursion */)); - // If the versions differ, pick the satisfactory one and if both are - // satisfactory, then keep the preferred. + if (ud.upgrade || ud.deorphan) + { + // Upgrade wins over patch. // - if (p1->available_version () != p2->available_version ()) - { - using constraint_type = build_package::constraint_type; + if (ud.upgrade && (!r.upgrade || *r.upgrade < *ud.upgrade)) + r.upgrade = *ud.upgrade; - // See if pv's version satisfies pc's constraints. Return the - // pointer to the unsatisfied constraint or NULL if all are - // satisfied. - // - auto test = [] (build_package* pv, - build_package* pc) -> const constraint_type* - { - for (const constraint_type& c: pc->constraints) - { - if (!satisfies (pv->available_version (), c.value)) - return &c; - } - - return nullptr; - }; - - // First see if p1 satisfies p2's constraints. - // - if (auto c2 = test (p1, p2)) - { - // If not, try the other way around. - // - if (auto c1 = test (p2, p1)) - { - const package_name& n (i->first.name); - const string& d1 (c1->dependent); - const string& d2 (c2->dependent); - - fail << "unable to satisfy constraints on package " << n << - info << d1 << c1->db << " depends on (" << n << " " - << c1->value << ")" << - info << d2 << c2->db << " depends on (" << n << " " - << c2->value << ")" << - info << "available " << p1->available_name_version () << - info << "available " << p2->available_name_version () << - info << "explicitly specify " << n << " version to manually " - << "satisfy both constraints"; - } - else - swap (p1, p2); - } - - l4 ([&]{trace << "pick " << p1->available_name_version_db () - << " over " << p2->available_name_version_db ();}); - } + if (ud.deorphan) + r.deorphan = true; - // See if we are replacing the object. If not, then we don't need to - // collect its prerequisites since that should have already been - // done. Remember, p1 points to the object we want to keep. + // If we both upgrade and deorphan, then we can bail out (see above + // for details). // - bool replace (p1 != &i->second.package); - - if (replace) + if (r.upgrade && *r.upgrade && r.deorphan) { - swap (*p1, *p2); - swap (p1, p2); // Setup for merge below. - } - - p1->merge (move (*p2)); - - if (replace) - { - if (p1->available_version () != p2->available_version () || - p1->system != p2->system) - { - // See if in-place replacement is possible (no dependencies, - // etc) and set scratch to false if that's the case. - // - // Firstly, such a package should not participate in any - // configuration negotiation. - // - // Other than that, it looks like the only optimization we can - // do easily is if the package has no dependencies (and thus - // cannot impose any constraints). Anything more advanced would - // require analyzing our dependencies (which we currently cannot - // easily get) and (1) either dropping the dependency - // build_package altogether if we are the only dependent (so - // that it doesn't influence any subsequent dependent) or (2) - // making sure our constraint is a sub-constraint of any other - // constraint and removing it from the dependency build_package. - // Maybe/later. - // - // NOTE: remember to update collect_drop() if changing anything - // here. - // - bool scratch (true); - - // While checking if the package has any dependencies skip the - // toolchain build-time dependencies since they should be quite - // common. - // - if (!has_dependencies (options, p2->available->dependencies)) - scratch = false; - - l5 ([&]{trace << p2->available_name_version_db () - << " package version needs to be replaced " - << (!scratch ? "in-place " : "") << "with " - << p1->available_name_version_db ();}); - - if (scratch) - replace_ver (*p1); - } - else - { - // It doesn't seem possible that replacing the build object - // without changing the package version may result in changing - // the package configuration since the configuration always gets - // into the initial package build entry (potentially - // pre-entered, etc). If it wouldn't be true then we would also - // need to add the replacement version entry and re-collect from - // scratch. - } + cache[move (k)] = r; + return r; } - else - return nullptr; } } - else - { - // Treat the replacement of the existing dependent that is - // participating in the configuration negotiation also as a version - // replacement. This way we will not be treating the dependent as an - // existing on the re-collection (see query_existing_dependents() for - // details). - // - // Note: an existing dependent may not be configured as system. - // - if (pkg.selected != nullptr && - (pkg.selected->version != pkg.available_version () || - pkg.system)) - { - - for (const postponed_configuration& cfg: postponed_cfgs) - { - auto i (cfg.dependents.find (pk)); - - if (i != cfg.dependents.end () && i->second.existing) - replace_ver (pkg); - } - } - - // This is the first time we are adding this package name to the map. - // - l4 ([&]{trace << "add " << pkg.available_name_version_db ();}); - - i = map_.emplace (move (pk), data_type {end (), move (pkg)}).first; - } - - build_package& p (i->second.package); - - if (vpb) - vpb (p, false /* scratch */); - - // Recursively collect build prerequisites, if requested. - // - // Note that detecting dependency cycles during the satisfaction phase - // would be premature since they may not be present in the final package - // list. Instead we check for them during the ordering phase. - // - // The question, of course, is whether we can still end up with an - // infinite recursion here? Note that for an existing map entry we only - // recurse after the entry replacement. The infinite recursion would - // mean that we may replace a package in the map with the same version - // multiple times: - // - // ... p1 -> p2 -> ... p1 - // - // Every replacement increases the entry version and/or tightens the - // constraints the next replacement will need to satisfy. It feels - // impossible that a package version can "return" into the map being - // replaced once. So let's wait until some real use case proves this - // reasoning wrong. - // - if (recursive) - collect_build_prerequisites (options, - p, - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - *dep_chain, - postponed_repo, - postponed_alts, - 0 /* max_alt_index */, - *postponed_deps, - postponed_cfgs, - *postponed_poss); - - return &p; } - // Collect prerequisites of the package being built recursively. - // - // But first "prune" this process if the package we build is a system one - // or is already configured, since that would mean all its prerequisites - // are configured as well. Note that this is not merely an optimization: - // the package could be an orphan in which case the below logic will fail - // (no repository fragment in which to search for prerequisites). By - // skipping the prerequisite check we are able to gracefully handle - // configured orphans. - // - // There are, however, some cases when we still need to re-collect - // prerequisites of a configured package: - // - // - For the repointed dependent we still need to collect its prerequisite - // replacements to make sure its dependency constraints are satisfied. - // - // - If configuration variables are specified for the dependent which has - // any buildfile clauses in the dependencies, then we need to - // re-evaluate them. This can result in a different set of dependencies - // required by this dependent (due to conditional dependencies, etc) - // and, potentially, for its reconfigured existing prerequisites, - // recursively. - // - // - For an existing dependent being re-evaluated to the specific - // dependency position. - // - // Note that for these cases, as it was said above, we can potentially - // fail if the dependent is an orphan, but this is exactly what we need to - // do in that case, since we won't be able to re-collect its dependencies. - // - // Only a single true dependency alternative can be selected per function - // call. Such an alternative can only be selected if its index in the - // postponed alternatives list is less than the specified maximum (used by - // the heuristics that determines in which order to process packages with - // alternatives; if 0 is passed, then no true alternative will be - // selected). - // - // The idea here is to postpone the true alternatives selection till the - // end of the packages collection and then try to optimize the overall - // resulting selection (over all the dependents) by selecting alternatives - // with the lower indexes first (see collect_build_postponed() for - // details). - // - // Always postpone recursive collection of dependencies for a dependent - // with configuration clauses, recording them in postponed_deps (see - // postponed_dependencies for details) and also recording the dependent in - // postponed_cfgs (see postponed_configurations for details). If it turns - // out that some dependency of such a dependent has already been collected - // via some other dependent without configuration clauses, then throw the - // postpone_dependency exception. This exception is handled via - // re-collecting packages from scratch, but now with the knowledge about - // premature dependency collection. If it turns out that some dependency - // configuration has already been negotiated between some other - // dependents, then up-negotiate the configuration and throw - // retry_configuration exception so that the configuration refinement can - // be performed (see the collect lambda implementation for details on the - // configuration refinement machinery). - // - // If the package is a dependency of a configured dependent with - // configuration clause and needs to be reconfigured (being upgraded, has - // configuration specified, etc), then postpone its recursive collection - // by recording it in postponed_cfgs as a single-dependency cluster with - // an existing dependent (see postponed_configurations for details). If - // this dependent already belongs to some (being) negotiated configuration - // cluster with a greater dependency position then record this dependency - // position in postponed_poss and throw postpone_position. This exception - // is handled by re-collecting packages from scratch, but now with the - // knowledge about position this dependent needs to be re-evaluated to. - // - struct postpone_dependency: scratch_collection - { - package_key package; - - explicit - postpone_dependency (package_key p) - : scratch_collection ("prematurely collected dependency"), - package (move (p)) - { - scratch_collection::package = &package; - } - }; - - struct postpone_position: scratch_collection - { - postpone_position () - : scratch_collection ("earlier dependency position") {} - }; + cache[move (k)] = r; + return r; + } - void - collect_build_prerequisites (const pkg_build_options& options, - build_package& pkg, - const function<find_database_function>& fdb, - const repointed_dependents& rpt_depts, - const function<add_priv_cfg_function>& apc, - bool initial_collection, - replaced_versions& replaced_vers, - build_package_refs& dep_chain, - postponed_packages* postponed_repo, - postponed_packages* postponed_alts, - size_t max_alt_index, - postponed_dependencies& postponed_deps, - postponed_configurations& postponed_cfgs, - postponed_positions& postponed_poss, - pair<size_t, size_t> reeval_pos = - make_pair(0, 0)) - { - // NOTE: don't forget to update collect_build_postponed() if changing - // anything in this function. - // - tracer trace ("collect_build_prerequisites"); + // Evaluate a package (not necessarily dependency) and return a new desired + // version. If the result is absent (nullopt), then no changes to the + // package are necessary. Otherwise, the result is available_package to + // upgrade/downgrade to or replace with, as well as the repository fragment + // it must come from. + // + // If the system package cannot be upgraded to the source one, not being + // found in the dependents repositories, then return nullopt if + // ignore_unsatisfiable argument is true and fail otherwise (see the + // evaluate_dependency() function description for details). + // + static optional<evaluate_result> + evaluate_recursive (const common_options& o, + database& db, + const shared_ptr<selected_package>& sp, + const recursive_packages& recs, + const existing_dependencies& existing_deps, + const deorphaned_dependencies& deorphaned_deps, + const build_packages& pkgs, + bool ignore_unsatisfiable, + upgrade_dependencies_cache& cache) + { + tracer trace ("evaluate_recursive"); - assert (pkg.action && *pkg.action == build_package::build); + assert (sp != nullptr); - const package_name& nm (pkg.name ()); - database& pdb (pkg.db); - package_key pk (pdb, nm); + // Build a set of repository fragment the dependent packages come from. + // Also cache the dependents and the constraints they apply to this + // dependency. + // + config_repo_fragments repo_frags; + dependent_constraints dpt_constrs; - bool reeval (reeval_pos.first != 0); + // Only collect repository fragments (for best version selection) of + // (immediate) dependents that have a hit (direct or indirect) in recs. + // Note, however, that we collect constraints from all the dependents. + // + upgrade_deorphan ud {nullopt /* upgrade */, false /* deorphan */}; - // The being re-evaluated dependent cannot be recursively collected yet. - // Also, we don't expect it being configured as system. - // - // Note that the configured package can still be re-evaluated after - // collect_build_prerequisites() has been called but didn't end up with - // the recursive collection. - // - assert (!reeval || - ((!pkg.recursive_collection || - !pkg.recollect_recursively (rpt_depts)) && - !pkg.skeleton && !pkg.system)); - - // If this package is not being re-evaluated, is not yet collected - // recursively, needs to be reconfigured, and is not yet postponed, then - // check if it is a dependency of any dependent with configuration - // clause and postpone the collection if that's the case. - // - // The reason why we don't need to do this for the re-evaluated case is - // as follows: this logic is used for an existing dependent that is not - // otherwise built (e.g., reconfigured) which means its externally- - // imposed configuration (user, dependents) is not being changed. - // - if (!reeval && - !pkg.recursive_collection && - pkg.reconfigure () && - postponed_cfgs.find_dependency (pk) == nullptr) + for (database& ddb: db.dependent_configs ()) + { + for (auto& pd: query_dependents_cache (ddb, sp->name, db)) { - // If the dependent is being built, then check if it was re-evaluated - // to the position greater than the dependency position. Return true - // if that's the case, so this package is added to the resulting list - // and we can handle this situation below. - // - // Note that we rely on "small function object" optimization here. - // - const function<verify_dependent_build_function> verify ( - [&postponed_cfgs] - (const package_key& pk, pair<size_t, size_t> pos) - { - for (const postponed_configuration& cfg: postponed_cfgs) - { - if (cfg.negotiated) - { - if (const pair<size_t, size_t>* p = - cfg.existing_dependent_position (pk)) - { - if (p->first > pos.first) - return true; - } - } - } + shared_ptr<selected_package> p (ddb.load<selected_package> (pd.name)); - return false; - }); + dpt_constrs.emplace_back (ddb, p, move (pd.constraint)); - // Note that there can be multiple existing dependents for a - // dependency. Strictly speaking, we only need to add the first one - // with the assumption that the remaining dependents will also be - // considered comes the time for the negotiation. Let's, however, - // process all of them to detect the potential "re-evaluation on the - // greater dependency index" situation earlier. And, generally, have - // as much information as possible up front. - // - vector<existing_dependent> eds ( - query_existing_dependents (trace, - pk.db, - pk.name, - replaced_vers, - rpt_depts, - verify)); + upgrade_deorphan u (upgrade_dependencies (ddb, pd.name, recs, cache)); - if (!eds.empty ()) + if (u.upgrade || u.deorphan) { - for (existing_dependent& ed: eds) - { - package_key dpk (ed.db, ed.selected->name); - size_t& di (ed.dependency_position.first); - - const build_package* bp (&pkg); - - // Check if this dependent needs to be re-evaluated to an earlier - // dependency position and, if that's the case, create the - // configuration cluster with this dependency instead. - // - // Note that if the replace flag is false, we proceed normally - // with the assumption that the dependency referred by the entry - // will be collected later and its configuration cluster will be - // created normally and will be negotiated earlier than the - // cluster being created for the current dependency (see - // collect_build_postponed() for details). - // - { - auto pi (postponed_poss.find (dpk)); - - if (pi != postponed_poss.end () && pi->second.first < di) - { - // If requested, override the first encountered non-replace - // position to replace. See collect_build_postponed () for - // details. - // - if (!pi->second.replace && postponed_poss.replace) - { - pi->second.replace = true; - postponed_poss.replace = false; - } - - if (pi->second.replace) - { - // Overwrite the existing dependent dependency information - // and fall through to proceed as for the normal case. - // - bp = replace_existing_dependent_dependency ( - trace, - options, - ed, // Note: modified. - pi->second, - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - postponed_cfgs); - - pk = package_key (bp->db, bp->name ()); - - // Note that here we side-step the bogus logic (by not - // setting the skipped flag) because in this case - // (replace=true) our choices are either (potentially) bogus - // or pathological (where we have evaluated too far). In - // other words, the postponed entry may cause the depends - // entry that triggered it to disappear (and thus, strictly - // speaking, to become bogus) but if we cancel it, we will - // be back to square one. - } - } - } - - // Make sure that this existing dependent doesn't belong to any - // (being) negotiated configuration cluster with a greater - // dependency index. That would mean that this dependent has - // already been re-evaluated to this index and so cannot - // participate in the configuration negotiation of this earlier - // dependency. - // - for (const postponed_configuration& cfg: postponed_cfgs) - { - if (const pair<size_t, size_t>* p = - cfg.existing_dependent_position (pk)) - { - size_t ei (p->first); - - if (di < ei && cfg.negotiated) - { - // Feels like there cannot be an earlier position. - // - postponed_position pp (ed.dependency_position, - false /* replace */); - - auto p (postponed_poss.emplace (move (pk), pp)); - if (!p.second) - { - assert (p.first->second > pp); - p.first->second = pp; - } - - l5 ([&]{trace << "cannot cfg-postpone dependency " - << bp->available_name_version_db () - << " of existing dependent " << *ed.selected - << ed.db << " (index " << di - << ") due to earlier dependency index " << ei - << " in " << cfg << ", throwing " - << "postpone_position";}); - - // Don't print the "while satisfying..." chain. - // - dep_chain.clear (); - - throw postpone_position (); - } - - if (di == ei) - { - // For the negotiated cluster all the dependency packages - // should have been added. For non-negotiated cluster we - // cannot add the missing dependencies at the moment and - // will do it as a part of the dependent re-evaluation. - // - assert (!cfg.negotiated); - } - } - } - - l5 ([&]{trace << "cfg-postpone dependency " - << bp->available_name_version_db () - << " of existing dependent " << *ed.selected - << ed.db;}); - - postponed_cfgs.add (move (dpk), ed.dependency_position, move (pk)); - } + // Upgrade wins over patch. + // + if (u.upgrade && (!ud.upgrade || *ud.upgrade < *u.upgrade)) + ud.upgrade = *u.upgrade; - return; + if (u.deorphan) + ud.deorphan = true; } - } - - pkg.recursive_collection = true; + else + continue; - if (pkg.system) - { - l5 ([&]{trace << "skip system " << pkg.available_name_version_db ();}); - return; + // While we already know that the dependency upgrade is required, we + // continue to iterate over dependents, collecting the repository + // fragments and the constraints. + // + add_dependent_repo_fragments (ddb, p, repo_frags); } + } - const shared_ptr<available_package>& ap (pkg.available); - assert (ap != nullptr); - - const shared_ptr<selected_package>& sp (pkg.selected); + if (!ud.upgrade && !ud.deorphan) + { + l5 ([&]{trace << *sp << db << ": no hit";}); + return nullopt; + } - // True if this is an up/down-grade. - // - bool ud (sp != nullptr && sp->version != pkg.available_version ()); + pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>> rp ( + find_existing (db, sp->name, nullopt /* version_constraint */)); - // If this is a repointed dependent, then it points to its prerequisite - // replacements flag map (see repointed_dependents for details). - // - const map<package_key, bool>* rpt_prereq_flags (nullptr); + optional<evaluate_result> r ( + evaluate_dependency (o, + db, + sp, + nullopt /* desired */, + false /* desired_sys */, + rp.first != nullptr /* existing */, + db, + sp, + ud.upgrade, + ud.deorphan, + false /* explicitly */, + repo_frags, + dpt_constrs, + existing_deps, + deorphaned_deps, + pkgs, + ignore_unsatisfiable)); - // Bail out if this is a configured non-system package and no recursive - // collection is required. - // - bool src_conf (sp != nullptr && - sp->state == package_state::configured && - sp->substate != package_substate::system); + // Translate the "no change" result into nullopt. + // + assert (!r || !r->unused); + return r && r->available == nullptr ? nullopt : r; + } - // The being re-evaluated dependent must be configured as a source - // package and should not be collected recursively (due to upgrade, - // etc). - // - assert (!reeval || (src_conf && !pkg.recollect_recursively (rpt_depts))); + // Stack of the command line adjustments as per unsatisfied_dependents + // description. + // + struct cmdline_adjustment + { + enum class adjustment_type: uint8_t + { + hold_existing, // Adjust constraint in existing build-to-hold spec. + dep_existing, // Adjust constraint in existing dependency spec. + hold_new, // Add new build-to-hold spec. + dep_new // Add new dependency spec. + }; - if (src_conf) - { - repointed_dependents::const_iterator i (rpt_depts.find (pk)); + adjustment_type type; + reference_wrapper<database> db; + package_name name; + bpkg::version version; // Replacement. + + // Meaningful only for the *_new types. + // + optional<bool> upgrade; + bool deorphan = false; + + // For the newly created or popped from the stack object the following + // three members contain the package version replacement information. + // Otherwise (pushed to the stack), they contain the original command line + // spec information. + // + shared_ptr<available_package> available; // NULL for dep_* types. + lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; // As above. + optional<version_constraint> constraint; + + // Create object of the hold_existing type. + // + cmdline_adjustment (database& d, + const package_name& n, + shared_ptr<available_package>&& a, + lazy_shared_ptr<bpkg::repository_fragment>&& f) + : type (adjustment_type::hold_existing), + db (d), + name (n), + version (a->version), + available (move (a)), + repository_fragment (move (f)), + constraint (version_constraint (version)) {} + + // Create object of the dep_existing type. + // + cmdline_adjustment (database& d, + const package_name& n, + const bpkg::version& v) + : type (adjustment_type::dep_existing), + db (d), + name (n), + version (v), + constraint (version_constraint (version)) {} + + // Create object of the hold_new type. + // + cmdline_adjustment (database& d, + const package_name& n, + shared_ptr<available_package>&& a, + lazy_shared_ptr<bpkg::repository_fragment>&& f, + optional<bool> u, + bool o) + : type (adjustment_type::hold_new), + db (d), + name (n), + version (a->version), + upgrade (u), + deorphan (o), + available (move (a)), + repository_fragment (move (f)), + constraint (version_constraint (version)) {} + + // Create object of the dep_new type. + // + cmdline_adjustment (database& d, + const package_name& n, + const bpkg::version& v, + optional<bool> u, + bool o) + : type (adjustment_type::dep_new), + db (d), + name (n), + version (v), + upgrade (u), + deorphan (o), + constraint (version_constraint (version)) {} + }; - if (i != rpt_depts.end ()) - rpt_prereq_flags = &i->second; + class cmdline_adjustments + { + public: + cmdline_adjustments (vector<build_package>& hps, dependency_packages& dps) + : hold_pkgs_ (hps), + dep_pkgs_ (dps) {} - if (!reeval && !pkg.recollect_recursively (rpt_depts)) - { - l5 ([&]{trace << "skip configured " - << pkg.available_name_version_db ();}); - return; - } - } + // Apply the specified adjustment to the command line, push the adjustment + // to the stack, and record the resulting command line state as the SHA256 + // checksum. + // + void + push (cmdline_adjustment&& a) + { + using type = cmdline_adjustment::adjustment_type; - // Iterate over dependencies, trying to unambiguously select a - // satisfactory dependency alternative for each of them. Fail or - // postpone the collection if unable to do so. + // We always set the `== <version>` constraint in the resulting spec. // - const dependencies& deps (ap->dependencies); + assert (a.constraint); - // The skeleton can be pre-initialized before the recursive collection - // starts (as a part of dependency configuration negotiation, etc). The - // dependencies and alternatives members must both be either present or - // not. - // - assert ((!pkg.dependencies || pkg.skeleton) && - pkg.dependencies.has_value () == pkg.alternatives.has_value ()); + database& db (a.db); + const package_name& nm (a.name); + package_version_key cmd_line (db.main_database (), "command line"); - // Note that the selected alternatives list can be filled partially (see - // build_package::dependencies for details). In this case we continue - // collecting where we stopped previously. - // - if (!pkg.dependencies) + switch (a.type) { - l5 ([&]{trace << (reeval ? "reeval " : "begin ") - << pkg.available_name_version_db ();}); - - pkg.dependencies = dependencies (); - pkg.alternatives = vector<size_t> (); - - if (size_t n = deps.size ()) + case type::hold_existing: { - pkg.dependencies->reserve (n); - pkg.alternatives->reserve (n); - } + auto i (find_hold_pkg (a)); + assert (i != hold_pkgs_.end ()); // As per adjustment type. - if (!pkg.skeleton) - pkg.init_skeleton (options); - } - else - l5 ([&]{trace << "resume " << pkg.available_name_version_db ();}); + build_package& bp (*i); + swap (bp.available, a.available); + swap (bp.repository_fragment, a.repository_fragment); - dependencies& sdeps (*pkg.dependencies); - vector<size_t>& salts (*pkg.alternatives); - - assert (sdeps.size () == salts.size ()); // Must be parallel. - - // Check if there is nothing to collect anymore. - // - if (sdeps.size () == deps.size ()) - { - l5 ([&]{trace << "end " << pkg.available_name_version_db ();}); - return; - } - - // Show how we got here if things go wrong. - // - // To suppress printing this information clear the dependency chain - // before throwing an exception. - // - auto g ( - make_exception_guard ( - [&dep_chain] () + if (!bp.constraints.empty ()) { - // Note that we also need to clear the dependency chain, to - // prevent the caller's exception guard from printing it. - // - while (!dep_chain.empty ()) - { - info << "while satisfying " - << dep_chain.back ().get ().available_name_version_db (); - - dep_chain.pop_back (); - } - })); - - dep_chain.push_back (pkg); - - assert (sdeps.size () < deps.size ()); - - package_skeleton& skel (*pkg.skeleton); - - auto fail_reeval = [&pkg] () - { - fail << "unable to re-create dependency information of already " - << "configured package " << pkg.available_name_version_db () << - info << "likely cause is change in external environment" << - info << "consider resetting the build configuration"; - }; - - bool postponed (false); - bool reevaluated (false); - - for (size_t di (sdeps.size ()); di != deps.size (); ++di) - { - // Fail if we missed the re-evaluation target position for any reason. - // - if (reeval && di == reeval_pos.first) // Note: reeval_pos is 1-based. - fail_reeval (); - - const dependency_alternatives_ex& das (deps[di]); - - // Add an empty alternatives list into the selected dependency list if - // this is a toolchain build-time dependency. - // - dependency_alternatives_ex sdas (das.buildtime, das.comment); - - if (toolchain_buildtime_dependency (options, das, &nm)) - { - sdeps.push_back (move (sdas)); - salts.push_back (0); // Keep parallel to sdeps. - continue; - } - - // Evaluate alternative conditions and filter enabled alternatives. - // Add an empty alternatives list into the selected dependency list if - // there are none. - // - build_package::dependency_alternatives_refs edas; - - if (pkg.postponed_dependency_alternatives) - { - edas = move (*pkg.postponed_dependency_alternatives); - pkg.postponed_dependency_alternatives = nullopt; - } - else - { - for (size_t i (0); i != das.size (); ++i) + swap (bp.constraints[0].value, *a.constraint); + } + else { - const dependency_alternative& da (das[i]); - - if (!da.enable || - skel.evaluate_enable (*da.enable, make_pair (di, i))) - edas.push_back (make_pair (ref (da), i)); + bp.constraints.emplace_back (move (*a.constraint), + cmd_line.db, + cmd_line.name.string ()); + a.constraint = nullopt; } - } - if (edas.empty ()) + break; + } + case type::dep_existing: { - sdeps.push_back (move (sdas)); - salts.push_back (0); // Keep parallel to sdeps. - continue; + auto i (find_dep_pkg (a)); + assert (i != dep_pkgs_.end ()); // As per adjustment type. + swap (i->constraint, a.constraint); + break; } - - // Try to pre-collect build information (pre-builds) for the - // dependencies of an alternative. Optionally, issue diagnostics into - // the specified diag record. - // - // Note that rather than considering an alternative as unsatisfactory - // (returning no pre-builds) the function can fail in some cases - // (multiple possible configurations for a build-time dependency, - // orphan or broken selected package, etc). The assumption here is - // that the user would prefer to fix a dependency-related issue first - // instead of proceeding with the build which can potentially end up - // with some less preferable dependency alternative. - // - struct prebuild - { - bpkg::dependency dependency; - reference_wrapper<database> db; - shared_ptr<selected_package> selected; - shared_ptr<available_package> available; - lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; - bool system; - bool specified_dependency; - bool force; - - // True if the dependency package is either selected in the - // configuration or is already being built. - // - bool reused; - }; - using prebuilds = small_vector<prebuild, 1>; - - class precollect_result + case type::hold_new: { - public: - // Nullopt if some dependencies cannot be resolved. - // - optional<prebuilds> builds; - - // True if dependencies can all be resolved (builds is present) and - // are all reused (see above). - // - bool reused; - - // True if some of the dependencies cannot be resolved (builds is - // nullopt) and the dependent package prerequisites collection needs - // to be postponed due to inability to find a version satisfying the - // pre-entered constraint from repositories available to the - // dependent package. - // - bool repo_postpone; - - // Create precollect result containing dependency builds. + // As per adjustment type. // - precollect_result (prebuilds&& bs, bool r) - : builds (move (bs)), reused (r), repo_postpone (false) {} + assert (find_hold_pkg (a) == hold_pkgs_.end ()); - // Create precollect result without builds (some dependency can't be - // satisfied, etc). + // Start the database transaction to perform the + // database::find<selected_package> call, unless we are already in + // the transaction. // - explicit - precollect_result (bool p): reused (false), repo_postpone (p) {} - }; - auto precollect = [&options, - &pkg, - &nm, - &pdb, - ud, - &fdb, - rpt_prereq_flags, - &apc, - postponed_repo, - &dep_chain, - &trace, - this] - (const dependency_alternative& da, - bool buildtime, - const package_prerequisites* prereqs, - diag_record* dr = nullptr) - -> precollect_result - { - prebuilds r; - bool reused (true); - - const lazy_shared_ptr<repository_fragment>& af ( - pkg.repository_fragment); - - for (const dependency& dp: da) - { - const package_name& dn (dp.name); - - if (buildtime && pdb.type == build2_config_type) - { - assert (dr == nullptr); // Should fail on the "silent" run. - - // Note that the dependent is not necessarily a build system - // module. - // - fail << "build-time dependency " << dn << " in build system " - << "module configuration" << - info << "build system modules cannot have build-time " - << "dependencies"; - } - - bool system (false); - bool specified (false); - - // If the user specified the desired dependency version - // constraint, then we will use it to overwrite the constraint - // imposed by the dependent package, checking that it is still - // satisfied. - // - // Note that we can't just rely on the execution plan refinement - // that will pick up the proper dependency version at the end of - // the day. We may just not get to the plan execution simulation, - // failing due to inability for dependency versions collected by - // two dependents to satisfy each other constraints (for an - // example see the - // pkg-build/dependency/apply-constraints/resolve-conflict{1,2} - // tests). - - // Points to the desired dependency version constraint, if - // specified, and is NULL otherwise. Can be used as boolean flag. - // - const version_constraint* dep_constr (nullptr); - - database* ddb (fdb (pdb, dn, buildtime)); - - auto i (ddb != nullptr - ? map_.find (*ddb, dn) - : map_.find_dependency (pdb, dn, buildtime)); - - if (i != map_.end ()) - { - const build_package& bp (i->second.package); - - specified = !bp.action; // Is pre-entered. - - if (specified && - // - // The version constraint is specified, - // - bp.hold_version && *bp.hold_version) - { - assert (bp.constraints.size () == 1); - - const build_package::constraint_type& c (bp.constraints[0]); - - dep_constr = &c.value; - system = bp.system; - - // If the user-specified dependency constraint is the wildcard - // version, then it satisfies any dependency constraint. - // - if (!wildcard (*dep_constr) && - !satisfies (*dep_constr, dp.constraint)) - { - if (dr != nullptr) - *dr << error << "unable to satisfy constraints on package " - << dn << - info << nm << pdb << " depends on (" << dn - << " " << *dp.constraint << ")" << - info << c.dependent << c.db << " depends on (" << dn - << " " << c.value << ")" << - info << "specify " << dn << " version to satisfy " - << nm << " constraint"; - - return precollect_result (false /* postpone */); - } - } - } - - const dependency& d (!dep_constr - ? dp - : dependency {dn, *dep_constr}); - - // First see if this package is already selected. If we already - // have it in the configuration and it satisfies our dependency - // version constraint, then we don't want to be forcing its - // upgrade (or, worse, downgrade). - // - // If the prerequisite configuration is explicitly specified by the - // user, then search for the prerequisite in this specific - // configuration. Otherwise, search recursively in the explicitly - // linked configurations of the dependent configuration. - // - // Note that for the repointed dependent we will always find the - // prerequisite replacement rather than the prerequisite being - // replaced. - // - pair<shared_ptr<selected_package>, database*> spd ( - ddb != nullptr - ? make_pair (ddb->find<selected_package> (dn), ddb) - : find_dependency (pdb, dn, buildtime)); - - if (ddb == nullptr) - ddb = &pdb; - - shared_ptr<selected_package>& dsp (spd.first); - - if (prereqs != nullptr && - (dsp == nullptr || - find_if (prereqs->begin (), prereqs->end (), - [&dsp] (const auto& v) - { - return v.first.object_id () == dsp->name; - }) == prereqs->end ())) - return precollect_result (false /* postpone */); - - pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> rp; - - shared_ptr<available_package>& dap (rp.first); - - bool force (false); - - if (dsp != nullptr) - { - // Switch to the selected package configuration. - // - ddb = spd.second; - - // If we are collecting prerequisites of the repointed - // dependent, then only proceed further if this is either a - // replacement or unamended prerequisite and we are - // up/down-grading (only for the latter). - // - if (rpt_prereq_flags != nullptr) - { - auto i (rpt_prereq_flags->find (package_key {*ddb, dn})); - - bool unamended (i == rpt_prereq_flags->end ()); - bool replacement (!unamended && i->second); - - // We can never end up with the prerequisite being replaced, - // since the fdb() function should always return the - // replacement instead (see above). - // - assert (unamended || replacement); - - if (!(replacement || (unamended && ud))) - continue; - } - - if (dsp->state == package_state::broken) - { - assert (dr == nullptr); // Should fail on the "silent" run. - - fail << "unable to build broken package " << dn << *ddb << - info << "use 'pkg-purge --force' to remove"; - } - - // If the constraint is imposed by the user we also need to make - // sure that the system flags are the same. - // - if (satisfies (dsp->version, d.constraint) && - (!dep_constr || dsp->system () == system)) - { - system = dsp->system (); - - version_constraint vc (dsp->version); - - // First try to find an available package for this exact - // version, falling back to ignoring version revision and - // iteration. In particular, this handles the case where a - // package moves from one repository to another (e.g., from - // testing to stable). For a system package we will try to - // find the available package that matches the selected - // package version (preferable for the configuration - // negotiation machinery) and, if fail, fallback to picking - // the latest one (its exact version doesn't really matter in - // this case). - // - // It seems reasonable to search for the package in the - // repositories explicitly added by the user if the selected - // package was explicitly specified on command line, and in - // the repository (and its complements/prerequisites) of the - // dependent being currently built otherwise. - // - if (dsp->hold_package) - { - linked_databases dbs (dependent_repo_configs (*ddb)); - - rp = find_available_one (dbs, - dn, - vc, - true /* prereq */, - true /* revision */); + transaction t (db, !transaction::has_current ()); - if (dap == nullptr) - rp = find_available_one (dbs, dn, vc); - - if (dap == nullptr && system) - rp = find_available_one (dbs, dn, nullopt); - } - else if (af != nullptr) - { - rp = find_available_one (dn, - vc, - af, - true /* prereq */, - true /* revision */); - - if (dap == nullptr) - rp = find_available_one (dn, vc, af); - - if (dap == nullptr && system) - rp = find_available_one (dn, nullopt, af); - } - - // A stub satisfies any version constraint so we weed them out - // (returning stub as an available package feels wrong). - // - if (dap == nullptr || dap->stub ()) - rp = make_available_fragment (options, *ddb, dsp); - } - else - // Remember that we may be forcing up/downgrade; we will deal - // with it below. - // - force = true; - } - - // If this is a build-time dependency and we build it for the - // first time, then we need to find a suitable configuration (of - // the host or build2 type) to build it in. - // - // If the current configuration (ddb) is of the suitable type, - // then we use that. Otherwise, we go through its immediate - // explicit links. If only one of them has the suitable type, then - // we use that. If there are multiple of them, then we fail - // advising the user to pick one explicitly. If there are none, - // then we create the private configuration and use that. If the - // current configuration is private, then search/create in the - // parent configuration instead. - // - // Note that if the user has explicitly specified the - // configuration for this dependency on the command line (using - // --config-*), then this configuration is used as the starting - // point for this search. - // - if (buildtime && - dsp == nullptr && - ddb->type != buildtime_dependency_type (dn)) - { - database* db (nullptr); - database& sdb (ddb->private_ () ? ddb->parent_config () : *ddb); - - const string& type (buildtime_dependency_type (dn)); - - // Skip the self-link. - // - const linked_configs& lcs (sdb.explicit_links ()); - for (auto i (lcs.begin_linked ()); i != lcs.end (); ++i) - { - database& ldb (i->db); - - if (ldb.type == type) - { - if (db == nullptr) - db = &ldb; - else - { - assert (dr == nullptr); // Should fail on the "silent" run. - - fail << "multiple possible " << type << " configurations " - << "for build-time dependency (" << dp << ")" << - info << db->config_orig << - info << ldb.config_orig << - info << "use --config-* to select the configuration"; - } - } - } - - // If no suitable configuration is found, then create and link - // it, unless the --no-private-config options is specified. In - // the latter case, print the dependency chain to stdout and - // exit with the specified code. - // - if (db == nullptr) - { - // The private config should be created on the "silent" run - // and so there always should be a suitable configuration on - // the diagnostics run. - // - assert (dr == nullptr); - - if (options.no_private_config_specified ()) - try - { - // Note that we don't have the dependency package version - // yet. We could probably rearrange the code and obtain the - // available dependency package by now, given that it comes - // from the main database and may not be specified as system - // (we would have the configuration otherwise). However, - // let's not complicate the code further and instead print - // the package name and the constraint, if present. - // - // Also, in the future, we may still need the configuration - // to obtain the available dependency package for some - // reason (may want to fetch repositories locally, etc). - // - cout << d << '\n'; - - // Note that we also need to clean the dependency chain, to - // prevent the exception guard from printing it to stderr. - // - for (build_package_refs dc (move (dep_chain)); - !dc.empty (); ) - { - const build_package& p (dc.back ()); - - cout << p.available_name_version () << ' ' - << p.db.get ().config << '\n'; - - dc.pop_back (); - } - - throw failed (options.no_private_config ()); - } - catch (const io_error&) - { - fail << "unable to write to stdout"; - } - - const strings mods {"cc"}; - - const strings vars { - "config.config.load=~" + type, - "config.config.persist+='config.*'@unused=drop"}; - - dir_path cd (bpkg_dir / dir_path (type)); - - // Wipe a potentially existing un-linked private configuration - // left from a previous faulty run. Note that trying to reuse - // it would be a bad idea since it can be half-prepared, with - // an outdated database schema version, etc. - // - cfg_create (options, - sdb.config_orig / cd, - optional<string> (type) /* name */, - type /* type */, - mods, - vars, - false /* existing */, - true /* wipe */); - - // Note that we will copy the name from the configuration - // unless it clashes with one of the existing links. - // - shared_ptr<configuration> lc ( - cfg_link (sdb, - sdb.config / cd, - true /* relative */, - nullopt /* name */, - true /* sys_rep */)); - - // Save the newly-created private configuration, together with - // the containing configuration database, for their subsequent - // re-link. - // - apc (sdb, move (cd)); - - db = &sdb.find_attached (*lc->id); - } - - ddb = db; // Switch to the dependency configuration. - } - - // Note that building a dependent which is not a build2 module in - // the same configuration with the build2 module it depends upon - // is an error. - // - if (buildtime && - !build2_module (nm) && - build2_module (dn) && - pdb == *ddb) - { - assert (dr == nullptr); // Should fail on the "silent" run. - - // Note that the dependent package information is printed by the - // above exception guard. - // - fail << "unable to build build system module " << dn - << " in its dependent package configuration " - << pdb.config_orig << - info << "use --config-* to select suitable configuration"; - } - - // If we didn't get the available package corresponding to the - // selected package, look for any that satisfies the constraint. - // - if (dap == nullptr) - { - // And if we have no repository fragment to look in, then that - // means the package is an orphan (we delay this check until we - // actually need the repository fragment to allow orphans - // without prerequisites). - // - if (af == nullptr) - { - assert (dr == nullptr); // Should fail on the "silent" run. - - fail << "package " << pkg.available_name_version_db () - << " is orphaned" << - info << "explicitly upgrade it to a new version"; - } - - // We look for prerequisites only in the repositories of this - // package (and not in all the repositories of this - // configuration). At first this might look strange, but it - // also kind of makes sense: we only use repositories "approved" - // for this package version. Consider this scenario as an - // example: hello/1.0.0 and libhello/1.0.0 in stable and - // libhello/2.0.0 in testing. As a prerequisite of hello, which - // version should libhello resolve to? While one can probably - // argue either way, resolving it to 1.0.0 is the conservative - // choice and the user can always override it by explicitly - // building libhello. - // - // Note though, that if this is a test package, then its special - // test dependencies (main packages that refer to it) should be - // searched upstream through the complement repositories - // recursively, since the test packages may only belong to the - // main package's repository and its complements. - // - // @@ Currently we don't implement the reverse direction search - // for the test dependencies, effectively only supporting the - // common case where the main and test packages belong to the - // same repository. Will need to fix this eventually. - // - // Note that this logic (naturally) does not apply if the - // package is already selected by the user (see above). - // - // Also note that for the user-specified dependency version - // constraint we rely on the satisfying package version be - // present in repositories of the first dependent met. As a - // result, we may fail too early if such package version doesn't - // belong to its repositories, but belongs to the ones of some - // dependent that we haven't met yet. Can we just search all - // repositories for an available package of the appropriate - // version and just take it, if present? We could, but then - // which repository should we pick? The wrong choice can - // introduce some unwanted repositories and package versions - // into play. So instead, we will postpone collecting the - // problematic dependent, expecting that some other one will - // find the appropriate version in its repositories. - // - // For a system package we will try to find the available - // package that matches the constraint (preferable for the - // configuration negotiation machinery) and, if fail, fallback - // to picking the latest one just to make sure the package is - // recognized. An unrecognized package means the broken/stale - // repository (see below). - // - rp = find_available_one (dn, d.constraint, af); - - if (dap == nullptr && system && d.constraint) - rp = find_available_one (dn, nullopt, af); - - if (dap == nullptr) - { - if (dep_constr && !system && postponed_repo != nullptr) - { - // We shouldn't be called in the diag mode for the postponed - // package builds. - // - assert (dr == nullptr); - - l5 ([&]{trace << "rep-postpone dependent " - << pkg.available_name_version_db () - << " due to dependency " << dp - << " and user-specified constraint " - << *dep_constr;}); - - postponed_repo->insert (&pkg); - return precollect_result (true /* postpone */); - } - - if (dr != nullptr) - { - *dr << error; - - // Issue diagnostics differently based on the presence of - // available packages for the unsatisfied dependency. - // - // Note that there can't be any stubs, since they satisfy - // any constraint and we won't be here if they were. - // - vector<shared_ptr<available_package>> aps ( - find_available (dn, nullopt /* version_constraint */, af)); - - if (!aps.empty ()) - { - *dr << "unable to satisfy dependency constraint (" << dn; - - // We need to be careful not to print the wildcard-based - // constraint. - // - if (d.constraint && - (!dep_constr || !wildcard (*dep_constr))) - *dr << ' ' << *d.constraint; - - *dr << ") of package " << nm << pdb << - info << "available " << dn << " versions:"; - - for (const shared_ptr<available_package>& ap: aps) - *dr << ' ' << ap->version; - } - else - { - *dr << "no package available for dependency " << dn - << " of package " << nm << pdb; - } - - // Avoid printing this if the dependent package is external - // since it's more often confusing than helpful (they are - // normally not fetched manually). - // - if (!af->location.empty () && - !af->location.directory_based () && - (!dep_constr || system)) - *dr << info << "repository " << af->location << " appears " - << "to be broken" << - info << "or the repository state could be stale" << - info << "run 'bpkg rep-fetch' to update"; - } - - return precollect_result (false /* postpone */); - } - - // If all that's available is a stub then we need to make sure - // the package is present in the system repository and it's - // version satisfies the constraint. If a source package is - // available but there is a system package specified on the - // command line and it's version satisfies the constraint then - // the system package should be preferred. To recognize such a - // case we just need to check if the authoritative system - // version is set and it satisfies the constraint. If the - // corresponding system package is non-optional it will be - // preferred anyway. - // - if (dap->stub ()) - { - // Note that the constraint can safely be printed as it can't - // be a wildcard (produced from the user-specified dependency - // version constraint). If it were, then the system version - // wouldn't be NULL and would satisfy itself. - // - if (dap->system_version (*ddb) == nullptr) - { - if (dr != nullptr) - *dr << error << "dependency " << d << " of package " - << nm << " is not available in source" << - info << "specify ?sys:" << dn << " if it is available " - << "from the system"; - - return precollect_result (false /* postpone */); - } + build_package bp { + build_package::build, + db, + db.find<selected_package> (nm), + move (a.available), + move (a.repository_fragment), + nullopt, // Dependencies. + nullopt, // Dependencies alternatives. + nullopt, // Package skeleton. + nullopt, // Postponed dependency alternatives. + false, // Recursive collection. + true, // Hold package. + false, // Hold version. + {}, // Constraints. + false, // System. + false, // Keep output directory. + false, // Disfigure (from-scratch reconf). + false, // Configure-only. + nullopt, // Checkout root. + false, // Checkout purge. + strings (), // Configuration variables. + a.upgrade, + a.deorphan, + {cmd_line}, // Required by (command line). + false, // Required by dependents. + (a.deorphan + ? build_package::build_replace + : uint16_t (0))}; - if (!satisfies (*dap->system_version (*ddb), d.constraint)) - { - if (dr != nullptr) - *dr << error << "dependency " << d << " of package " - << nm << " is not available in source" << - info << package_string (dn, - *dap->system_version (*ddb), - true /* system */) - << " does not satisfy the constrains"; - - return precollect_result (false /* postpone */); - } + t.commit (); - system = true; - } - else - { - auto p (dap->system_version_authoritative (*ddb)); + bp.constraints.emplace_back (move (*a.constraint), + cmd_line.db, + cmd_line.name.string ()); - if (p.first != nullptr && - p.second && // Authoritative. - satisfies (*p.first, d.constraint)) - system = true; - } - } + a.constraint = nullopt; - // If the dependency package of a different version is already - // being built, then we also need to make sure that we will be - // able to choose one of them (either existing or new) which - // satisfies all the dependents. - // - // Note that collect_build() also performs this check but - // postponing it till then can end up in failing instead of - // selecting some other dependency alternative. - // - assert (dap != nullptr); // Otherwise we would fail earlier. + hold_pkgs_.push_back (move (bp)); + break; + } + case type::dep_new: + { + // As per adjustment type. + // + assert (find_dep_pkg (a) == dep_pkgs_.end ()); - if (i != map_.end () && d.constraint) - { - const build_package& bp (i->second.package); + // Start the database transaction to perform the + // database::find<selected_package> call, unless we are already in + // the transaction. + // + transaction t (db, !transaction::has_current ()); + + dep_pkgs_.push_back ( + dependency_package {&db, + nm, + move (*a.constraint), + false /* hold_version */, + db.find<selected_package> (nm), + false /* system */, + false /* existing */, + a.upgrade, + a.deorphan, + false /* keep_out */, + false /* disfigure */, + nullopt /* checkout_root */, + false /* checkout_purge */, + strings () /* config_vars */, + nullptr /* system_status */}); - if (bp.action && *bp.action == build_package::build) - { - const version& v1 (system - ? *dap->system_version (*ddb) - : dap->version); + t.commit (); - const version& v2 (bp.available_version ()); + a.constraint = nullopt; + break; + } + } - if (v1 != v2) - { - using constraint_type = build_package::constraint_type; + packages_.insert (package_version_key (db, nm, a.version)); + adjustments_.push_back (move (a)); + former_states_.insert (state ()); + } - constraint_type c1 {pdb, nm.string (), *d.constraint}; + // Roll back the latest (default) or first command line adjustment, pop it + // from the stack, and return the popped adjustment. Assume that the stack + // is not empty. + // + // Note that the returned object can be pushed to the stack again. + // + cmdline_adjustment + pop (bool first = false) + { + using type = cmdline_adjustment::adjustment_type; - if (!satisfies (v2, c1.value)) - { - for (const constraint_type& c2: bp.constraints) - { - if (!satisfies (v1, c2.value)) - { - if (dr != nullptr) - { - const package_name& n (d.name); - const string& d1 (c1.dependent); - const string& d2 (c2.dependent); - - *dr << error << "unable to satisfy constraints on " - << "package " << n << - info << d2 << c2.db << " depends on (" << n << ' ' - << c2.value << ")" << - info << d1 << c1.db << " depends on (" << n << ' ' - << c1.value << ")" << - info << "available " - << bp.available_name_version () << - info << "available " - << package_string (n, v1, system) << - info << "explicitly specify " << n << " version " - << "to manually satisfy both constraints"; - } - - return precollect_result (false /* postpone */); - } - } - } - } - } - } + assert (!empty ()); - bool ru (i != map_.end () || dsp != nullptr); + // Pop the adjustment. + // + cmdline_adjustment a (move (!first + ? adjustments_.back () + : adjustments_.front ())); + if (!first) + adjustments_.pop_back (); + else + adjustments_.erase (adjustments_.begin ()); - if (!ru) - reused = false; + packages_.erase (package_version_key (a.db, a.name, a.version)); - r.push_back (prebuild {d, - *ddb, - move (dsp), - move (dap), - move (rp.second), - system, - specified, - force, - ru}); - } + // Roll back the adjustment. + // + switch (a.type) + { + case type::hold_existing: + { + auto i (find_hold_pkg (a)); + assert (i != hold_pkgs_.end ()); - return precollect_result (move (r), reused); - }; + build_package& bp (*i); + swap (bp.available, a.available); + swap (bp.repository_fragment, a.repository_fragment); - // Try to collect the previously collected pre-builds. - // - // Return false if the dependent has configuration clauses and is - // postponed until dependencies configuration negotiation. - // - auto collect = [&options, - &pkg, - &pdb, - &nm, - &pk, - &fdb, - &rpt_depts, - &apc, - initial_collection, - &replaced_vers, - &dep_chain, - postponed_repo, - postponed_alts, - &postponed_deps, - &postponed_cfgs, - &postponed_poss, - &di, - reeval, - &reeval_pos, - &reevaluated, - &fail_reeval, - &trace, - this] - (const dependency_alternative& da, - size_t dai, - prebuilds&& bs) - { - // Dependency alternative position. + // Must contain the replacement version. // - pair<size_t, size_t> dp (di + 1, dai + 1); + assert (!bp.constraints.empty ()); - if (reeval && - dp.first == reeval_pos.first && - dp.second != reeval_pos.second) - fail_reeval (); + version_constraint& c (bp.constraints[0].value); - postponed_configuration::packages cfg_deps; - - for (prebuild& b: bs) + if (a.constraint) // Original spec contains a version constraint? { - build_package bp { - build_package::build, - b.db, - b.selected, - b.available, - move (b.repository_fragment), - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - b.system, - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {pk}, // Required by (dependent). - true, // Required by dependents. - 0}; // State flags. - - const optional<version_constraint>& constraint ( - b.dependency.constraint); - - // Add our constraint, if we have one. - // - // Note that we always add the constraint implied by the dependent. - // The user-implied constraint, if present, will be added when - // merging from the pre-entered entry. So we will have both - // constraints for completeness. - // - if (constraint) - bp.constraints.emplace_back (pdb, nm.string (), *constraint); - - // Now collect this prerequisite. If it was actually collected - // (i.e., it wasn't already there) and we are forcing a downgrade - // or upgrade, then refuse for a held version, warn for a held - // package, and print the info message otherwise, unless the - // verbosity level is less than two. - // - // Note though that while the prerequisite was collected it could - // have happen because it is an optional package and so not being - // pre-collected earlier. Meanwhile the package was specified - // explicitly and we shouldn't consider that as a - // dependency-driven up/down-grade enforcement. - // - // Here is an example of the situation we need to handle properly: - // - // repo: foo/2(->bar/2), bar/0+1 - // build sys:bar/1 - // build foo ?sys:bar/2 - // - // Pass the function which verifies we don't try to force - // up/downgrade of the held version and makes sure we don't print - // the dependency chain if replace_version will be thrown. - // - // Also note that we rely on "small function object" optimization - // here. - // - struct - { - const build_package& dependent; - const prebuild& prerequisite; - } dpn {pkg, b}; - - const function<verify_package_build_function> verify ( - [&dpn, &dep_chain] (const build_package& p, bool scratch) - { - const prebuild& prq (dpn.prerequisite); - const build_package& dep (dpn.dependent); - - if (prq.force && !prq.specified_dependency) - { - // Fail if the version is held. Otherwise, warn if the - // package is held. - // - bool f (prq.selected->hold_version); - bool w (!f && prq.selected->hold_package); - - // Note that there is no sense to warn or inform the user if - // we are about to start re-collection from scratch. - // - // @@ It seems that we may still warn/inform multiple times - // about the same package if we start from scratch. The - // intermediate diagnostics can probably be irrelevant to - // the final result. - // - // Perhaps what we should do is queue the diagnostics and - // then, if the run is not scratched, issues it. And if - // it is scratched, then drop it. - // - if (f || ((w || verb >= 2) && !scratch)) - { - const version& av (p.available_version ()); - - bool u (av > prq.selected->version); - bool c (prq.dependency.constraint); - - diag_record dr; - - (f ? dr << fail : - w ? dr << warn : - dr << info) - << "package " << dep.name () << dep.db - << " dependency on " << (c ? "(" : "") << prq.dependency - << (c ? ")" : "") << " is forcing " - << (u ? "up" : "down") << "grade of " << *prq.selected - << prq.db << " to "; - - // Print both (old and new) package names in full if the - // system attribution changes. - // - if (prq.selected->system ()) - dr << p.available_name_version (); - else - dr << av; // Can't be a system version so is never wildcard. - - if (prq.selected->hold_version) - dr << info << "package version " << *prq.selected - << prq.db<< " is held"; - - if (f) - dr << info << "explicitly request version " - << (u ? "up" : "down") << "grade to continue"; - } - } - - // Don't print the "while satisfying..." chain if we are about - // to re-collect the packages. - // - if (scratch) - dep_chain.clear (); - }); - - // Note: non-recursive. - // - build_package* p ( - collect_build (options, - move (bp), - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - postponed_cfgs, - nullptr /* dep_chain */, - nullptr /* postponed_repo */, - nullptr /* postponed_alts */, - nullptr /* postponed_deps */, - nullptr /* postponed_poss */, - verify)); - - package_key dpk (b.db, b.available->id.name); - - // Do not collect prerequisites recursively for dependent - // re-evaluation. Instead, if the re-evaluation position is - // reached, collect the dependency packages to add them to the - // existing dependent's cluster. - // - if (reeval) - { - if (dp == reeval_pos) - cfg_deps.push_back (move (dpk)); - - continue; - } - - // Do not recursively collect a dependency of a dependent with - // configuration clauses, which could be this or some other - // (indicated by the presence in postponed_deps) dependent. In the - // former case if the prerequisites were prematurely collected, - // throw postpone_dependency. - // - // Note that such a dependency will be recursively collected - // directly right after the configuration negotiation (rather than - // via the dependent). - // - bool collect_prereqs (p != nullptr); - - { - build_package* bp (entered_build (dpk)); - assert (bp != nullptr); - - if (da.prefer || da.require) - { - // Indicate that the dependent with configuration clauses is - // present. - // - { - auto i (postponed_deps.find (dpk)); - - // Do not override postponements recorded during postponed - // collection phase with those recorded during initial - // phase. - // - if (i == postponed_deps.end ()) - { - postponed_deps.emplace (dpk, - postponed_dependency { - false /* without_config */, - true /* with_config */, - initial_collection}); - } - else - i->second.with_config = true; - } - - // Prematurely collected before we saw any config clauses. - // - if (bp->recursive_collection && - postponed_cfgs.find_dependency (dpk) == nullptr) - { - l5 ([&]{trace << "cannot cfg-postpone dependency " - << bp->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db () - << " (collected prematurely), " - << "throwing postpone_dependency";}); - - // Don't print the "while satisfying..." chain. - // - dep_chain.clear (); - - throw postpone_dependency (move (dpk)); - } - - // Postpone until (re-)negotiation. - // - l5 ([&]{trace << "cfg-postpone dependency " - << bp->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db ();}); - - cfg_deps.push_back (move (dpk)); - - collect_prereqs = false; - } - else - { - // Indicate that the dependent without configuration clauses - // is also present. - // - auto i (postponed_deps.find (dpk)); - if (i != postponed_deps.end ()) - { - l5 ([&]{trace << "dep-postpone dependency " - << bp->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db ();}); - - i->second.wout_config = true; - - collect_prereqs = false; - } - else - { - l5 ([&]{trace << "no cfg-clause for dependency " - << bp->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db ();}); - } - } - } - - if (collect_prereqs) - collect_build_prerequisites (options, - *p, - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - dep_chain, - postponed_repo, - postponed_alts, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); + swap (c, *a.constraint); } - - // If this dependent has any dependencies with configurations - // clauses, then we need to deal with that. - // - // This is what we refer to as the "up-negotiation" where we - // negotiate the configuration of dependents that could not be - // postponed and handled all at once during "initial negotiation" in - // collect_build_postponed(). - // - if (!cfg_deps.empty ()) + else { - // Re-evaluation is a special case (it happens during cluster - // negotiation; see collect_build_postponed()). - // - if (reeval) - { - reevaluated = true; - - // Note: the dependent may already exist in the cluster with a - // subset of dependencies. - // - postponed_configuration& cfg ( - postponed_cfgs.add (pk, - true /* existing */, - dp, - cfg_deps).first); - - // Can we merge clusters as a result? Seems so. - // - // - Simple case is if the cluster(s) being merged are not - // negotiated. Then perhaps we could handle this via the same - // logic that handles the addition of extra dependencies. - // - // - For the complex case, perhaps just making the resulting - // cluster shadow and rolling back, just like in the other - // case (non-existing dependent). - // - // Note: this is a special case of the below more general logic. - // - // Also note that we can distinguish the simple case by the fact - // that the resulting cluster is not negotiated. Note however, - // that in this case it is guaranteed that all the involved - // clusters will be merged into the cluster which the being - // re-evaluated dependent belongs to since this cluster (while - // not being negotiated) already has non-zero depth (see - // collect_build_postponed() for details). - // - assert (cfg.depth != 0); - - if (cfg.negotiated) - { - l5 ([&]{trace << "re-evaluating dependent " - << pkg.available_name_version_db () - << " involves negotiated configurations and " - << "results in " << cfg << ", throwing " - << "merge_configuration";}); - - // Don't print the "while satisfying..." chain. - // - dep_chain.clear (); - - throw merge_configuration {cfg.depth}; - } - - l5 ([&]{trace << "re-evaluating dependent " - << pkg.available_name_version_db () - << " results in " << cfg;}); - - return false; - } - - // As a first step add this dependent/dependencies to one of the - // new/existing postponed_configuration clusters, which could - // potentially cause some of them to be merged. Here are the - // possibilities and what we should do in each case. - // - // 1. Got added to a new cluster -- this dependent got postponed - // and we return false. - // - // 2. Got added to an existing non-yet-negotiated cluster (which - // could potentially involve merging a bunch of them) -- ditto. - // - // 3. Got added to an existing already-[being]-negotiated cluster - // (which could potentially involve merging a bunch of them, - // some negotiated, some being negotiated, and some not yet - // negotiated) -- see below logic. - // - // Note that if a dependent is postponed, it will be recursively - // recollected right after the configuration negotiation. - - // Note: don't move the argument from since may be needed for - // constructing exception. - // - pair<postponed_configuration&, optional<bool>> r ( - postponed_cfgs.add (pk, false /* existing */, dp, cfg_deps)); - - postponed_configuration& cfg (r.first); - - if (cfg.depth == 0) - return false; // Cases (1) or (2). - else - { - // Case (3). - // - // There is just one complication: - // - // If all the merged clusters are already negotiated, then all - // is good: all the dependencies in cfg_deps have been collected - // recursively as part of the configuration negotiation (because - // everything in this cluster is already negotiated) and we can - // return true (no need to postpone any further steps). - // - // But if we merged clusters not yet negotiated, or, worse, - // being in the middle of negotiation, then we need to get this - // merged cluster into the fully negotiated state. The way we do - // it is by throwing merge_configuration (see below). - // - // When we are back here after throwing merge_configuration, - // then all the clusters have been pre-merged and our call to - // add() shouldn't have added any new cluster. In this case the - // cluster can either be already negotiated or being negotiated - // and we can proceed as in the "everything is negotiated case" - // above (we just need to get the the dependencies that we care - // about into the recursively collected state). - // - - // To recap, r.second values mean: - // - // absent -- shadow cluster-based merge is/being negotiated - // false -- some non or being negotiated - // true -- all have been negotiated - // - if (r.second && !*r.second) - { - // The partially negotiated case. - // - // Handling this in a straightforward way is not easy due to - // the being negotiated cases -- we have code up the stack - // that is in the middle of the negotiation logic. - // - // Another idea is to again throw to the outer try/catch frame - // (thus unwinding all the being negotiated code) and complete - // the work there. The problem with this approach is that - // without restoring the state we may end up with unrelated - // clusters that will have no corresponding try-catch frames - // (because we may unwind them in the process). - // - // So the approach we will use is the "shadow" idea for - // merging clusters. Specifically, we throw - // merge_configuration to the outer try/catch. At the catch - // site we make the newly merged cluster a shadow of the - // restored cluster and retry the same steps similar to - // retry_configuration. As we redo these steps, we consult the - // shadow cluster and if the dependent/dependency entry is - // there, then instead of adding it to another (new/existing) - // cluster that would later be merged into this non-shadow - // cluster, we add it directly to the non-shadow cluster - // (potentially merging other cluster which it feels like by - // definition should all be already fully negotiated). The end - // result is that once we reach this point again, there will - // be nothing to merge. - // - // The shadow check is part of postponed_configs::add(). - // - l5 ([&]{trace << "cfg-postponing dependent " - << pkg.available_name_version_db () - << " merges non-negotiated and/or being " - << "negotiated configurations in and results in " - << cfg << ", throwing merge_configuration";}); - - // Don't print the "while satisfying..." chain. - // - dep_chain.clear (); - - throw merge_configuration {cfg.depth}; - } - - // Up-negotiate the configuration and if it has changed, throw - // retry_configuration to the try/catch frame corresponding to - // the negotiation of the outermost merged cluster in order to - // retry the same steps (potentially refining the configuration - // as we go along) and likely (but not necessarily) ending up - // here again, at which point we up-negotiate again with the - // expectation that the configuration won't change (but if it - // does, then we throw again and do another refinement pass). - // - // In a sense, semantically, we should act like a one more - // iteration of the initial negotiation loop with the exception - // acting like a request to restart the refinement process from - // the beginning. - // - bool changed; - { - // Similar to initial negotiation, resolve package skeletons - // for this dependent and its dependencies. - // - package_skeleton* dept; - { - build_package* b (entered_build (pk)); - assert (b != nullptr && b->skeleton); - dept = &*b->skeleton; - } - - // If a dependency has already been recursively collected, - // then we can no longer call reload_defaults() or - // verify_sensible() on its skeleton. We could reset it, but - // then we wouldn't be able to continue using it if - // negotiate_configuration() below returns false. So it seems - // the most sensible approach is to make a temporary copy and - // reset that. - // - small_vector<reference_wrapper<package_skeleton>, 1> depcs; - forward_list<package_skeleton> depcs_storage; // Ref stability. - { - depcs.reserve (cfg_deps.size ()); - for (const package_key& pk: cfg_deps) - { - build_package* b (entered_build (pk)); - assert (b != nullptr); - - package_skeleton* depc; - if (b->recursive_collection) - { - assert (b->skeleton); - - depcs_storage.push_front (*b->skeleton); - depc = &depcs_storage.front (); - depc->reset (); - } - else - depc = &(b->skeleton - ? *b->skeleton - : b->init_skeleton (options)); - - depcs.push_back (*depc); - } - } - - changed = negotiate_configuration ( - cfg.dependency_configurations, *dept, dp, depcs); - } - - // If the configuration hasn't changed, then we carry on. - // Otherwise, retry the negotiation from the beginning to - // refine the resulting configuration (see the catch block - // for retry_configuration). - // - if (changed) - { - l5 ([&]{trace << "cfg-postponing dependent " - << pkg.available_name_version_db () - << " involves (being) negotiated configurations " - << "and results in " << cfg - << ", throwing retry_configuration";}); - - // Don't print the "while satisfying..." chain. - // - dep_chain.clear (); - - throw retry_configuration {cfg.depth, move (pk)}; - } - - l5 ([&]{trace << "configuration for cfg-postponed " - << "dependencies of dependent " - << pkg.available_name_version_db () << " is " - << (r.second ? "" : "shadow-") << "negotiated";}); - - // Note that even in the fully negotiated case we may still add - // extra dependencies to this cluster which we still need to - // configure and recursively collect before indicating to the - // caller (returning true) that we are done with this depends - // value and the dependent is not postponed. - // - for (const package_key& p: cfg_deps) - { - build_package* b (entered_build (p)); - assert (b != nullptr); - - // Reconfigure the configured dependencies (see - // collect_build_postponed() for details). - // - if (b->selected != nullptr && - b->selected->state == package_state::configured) - b->flags |= build_package::adjust_reconfigure; - - if (!b->recursive_collection) - { - l5 ([&]{trace << "collecting cfg-postponed dependency " - << b->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db ();}); - - // Similar to the inital negotiation case, verify and set - // the dependent configuration for this dependency. - // - { - assert (b->skeleton); // Should have been init'ed above. - - const package_configuration& pc ( - cfg.dependency_configurations[p]); - - pair<bool, string> pr (b->skeleton->available != nullptr - ? b->skeleton->verify_sensible (pc) - : make_pair (true, string ())); - - if (!pr.first) - { - diag_record dr (fail); - dr << "unable to negotiate sensible configuration for " - << "dependency " << p << '\n' - << " " << pr.second; - - dr << info << "negotiated configuration:\n"; - pc.print (dr, " "); - } - - b->skeleton->dependent_config (pc); - } - - collect_build_prerequisites (options, - *b, - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - dep_chain, - postponed_repo, - postponed_alts, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); - } - else - l5 ([&]{trace << "dependency " - << b->available_name_version_db () - << " of dependent " - << pkg.available_name_version_db () - << " is already (being) recursively " - << "collected, skipping";}); - } - - return true; - } + a.constraint = move (c); + bp.constraints.clear (); } - return true; - }; - - // Select a dependency alternative, copying it alone into the - // resulting dependencies list and evaluating its reflect clause, if - // present. - // - bool selected (false); - auto select = [&sdeps, &salts, &sdas, &skel, di, &selected] - (const dependency_alternative& da, size_t dai) + break; + } + case type::dep_existing: { - assert (sdas.empty ()); - - // Avoid copying enable/reflect not to evaluate them repeatedly. - // - sdas.emplace_back (nullopt /* enable */, - nullopt /* reflect */, - da.prefer, - da.accept, - da.require, - da /* dependencies */); - - sdeps.push_back (move (sdas)); - salts.push_back (dai); - - if (da.reflect) - skel.evaluate_reflect (*da.reflect, make_pair (di, dai)); - - selected = true; - }; - - // Postpone the prerequisite builds collection, optionally inserting - // the package to the postponements set (can potentially already be - // there) and saving the enabled alternatives. - // - auto postpone = [&pkg, &edas, &postponed] - (postponed_packages* postpones) + auto i (find_dep_pkg (a)); + assert (i != dep_pkgs_.end ()); + swap (i->constraint, a.constraint); + break; + } + case type::hold_new: { - if (postpones != nullptr) - postpones->insert (&pkg); - - pkg.postponed_dependency_alternatives = move (edas); - postponed = true; - }; - - // Iterate over the enabled dependencies and try to select a - // satisfactory alternative. - // - // If the package is already configured as source and is not - // up/downgraded, then we will try to resolve its dependencies to the - // current prerequisites. To achieve this we will first try to select - // an alternative in the "recreate dependency decisions" mode, - // filtering out all the alternatives where dependencies do not all - // belong to the list of current prerequisites. If we end up with no - // alternative selected, then we retry in the "make dependency - // decisions" mode and select the alternative ignoring the current - // prerequisites. - // - // Note though, that if we are re-evaluating an existing dependent - // then we fail if we didn't succeed in the "recreate dependency - // decisions" mode. - // - const package_prerequisites* prereqs (src_conf && !ud - ? &sp->prerequisites - : nullptr); + auto i (find_hold_pkg (a)); + assert (i != hold_pkgs_.end ()); - // During the dependent re-evaluation we always try to reproduce the - // existing setup. - // - assert (!reeval || prereqs != nullptr); - - for (;;) - { - // The index and pre-collection result of the first satisfactory - // alternative. - // - optional<pair<size_t, precollect_result>> first_alt; + build_package& bp (*i); + a.available = move (bp.available); + a.repository_fragment = move (bp.repository_fragment); - // The number of satisfactory alternatives. + // Must contain the replacement version. // - size_t alts_num (0); - - for (size_t i (0); i != edas.size (); ++i) - { - const dependency_alternative& da (edas[i].first); - - precollect_result r (precollect (da, das.buildtime, prereqs)); - - // If we didn't come up with satisfactory dependency builds, then - // skip this alternative and try the next one, unless the - // collecting is postponed in which case just bail out. - // - // Should we skip alternatives for which we are unable to satisfy - // the constraint? On one hand, this could be a user error: there - // is no package available from dependent's repositories that - // satisfies the constraint. On the other hand, it could be that - // it's other dependent's constraints that we cannot satisfy - // together with others. And in this case we may want some other - // alternative. Consider, as an example, something like this: - // - // depends: libfoo >= 2.0.0 | libfoo >= 1.0.0 libbar - // - if (!r.builds) - { - if (r.repo_postpone) - { - if (reeval) - fail_reeval (); - - postpone (nullptr); // Already inserted into postponed_repo. - break; - } + assert (!bp.constraints.empty ()); - continue; - } - - ++alts_num; + a.constraint = move (bp.constraints[0].value); - // Note that when we see the first satisfactory alternative, we - // don't know yet if it is a single alternative or the first of - // the (multiple) true alternatives (those are handled - // differently). Thus, we postpone its processing until the second - // satisfactory alternative is encountered or the end of the - // alternatives list is reached. - // - if (!first_alt) - { - first_alt = make_pair (i, move (r)); - continue; - } - - // Try to select a true alternative, returning true if the - // alternative is selected or the selection is postponed. Return - // false if the alternative is ignored (not postponed and not all - // of it dependencies are reused). - // - auto try_select = [postponed_alts, &max_alt_index, - &edas, &pkg, - reeval, - &trace, - &postpone, &collect, &select] - (size_t index, precollect_result&& r) - { - const auto& eda (edas[index]); - const dependency_alternative& da (eda.first); - size_t dai (eda.second); - - // Postpone the collection if the alternatives maximum index is - // reached. - // - if (postponed_alts != nullptr && index >= max_alt_index) - { - // For a dependent re-evaluation max_alt_index is expected to - // be max size_t. - // - assert (!reeval); - - l5 ([&]{trace << "alt-postpone dependent " - << pkg.available_name_version_db () - << " since max index is reached: " << index << - info << "dependency alternative: " << da;}); - - postpone (postponed_alts); - return true; - } - - // Select this alternative if all its dependencies are reused - // and do nothing about it otherwise. - // - if (r.reused) - { - // On the diagnostics run there shouldn't be any alternatives - // that we could potentially select. - // - assert (postponed_alts != nullptr); - - if (!collect (da, dai, move (*r.builds))) - { - postpone (nullptr); // Already inserted into postponed_cfgs. - return true; - } - - select (da, dai); - - // Make sure no more true alternatives are selected during - // this function call unless we are re-evaluating a dependent. - // - if (!reeval) - max_alt_index = 0; - - return true; - } - else - return false; - }; - - // If we encountered the second satisfactory alternative, then - // this is the "multiple true alternatives" case. In this case we - // also need to process the first satisfactory alternative, which - // processing was delayed. - // - if (alts_num == 2) - { - assert (first_alt); - - if (try_select (first_alt->first, move (first_alt->second))) - break; - } - - if (try_select (i, move (r))) - break; - - // Not all of the alternative dependencies are reused, so go to - // the next alternative. - } + hold_pkgs_.erase (i); + break; + } + case type::dep_new: + { + auto i (find_dep_pkg (a)); + assert (i != dep_pkgs_.end ()); - // Bail out if the collection is postponed for any reason. - // - if (postponed) - break; + a.constraint = move (i->constraint); - // Select the single satisfactory alternative (regardless of its - // dependencies reuse). - // - if (!selected && alts_num == 1) - { - assert (first_alt && first_alt->second.builds); + dep_pkgs_.erase (i); + break; + } + } - const auto& eda (edas[first_alt->first]); - const dependency_alternative& da (eda.first); - size_t dai (eda.second); + return a; + } - if (!collect (da, dai, move (*first_alt->second.builds))) - { - postpone (nullptr); // Already inserted into postponed_cfgs. - break; - } + // Return the specified adjustment's string representation in the + // following form: + // + // hold_existing: '<pkg>[ <constraint>][ <database>]' -> '<pkg> <constraint>' + // dep_existing: '?<pkg>[ <constraint>][ <database>]' -> '?<pkg> <constraint>' + // hold_new: '<pkg> <constraint>[ <database>]' + // dep_new: '?<pkg> <constraint>[ <database>]' + // + // Note: the adjustment is assumed to be newly created or be popped from + // the stack. + // + string + to_string (const cmdline_adjustment& a) const + { + using type = cmdline_adjustment::adjustment_type; - select (da, dai); - } + assert (a.constraint); // Since not pushed. - // If an alternative is selected, then we are done. - // - if (selected) - break; + const string& s (a.db.get ().string); - // Fail or postpone the collection if no alternative is selected, - // unless we are re-evaluating a dependent or are in the "recreate - // dependency decisions" mode. In the latter case fail for - // re-evaluation and fall back to the "make dependency decisions" - // mode and retry otherwise. - // - if (prereqs != nullptr) - { - if (reeval) - fail_reeval (); + switch (a.type) + { + case type::hold_existing: + { + string r ("'" + a.name.string ()); - prereqs = nullptr; - continue; - } + auto i (find_hold_pkg (a)); + assert (i != hold_pkgs_.end ()); - // Issue diagnostics and fail if there are no satisfactory - // alternatives. - // - if (alts_num == 0) - { - diag_record dr; - for (const auto& da: edas) - precollect (da.first, das.buildtime, nullptr /* prereqs */, &dr); + const build_package& bp (*i); + if (!bp.constraints.empty ()) + r += ' ' + bp.constraints[0].value.string (); - assert (!dr.empty ()); + if (!s.empty ()) + r += ' ' + s; - dr.flush (); - throw failed (); - } + r += "' -> '" + a.name.string () + ' ' + a.constraint->string () + + "'"; - // Issue diagnostics and fail if there are multiple alternatives - // with non-reused dependencies, unless the failure needs to be - // postponed. - // - assert (alts_num > 1); + return r; + } + case type::dep_existing: + { + string r ("'?" + a.name.string ()); - if (postponed_alts != nullptr) - { - if (verb >= 5) - { - diag_record dr (trace); - dr << "alt-postpone dependent " - << pkg.available_name_version_db () - << " due to ambiguous alternatives"; + auto i (find_dep_pkg (a)); + assert (i != dep_pkgs_.end ()); - for (const auto& da: edas) - dr << info << "alternative: " << da.first; - } + if (i->constraint) + r += ' ' + i->constraint->string (); - postpone (postponed_alts); - break; - } + if (!s.empty ()) + r += ' ' + s; - diag_record dr (fail); - dr << "unable to select dependency alternative for package " - << pkg.available_name_version_db () << - info << "explicitly specify dependency packages to manually " - << "select the alternative"; + r += "' -> '?" + a.name.string () + ' ' + a.constraint->string () + + "'"; - for (const auto& da: edas) - { - precollect_result r ( - precollect (da.first, das.buildtime, nullptr /* prereqs */)); + return r; + } + case type::hold_new: + { + assert (find_hold_pkg (a) == hold_pkgs_.end ()); - if (r.builds) - { - assert (!r.reused); // We shouldn't be failing otherwise. + string r ("'" + a.name.string () + ' ' + a.constraint->string ()); - dr << info << "alternative:"; + if (!s.empty ()) + r += ' ' + s; - // Only print the non-reused dependencies, which needs to be - // explicitly specified by the user. - // - for (const prebuild& b: *r.builds) - { - if (!b.reused) - dr << ' ' << b.dependency.name; - } - } - } + r += "'"; + return r; } + case type::dep_new: + { + assert (find_dep_pkg (a) == dep_pkgs_.end ()); - if (postponed) - break; - } + string r ("'?" + a.name.string () + ' ' + a.constraint->string ()); - if (reeval) - { - if (!reevaluated) - fail_reeval (); + if (!s.empty ()) + r += ' ' + s; - assert (postponed); + r += "'"; + return r; + } } - dep_chain.pop_back (); - - l5 ([&]{trace << (!postponed ? "end " : - reeval ? "re-evaluated " : - "postpone ") - << pkg.available_name_version_db ();}); + assert (false); // Can't be here. + return ""; } - void - collect_build_prerequisites (const pkg_build_options& o, - database& db, - const package_name& name, - const function<find_database_function>& fdb, - const repointed_dependents& rpt_depts, - const function<add_priv_cfg_function>& apc, - bool initial_collection, - replaced_versions& replaced_vers, - postponed_packages& postponed_repo, - postponed_packages& postponed_alts, - size_t max_alt_index, - postponed_dependencies& postponed_deps, - postponed_configurations& postponed_cfgs, - postponed_positions& postponed_poss) + // Return true, if there are no adjustments in the stack. + // + bool + empty () const { - auto mi (map_.find (db, name)); - assert (mi != map_.end ()); - - build_package_refs dep_chain; - - collect_build_prerequisites (o, - mi->second.package, - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - dep_chain, - &postponed_repo, - &postponed_alts, - max_alt_index, - postponed_deps, - postponed_cfgs, - postponed_poss); + return adjustments_.empty (); } - // Collect the repointed dependents and their replaced prerequisites, - // recursively. - // - // If a repointed dependent is already pre-entered or collected with an - // action other than adjustment, then just mark it for reconfiguration - // unless it is already implied. Otherwise, collect the package build with - // the repoint sub-action and reconfigure adjustment flag. + // Return true, if push() has been called at least once. // - void - collect_repointed_dependents ( - const pkg_build_options& o, - const repointed_dependents& rpt_depts, - replaced_versions& replaced_vers, - postponed_packages& postponed_repo, - postponed_packages& postponed_alts, - postponed_dependencies& postponed_deps, - postponed_configurations& postponed_cfgs, - postponed_positions& postponed_poss, - const function<find_database_function>& fdb, - const function<add_priv_cfg_function>& apc) + bool + tried () const { - for (const auto& rd: rpt_depts) - { - database& db (rd.first.db); - const package_name& nm (rd.first.name); - - auto i (map_.find (db, nm)); - if (i != map_.end ()) - { - build_package& b (i->second.package); - - if (!b.action || *b.action != build_package::adjust) - { - if (!b.action || - (*b.action != build_package::drop && !b.reconfigure ())) - b.flags |= build_package::adjust_reconfigure; - - continue; - } - } - - shared_ptr<selected_package> sp (db.load<selected_package> (nm)); - - // The repointed dependent can be an orphan, so just create the - // available package from the selected package. - // - auto rp (make_available_fragment (o, db, sp)); - - // Add the prerequisite replacements as the required-by packages. - // - set<package_key> required_by; - for (const auto& prq: rd.second) - { - if (prq.second) // Prerequisite replacement? - { - const package_key& pk (prq.first); - required_by.emplace (pk.db, pk.name); - } - } - - build_package p { - build_package::build, - db, - sp, - move (rp.first), - move (rp.second), - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - sp->system (), - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - move (required_by), // Required by (dependencies). - false, // Required by dependents. - build_package::adjust_reconfigure | build_package::build_repoint}; - - build_package_refs dep_chain; - - // Note: recursive. - // - collect_build (o, - move (p), - fdb, - rpt_depts, - apc, - true /* initial_collection */, - replaced_vers, - postponed_cfgs, - &dep_chain, - &postponed_repo, - &postponed_alts, - &postponed_deps, - &postponed_poss); - } + return !former_states_.empty (); } - // Collect the package being dropped. - // - // Add entry to replaced_vers and throw replace_version if the existing - // version needs to be dropped but this can't be done in-place (see - // replaced_versions for details). + // Return the number of adjustments in the stack. // - void - collect_drop (const pkg_build_options& options, - database& db, - shared_ptr<selected_package> sp, - replaced_versions& replaced_vers) + size_t + size () const { - tracer trace ("collect_drop"); + return adjustments_.size (); + } - package_key pk (db, sp->name); + // Return true if replacing a package build with the specified version + // will result in a command line which has already been (unsuccessfully) + // tried as a starting point for the package builds re-collection. + // + bool + tried_earlier (database& db, const package_name& n, const version& v) const + { + if (former_states_.empty ()) + return false; - // If there is an entry for building specific version of the package - // (the available member is not NULL), then it wasn't created to prevent - // out drop (see replaced_versions for details). This rather mean that - // the replacement version is not being built anymore due to the plan - // refinement. Thus, just erase the entry in this case and continue. + // Similar to the state() function, calculate the checksum over the + // packages set, but also consider the specified package version as if + // it were present in the set. // - auto vi (replaced_vers.find (pk)); - if (vi != replaced_vers.end () && !vi->second.replaced) - { - replaced_version& v (vi->second); - const shared_ptr<available_package>& ap (v.available); + // Note that the specified package version may not be in the set, since + // we shouldn't be trying to replace with the package version which is + // already in the command line. + // + sha256 cs; - if (ap != nullptr) - { - if (verb >= 5) - { - bool s (v.system); - const version& av (s ? *ap->system_version (db) : ap->version); + auto lt = [&db, &n, &v] (const package_version_key& pvk) + { + if (int r = n.compare (pvk.name)) + return r < 0; - l5 ([&]{trace << "erase version replacement for " - << package_string (ap->id.name, av, s) << db;}); - } + if (int r = v.compare (*pvk.version)) + return r < 0; - replaced_vers.erase (vi); - vi = replaced_vers.end (); // Keep it valid for the below check. - } - else - v.replaced = true; - } + return db < pvk.db; + }; - build_package p { - build_package::drop, - db, - move (sp), - nullptr, - nullptr, - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - false, // System package. - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {}, // Required by. - false, // Required by dependents. - 0}; // State flags. - - auto i (map_.find (pk)); - - if (i != map_.end ()) + bool appended (false); + for (const package_version_key& p: packages_) { - build_package& bp (i->second.package); + assert (p.version); // Only the real packages can be here. - if (bp.available != nullptr) + if (!appended && lt (p)) { - // Similar to the version replacement in collect_build(), see if - // in-place drop is possible (no dependencies, etc) and set scratch - // to false if that's the case. - // - bool scratch (true); - - // While checking if the package has any dependencies skip the - // toolchain build-time dependencies since they should be quite - // common. - // - if (!has_dependencies (options, bp.available->dependencies)) - scratch = false; + cs.append (db.config.string ()); + cs.append (n.string ()); + cs.append (v.string ()); - l5 ([&]{trace << bp.available_name_version_db () - << " package version needs to be replaced " - << (!scratch ? "in-place " : "") << "with drop";}); - - if (scratch) - { - if (vi != replaced_vers.end ()) - vi->second = replaced_version (); - else - replaced_vers.emplace (move (pk), replaced_version ()); - - throw replace_version (); - } + appended = true; } - // Overwrite the existing (possibly pre-entered, adjustment, or - // repoint) entry. - // - l4 ([&]{trace << "overwrite " << pk;}); - - bp = move (p); + cs.append (p.db.get ().config.string ()); + cs.append (p.name.string ()); + cs.append (p.version->string ()); } - else - { - l4 ([&]{trace << "add " << pk;}); - map_.emplace (move (pk), data_type {end (), move (p)}); + if (!appended) + { + cs.append (db.config.string ()); + cs.append (n.string ()); + cs.append (v.string ()); } + + return former_states_.find (cs.string ()) != former_states_.end (); } - // Collect the package being unheld. + private: + // Return the SHA256 checksum of the current command line state. // - void - collect_unhold (database& db, const shared_ptr<selected_package>& sp) + string + state () const { - auto i (map_.find (db, sp->name)); - - // Currently, it must always be pre-entered. + // NOTE: remember to update tried_earlier() if changing anything here. // - assert (i != map_.end ()); - - build_package& bp (i->second.package); - - if (!bp.action) // Pre-entered. + sha256 cs; + for (const package_version_key& p: packages_) { - build_package p { - build_package::adjust, - db, - sp, - nullptr, - nullptr, - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - sp->system (), - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {}, // Required by. - false, // Required by dependents. - build_package::adjust_unhold}; - - p.merge (move (bp)); - bp = move (p); + assert (p.version); // Only the real packages can be here. + + cs.append (p.db.get ().config.string ()); + cs.append (p.name.string ()); + cs.append (p.version->string ()); } - else - bp.flags |= build_package::adjust_unhold; + + return cs.string (); } - void - collect_build_postponed (const pkg_build_options& o, - replaced_versions& replaced_vers, - postponed_packages& postponed_repo, - postponed_packages& postponed_alts, - postponed_dependencies& postponed_deps, - postponed_configurations& postponed_cfgs, - strings& postponed_cfgs_history, - postponed_positions& postponed_poss, - const function<find_database_function>& fdb, - const repointed_dependents& rpt_depts, - const function<add_priv_cfg_function>& apc, - postponed_configuration* pcfg = nullptr) + // Find the command line package spec an adjustment applies to. + // + vector<build_package>::iterator + find_hold_pkg (const cmdline_adjustment& a) const { - // Snapshot of the package builds collection state. - // - // Note: should not include postponed_cfgs_history. - // - class snapshot - { - public: - snapshot (const build_packages& pkgs, - const postponed_packages& postponed_repo, - const postponed_packages& postponed_alts, - const postponed_dependencies& postponed_deps, - const postponed_configurations& postponed_cfgs) - : pkgs_ (pkgs), - postponed_deps_ (postponed_deps), - postponed_cfgs_ (postponed_cfgs) - { - auto save = [] (vector<package_key>& d, - const postponed_packages& s) - { - d.reserve (s.size ()); - - for (const build_package* p: s) - d.emplace_back (p->db, p->name ()); - }; - - save (postponed_repo_, postponed_repo); - save (postponed_alts_, postponed_alts); - } - - void - restore (build_packages& pkgs, - postponed_packages& postponed_repo, - postponed_packages& postponed_alts, - postponed_dependencies& postponed_deps, - postponed_configurations& postponed_cfgs) - { - pkgs = move (pkgs_); - postponed_cfgs = move (postponed_cfgs_); - postponed_deps = move (postponed_deps_); - - auto restore = [&pkgs] (postponed_packages& d, - const vector<package_key>& s) - { - d.clear (); - - for (const package_key& p: s) - { - build_package* b (pkgs.entered_build (p)); - assert (b != nullptr); - d.insert (b); - } - }; - - restore (postponed_repo, postponed_repo_); - restore (postponed_alts, postponed_alts_); - } - - private: - // Note: try to use vectors instead of sets for storage to save - // memory. We could probably optimize this some more if - // necessary (there are still sets/maps inside). - // - build_packages pkgs_; - vector<package_key> postponed_repo_; - vector<package_key> postponed_alts_; - postponed_dependencies postponed_deps_; - postponed_configurations postponed_cfgs_; - }; - - // This exception is thrown if negotiation of the current cluster needs - // to be skipped until later. This is normally required if this cluster - // contains some existing dependent which needs to be re-evaluated to a - // dependency position greater than some other not yet negotiated - // cluster will re-evaluate this dependent to. Sometimes this another - // cluster yet needs to be created in which case the exception carries - // the information required for that (see the postponed_position's - // replace flag for details). - // - struct skip_configuration - { - optional<existing_dependent> dependent; - pair<size_t, size_t> new_position; - - skip_configuration () = default; - - skip_configuration (existing_dependent&& d, pair<size_t, size_t> n) - : dependent (move (d)), new_position (n) {} - }; - - size_t depth (pcfg != nullptr ? pcfg->depth : 0); - - string t ("collect_build_postponed (" + to_string (depth) + ")"); - tracer trace (t.c_str ()); - - string trace_suffix (verb >= 5 && pcfg != nullptr - ? " " + pcfg->string () - : ""); - - l5 ([&]{trace << "begin" << trace_suffix;}); - - if (pcfg != nullptr) - { - // This is what we refer to as the "initial negotiation" where we - // negotiate the configuration of dependents that could be postponed. - // Those that could not we "up-negotiate" in the collect() lambda of - // collect_build_prerequisites(). - // - using packages = postponed_configuration::packages; - - assert (!pcfg->negotiated); - - // Re-evaluate existing dependents with configuration clause for - // dependencies in this configuration cluster up to these - // dependencies. Omit dependents which are already being built or - // dropped. Note that these dependents, potentially with additional - // dependencies, will be added to this cluster with the `existing` - // flag as a part of the dependents' re-evaluation (see the collect - // lambda in collect_build_prerequisites() for details). - // - // After being re-evaluated the existing dependents are recursively - // collected in the same way as the new dependents. - // - { - // Map existing dependents to the dependencies they apply a - // configuration to. Also, collect the information which is required - // for a dependent re-evaluation and its subsequent recursive - // collection (selected package, etc). - // - // As mentioned earlier, we may end up adding additional - // dependencies to pcfg->dependencies which in turn may have - // additional existing dependents which we need to process. Feels - // like doing this iteratively is the best option. - // - // Note that we need to make sure we don't re-process the same - // existing dependents. - // - struct existing_dependent_ex: existing_dependent - { - packages dependencies; - bool reevaluated = false; - - existing_dependent_ex (existing_dependent&& ed) - : existing_dependent (move (ed)) {} - }; - map<package_key, existing_dependent_ex> dependents; - - const packages& deps (pcfg->dependencies); - - // Note that the below collect_build_prerequisites() call can only - // add new dependencies to the end of the cluster's dependencies - // list. Thus on each iteration we will only add existing dependents - // of unprocessed/new dependencies. We will also skip the already - // re-evaluated existing dependents. - // - for (size_t i (0); i != deps.size (); ) - { - size_t n (dependents.size ()); - - for (; i != deps.size (); ++i) - { - // Note: this reference is only used while deps is unchanged. - // - const package_key& p (deps[i]); - - // If the dependent is being built, then check if it was - // re-evaluated to the position greater than the dependency - // position. Return true if that's the case, so this package is - // added to the resulting list and we can handle this situation. - // - // Note that we rely on "small function object" optimization - // here. - // - const function<verify_dependent_build_function> verify ( - [&postponed_cfgs, pcfg] - (const package_key& pk, pair<size_t, size_t> pos) - { - for (const postponed_configuration& cfg: postponed_cfgs) - { - if (&cfg == pcfg || cfg.negotiated) - { - if (const pair<size_t, size_t>* p = - cfg.existing_dependent_position (pk)) + return find_if (hold_pkgs_.begin (), hold_pkgs_.end (), + [&a] (const build_package& p) { - if (p->first > pos.first) - return true; - } - } - } - - return false; - }); - - for (existing_dependent& ed: - query_existing_dependents (trace, - p.db, - p.name, - replaced_vers, - rpt_depts, - verify)) - { - package_key pk (ed.db, ed.selected->name); - - // If this dependent is present in postponed_deps, then it - // means someone depends on it with configuration and it's no - // longer considered an existing dependent (it will be - // reconfigured). However, this fact may not be reflected yet. - // And it can actually turn out bogus. - // - auto pi (postponed_deps.find (pk)); - if (pi != postponed_deps.end ()) - { - l5 ([&]{trace << "skip dep-postponed existing dependent " - << pk << " of dependency " << p;}); - - // Note that here we would re-evaluate the existing dependent - // without specifying any configuration for it. - // - pi->second.wout_config = true; - - continue; - } - - auto i (dependents.find (pk)); - size_t di (ed.dependency_position.first); - - // Skip re-evaluated dependent if the dependency index is - // greater than the one we have already re-evaluated to. If it - // is earlier, then add the entry to postponed_poss and throw - // postpone_position to recollect from scratch. Note that this - // entry in postponed_poss is with replacement. - // - if (i != dependents.end () && i->second.reevaluated) - { - size_t ci (i->second.dependency_position.first); - - if (di > ci) - continue; - - // The newly-introduced dependency must belong to the - // depends value other then the one we have re-evaluated to. - // - assert (di < ci); - - postponed_position pp (ed.dependency_position, - true /* replace */); - - auto p (postponed_poss.emplace (pk, pp)); - - if (!p.second) - { - assert (p.first->second > pp); - p.first->second = pp; - } - - l5 ([&]{trace << "cannot re-evaluate dependent " - << pk << " to dependency index " << di - << " since it is already re-evaluated to " - << "greater index " << ci << " in " << *pcfg - << ", throwing postpone_position";}); - - throw postpone_position (); - } - - // If the existing dependent is not in the map yet, then add - // it. Otherwise, if the dependency position is greater than - // that one in the existing map entry then skip it (this - // position will be up-negotiated, if it's still present). - // Otherwise, if the position is less then overwrite the - // existing entry. Otherwise (the position is equal), just - // add the dependency to the existing entry. - // - // Note that we want to re-evaluate the dependent up to the - // earliest dependency position and continue with the regular - // prerequisites collection (as we do for new dependents) - // afterwards. - // - if (i == dependents.end ()) - { - i = dependents.emplace ( - move (pk), existing_dependent_ex (move (ed))).first; - } - else - { - size_t ci (i->second.dependency_position.first); - - if (ci < di) - continue; - else if (ci > di) - i->second = existing_dependent_ex (move (ed)); - //else if (ci == di) - // ; - } - - i->second.dependencies.push_back (p); - } - } - - // Re-evaluate the newly added existing dependents, if any. - // - if (dependents.size () != n) - { - l5 ([&]{trace << "re-evaluate existing dependents for " << *pcfg;}); - - for (auto& d: dependents) - { - existing_dependent_ex& ed (d.second); - - // Skip re-evaluated. - // - if (ed.reevaluated) - continue; - - size_t di (ed.dependency_position.first); - const package_key& pk (d.first); - - // Check if there is an earlier dependency position for this - // dependent that will be participating in a configuration - // negotiation and skip this cluster if that's the case. There - // are two places to check: postponed_poss and other clusters. - // - auto pi (postponed_poss.find (pk)); - if (pi != postponed_poss.end () && pi->second.first < di) - { - l5 ([&]{trace << "pos-postpone existing dependent " - << pk << " re-evaluation to dependency " - << "index " << di << " due to recorded index " - << pi->second.first << ", skipping " << *pcfg;}); - - pi->second.skipped = true; - - // If requested, override the first encountered non-replace - // position to replace (see below for details). - // - if (!pi->second.replace && postponed_poss.replace) - { - pi->second.replace = true; - postponed_poss.replace = false; - } - - if (pi->second.replace) - throw skip_configuration (move (ed), pi->second); - else - throw skip_configuration (); - } - - // The other clusters check is a bit more complicated: if the - // other cluster (with the earlier position) is not yet - // negotiated, then we skip. Otherwise, we have to add an - // entry to postponed_poss and backtrack. - // - bool skip (false); - for (const postponed_configuration& cfg: postponed_cfgs) - { - // Skip the current cluster. - // - if (&cfg == pcfg) - continue; - - if (const pair<size_t, size_t>* p = - cfg.existing_dependent_position (pk)) - { - size_t ei (p->first); // Other position. + return p.name () == a.name && p.db == a.db; + }); + } - if (!cfg.negotiated) - { - if (ei < di) + dependency_packages::iterator + find_dep_pkg (const cmdline_adjustment& a) const + { + return find_if (dep_pkgs_.begin (), dep_pkgs_.end (), + [&a] (const dependency_package& p) { - l5 ([&]{trace << "cannot re-evaluate dependent " - << pk << " to dependency index " << di - << " due to earlier dependency index " - << ei << " in " << cfg << ", skipping " - << *pcfg;}); + return p.name == a.name && + p.db != nullptr && + *p.db == a.db; + }); + } - skip = true; - } - } - else - { - // If this were not the case, then this dependent - // wouldn't have been considered as an existing by - // query_existing_dependents() since as it is (being) - // negotiated then it is already re-evaluated and so is - // being built (see the verify lambda above). - // - assert (ei > di); + private: + vector<build_package>& hold_pkgs_; + dependency_packages& dep_pkgs_; - // Feels like there cannot be an earlier position. - // - postponed_position pp (ed.dependency_position, - false /* replace */); + vector<cmdline_adjustment> adjustments_; // Adjustments stack. + set<package_version_key> packages_; // Replacements. + set<string> former_states_; // Command line seen states. + }; - auto p (postponed_poss.emplace (pk, pp)); - if (!p.second) - { - assert (p.first->second > pp); - p.first->second = pp; - } + // Try to replace a collected package with a different available version, + // satisfactory for all its new and/or existing dependents. Return the + // command line adjustment if such a replacement is deduced and nullopt + // otherwise. In the latter case, also return the list of the being built + // dependents which are unsatisfied by some of the dependency available + // versions (unsatisfied_dpts argument). + // + // Specifically, try to find the best available package version considering + // all the imposed constraints as per unsatisfied_dependents description. If + // succeed, return the command line adjustment reflecting the replacement. + // + // Notes: + // + // - Doesn't perform the actual adjustment of the command line. + // + // - Expected to be called after the execution plan is fully refined. That, + // in particular, means that all the existing dependents are also + // collected and thus the constraints they impose are already in their + // dependencies' constraints lists. + // + // - The specified package version may or may not be satisfactory for its + // new and existing dependents. + // + // - The replacement is denied in the following cases: + // + // - If it turns out that the package have been specified on the command + // line (by the user or by us on some previous iteration) with an exact + // version constraint, then we cannot try any other version. + // + // - If the dependency is system, then it is either specified with the + // wildcard version or its exact version have been specified by the user + // or have been deduced by the system package manager. In the former + // case we actually won't be calling this function for this package + // since the wildcard version satisfies any constraint. Thus, an exact + // version has been specified/deduced for this dependency and so we + // cannot try any other version. + // + // - If the dependency is being built as an existing archive/directory, + // then its version is determined and so we cannot try any other + // version. + // + // - If the package is already configured with the version held and the + // user didn't specify this package on the command line and it is not + // requested to be upgraded, patched, and/or deorphaned, then we + // shouldn't be silently up/down-grading it. + // + optional<cmdline_adjustment> + try_replace_dependency (const common_options& o, + const build_package& p, + const build_packages& pkgs, + const vector<build_package>& hold_pkgs, + const dependency_packages& dep_pkgs, + const cmdline_adjustments& cmdline_adjs, + vector<package_key>& unsatisfied_dpts, + const char* what) + { + tracer trace ("try_replace_dependency"); - l5 ([&]{trace << "cannot re-evaluate dependent " - << pk << " to dependency index " << di - << " due to greater dependency " - << "index " << ei << " in " << cfg - << ", throwing postpone_position";}); + assert (p.available != nullptr); // By definition. - throw postpone_position (); - } - } - } + // Bail out for the system package build. + // + if (p.system) + { + l5 ([&]{trace << "replacement of " << what << " version " + << p.available_name_version_db () << " is denied " + << "since it is being configured as system";}); - if (skip) - throw skip_configuration (); + return nullopt; + } - // Finally, re-evaluate the dependent. - // - packages& ds (ed.dependencies); - - pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> rp ( - find_available_fragment (o, pk.db, ed.selected)); - - build_package p { - build_package::build, - pk.db, - move (ed.selected), - move (rp.first), - move (rp.second), - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - false, // System. - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - set<package_key> ( // Required by (dependency). - ds.begin (), ds.end ()), - false, // Required by dependents. - build_package::adjust_reconfigure | - build_package::build_reevaluate}; + // Bail out for an existing package archive/directory. + // + database& db (p.db); + const package_name& nm (p.name ()); + const version& ver (p.available->version); - // Note: not recursive. - // - collect_build (o, - move (p), - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - postponed_cfgs); - - build_package* b (entered_build (pk)); - assert (b != nullptr); - - // Re-evaluate up to the earliest position. - // - assert (ed.dependency_position.first != 0); + if (find_existing (db, + nm, + nullopt /* version_constraint */).first != nullptr) + { + l5 ([&]{trace << "replacement of " << what << " version " + << p.available_name_version_db () << " is denied since " + << "it is being built as existing archive/directory";}); - build_package_refs dep_chain; - collect_build_prerequisites (o, - *b, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - &postponed_repo, - &postponed_alts, - numeric_limits<size_t>::max (), - postponed_deps, - postponed_cfgs, - postponed_poss, - ed.dependency_position); - - ed.reevaluated = true; - - if (pi != postponed_poss.end ()) - { - // Otherwise we should have thrown skip_configuration above. - // - assert (di <= pi->second.first); + return nullopt; + } - pi->second.reevaluated = true; - } - } - } - } - } + // Find the package command line entry and stash the reference to its + // version constraint, if any. Bail out if the constraint is specified as + // an exact package version. + // + const build_package* hold_pkg (nullptr); + const dependency_package* dep_pkg (nullptr); + const version_constraint* constraint (nullptr); - l5 ([&]{trace << "cfg-negotiate begin " << *pcfg;}); + for (const build_package& hp: hold_pkgs) + { + if (hp.name () == nm && hp.db == db) + { + hold_pkg = &hp; - // Negotiate the configuration. - // - // The overall plan is as follows: continue refining the configuration - // until there are no more changes by giving each dependent a chance - // to make further adjustments. - // - for (auto b (pcfg->dependents.begin ()), - i (b), - e (pcfg->dependents.end ()); i != e; ) + if (!hp.constraints.empty ()) { - // Resolve package skeletons for the dependent and its dependencies. - // - // For the dependent, the skeleton should be already there (since we - // should have started recursively collecting it). For a dependency, - // it should not already be there (since we haven't yet started - // recursively collecting it). But we could be re-resolving the same - // dependency multiple times. + // Can only contain the user-specified constraint. // - package_skeleton* dept; - { - build_package* b (entered_build (i->first)); - assert (b != nullptr && b->skeleton); - dept = &*b->skeleton; - } - - pair<size_t, size_t> pos; - small_vector<reference_wrapper<package_skeleton>, 1> depcs; - { - // A non-negotiated cluster must only have one depends position - // for each dependent. - // - assert (i->second.dependencies.size () == 1); - - const postponed_configuration::dependency& ds ( - i->second.dependencies.front ()); - - pos = ds.position; - - depcs.reserve (ds.size ()); - for (const package_key& pk: ds) - { - build_package* b (entered_build (pk)); - assert (b != nullptr); - - depcs.push_back (b->skeleton - ? *b->skeleton - : b->init_skeleton (o /* options */)); - } - } - - if (negotiate_configuration ( - pcfg->dependency_configurations, *dept, pos, depcs)) - { - if (i != b) - { - i = b; // Restart from the beginning. - continue; - } - } - - ++i; - } - - // Being negotiated (so can only be up-negotiated). - // - pcfg->negotiated = false; - - // Note that we can be adding new packages to the being negotiated - // cluster by calling collect_build_prerequisites() for its - // dependencies and dependents. Thus, we need to stash the current - // list of dependencies and dependents and iterate over them. - // - // Note that whomever is adding new packages is expected to process - // them (they may also process existing packages, which we are - // prepared to ignore). - // - packages dependencies (pcfg->dependencies); - - packages dependents; - dependents.reserve (pcfg->dependents.size ()); + assert (hp.constraints.size () == 1); - for (const auto& p: pcfg->dependents) - dependents.push_back (p.first); + const version_constraint& c (hp.constraints[0].value); - // Process dependencies recursively with this config. - // - // Note that there could be inter-dependecies between these packages, - // which means the configuration can only be up-negotiated. - // - l5 ([&]{trace << "recursively collect cfg-negotiated dependencies";}); - - for (const package_key& p: dependencies) - { - build_package* b (entered_build (p)); - assert (b != nullptr); - - // Reconfigure the configured dependencies. - // - // Note that potentially this can be an overkill if the dependency - // configuration doesn't really change. Later we can implement some - // precise detection for that using configuration checksum or - // similar. - // - // Also note that for configured dependents which belong to the - // configuration cluster this flag is already set (see above). - // - if (b->selected != nullptr && - b->selected->state == package_state::configured) - b->flags |= build_package::adjust_reconfigure; - - // Skip the dependencies which are already collected recursively. - // - if (!b->recursive_collection) + if (c.min_version == c.max_version) { - // Verify and set the dependent configuration for this dependency. - // - // Note: see similar code for the up-negotiation case. - // - { - assert (b->skeleton); // Should have been init'ed above. - - const package_configuration& pc ( - pcfg->dependency_configurations[p]); - - // Skip the verification if this is a system package - // without skeleton info. - // - pair<bool, string> pr (b->skeleton->available != nullptr - ? b->skeleton->verify_sensible (pc) - : make_pair (true, string ())); - - if (!pr.first) - { - // Note that the diagnostics from the dependency will most - // likely be in the "error ..." form (potentially with - // additional info lines) and by printing it with a two-space - // indentation we make it "fit" into our diag record. - // - diag_record dr (fail); - dr << "unable to negotiate sensible configuration for " - << "dependency " << p << '\n' - << " " << pr.second; + l5 ([&]{trace << "replacement of " << what << " version " + << p.available_name_version_db () << " is denied " + << "since it is specified on command line as '" + << nm << ' ' << c << "'";}); - dr << info << "negotiated configuration:\n"; - pc.print (dr, " "); // Note 4 spaces since in nested info. - } - - b->skeleton->dependent_config (pc); - } - - build_package_refs dep_chain; - collect_build_prerequisites (o, - *b, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - &postponed_repo, - &postponed_alts, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); + return nullopt; } else - l5 ([&]{trace << "dependency " << b->available_name_version_db () - << " is already (being) recursively collected, " - << "skipping";}); + constraint = &c; } - // Continue processing dependents with this config. - // - l5 ([&]{trace << "recursively collect cfg-negotiated dependents";}); + break; + } + } - for (const auto& p: dependents) + if (hold_pkg == nullptr) + { + for (const dependency_package& dp: dep_pkgs) + { + if (dp.name == nm && dp.db != nullptr && *dp.db == db) { - // Select the dependency alternative for which configuration has - // been negotiated and collect this dependent starting from the next - // depends value. - // - build_package* b (entered_build (p)); + dep_pkg = &dp; - // We should have been started recursively collecting the dependent - // and it should have been postponed. - // - assert (b != nullptr && - b->available != nullptr && - b->dependencies && - b->skeleton && - b->postponed_dependency_alternatives); - - // Select the dependency alternative (evaluate reflect if present, - // etc) and position to the next depends value (see - // collect_build_prerequisites() for details). - // + if (dp.constraint) { - const bpkg::dependencies& deps (b->available->dependencies); - bpkg::dependencies& sdeps (*b->dependencies); - vector<size_t>& salts (*b->alternatives); - - size_t di (sdeps.size ()); + const version_constraint& c (*dp.constraint); - // Skip the dependent if it has been already collected as some - // package's dependency or some such. - // - if (di == deps.size ()) + if (c.min_version == c.max_version) { - l5 ([&]{trace << "dependent " << b->available_name_version_db () - << " is already recursively collected, skipping";}); + l5 ([&]{trace << "replacement of " << what << " version " + << p.available_name_version_db () << " is denied " + << "since it is specified on command line as '?" + << nm << ' ' << c << "'";}); - continue; + return nullopt; } - - l5 ([&]{trace << "select cfg-negotiated dependency alternative " - << "for dependent " - << b->available_name_version_db ();}); - - // Find the postponed dependency alternative. - // - auto i (pcfg->dependents.find (p)); - - assert (i != pcfg->dependents.end () && - i->second.dependencies.size () == 1); - - pair<size_t, size_t> dp (i->second.dependencies[0].position); - assert (dp.first == sdeps.size () + 1); - - build_package::dependency_alternatives_refs pdas ( - move (*b->postponed_dependency_alternatives)); - - b->postponed_dependency_alternatives = nullopt; - - auto j (find_if (pdas.begin (), pdas.end (), - [&dp] (const auto& da) - { - return da.second + 1 == dp.second; - })); - - assert (j != pdas.end ()); - - const dependency_alternative& da (j->first); - size_t dai (j->second); - - // Select the dependency alternative and position to the next - // depends value. - // - const dependency_alternatives_ex& das (deps[di]); - dependency_alternatives_ex sdas (das.buildtime, das.comment); - - sdas.emplace_back (nullopt /* enable */, - nullopt /* reflect */, - da.prefer, - da.accept, - da.require, - da /* dependencies */); - - sdeps.push_back (move (sdas)); - salts.push_back (dai); - - // Evaluate reflect, if present. - // - if (da.reflect) - b->skeleton->evaluate_reflect (*da.reflect, make_pair (di, dai)); + else + constraint = &c; } - // Continue recursively collecting the dependent. - // - build_package_refs dep_chain; - - collect_build_prerequisites ( - o, - *b, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - &postponed_repo, - &postponed_alts, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); + break; } - - // Negotiated (so can only be rolled back). - // - pcfg->negotiated = true; - - l5 ([&]{trace << "cfg-negotiate end " << *pcfg;}); - - // Fall through (to start another iteration of the below loop). } + } - // Try collecting postponed packages for as long as we are making - // progress. - // - vector<build_package*> spas; // Reuse. - - for (bool prog (!postponed_repo.empty () || - !postponed_cfgs.negotiated () || - !postponed_alts.empty () || - postponed_deps.has_bogus ()); - prog; ) - { - postponed_packages prs; - postponed_packages pas; - - // Try to collect the repository-related postponments first. - // - for (build_package* p: postponed_repo) - { - l5 ([&]{trace << "collect rep-postponed " - << p->available_name_version_db ();}); - - build_package_refs dep_chain; - - collect_build_prerequisites (o, - *p, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - &prs, - &pas, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); - } - - // Save the potential new dependency alternative-related postponements. - // - postponed_alts.insert (pas.begin (), pas.end ()); - - prog = (prs != postponed_repo); - - if (prog) - { - postponed_repo.swap (prs); - continue; - } - - // Now, as there is no more progress made in collecting repository- - // related postponements, collect the dependency configuration-related - // postponements. - // - // Note that we do it before alternatives since configurations we do - // perfectly (via backtracking) while alternatives -- heuristically. - // - // Note that since the potential snapshot restore replaces all the - // list entries we cannot iterate using the iterator here. Also note - // that the list size may change during iterating. - // - for (size_t ci (0); ci != postponed_cfgs.size (); ++ci) - { - postponed_configuration* pc (&postponed_cfgs[ci]); - - // Find the next configuration to try to negotiate, skipping the - // already negotiated ones. - // - if (pc->negotiated) - continue; - - size_t pcd (depth + 1); - pc->depth = pcd; - - // Either return or retry the same cluster or skip this cluster and - // proceed to the next one. - // - for (;;) - { - // First assume we can negotiate this configuration rolling back - // if this doesn't pan out. - // - snapshot s (*this, - postponed_repo, - postponed_alts, - postponed_deps, - postponed_cfgs); - - try - { - collect_build_postponed (o, - replaced_vers, - postponed_repo, - postponed_alts, - postponed_deps, - postponed_cfgs, - postponed_cfgs_history, - postponed_poss, - fdb, - rpt_depts, - apc, - pc); - - // If collect() returns (instead of throwing), this means it - // processed everything that was postponed. - // - assert (postponed_repo.empty () && - postponed_cfgs.negotiated () && - postponed_alts.empty () && - !postponed_deps.has_bogus ()); - - l5 ([&]{trace << "end" << trace_suffix;}); - - return; - } - catch (skip_configuration& e) - { - // Restore the state from snapshot. - // - // Note: postponed_cfgs is re-assigned. - // - s.restore (*this, - postponed_repo, - postponed_alts, - postponed_deps, - postponed_cfgs); - - pc = &postponed_cfgs[ci]; - - // Note that in this case we keep the accumulated configuration, - // if any. - - pc->depth = 0; - - // If requested, "replace" the "later" dependent-dependency - // cluster with an earlier. - // - if (e.dependent) - { - existing_dependent& ed (*e.dependent); - pair<size_t, size_t> pos (e.new_position); - - const build_package* bp ( - replace_existing_dependent_dependency ( - trace, - o, - ed, // Note: modified. - pos, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - postponed_cfgs)); - - postponed_cfgs.add (package_key (ed.db, ed.selected->name), - pos, - package_key (bp->db, bp->selected->name)); - } - - l5 ([&]{trace << "postpone cfg-negotiation of " << *pc;}); - - break; - } - catch (const retry_configuration& e) - { - // If this is not "our problem", then keep looking. - // - if (e.depth != pcd) - throw; - - package_configurations cfgs ( - move (pc->dependency_configurations)); - - // Restore the state from snapshot. - // - // Note: postponed_cfgs is re-assigned. - // - s.restore (*this, - postponed_repo, - postponed_alts, - postponed_deps, - postponed_cfgs); - - pc = &postponed_cfgs[ci]; - - l5 ([&]{trace << "cfg-negotiation of " << *pc << " failed due " - << "to dependent " << e.dependent << ", refining " - << "configuration";}); - - // Copy over the configuration for further refinement. - // - // Note that there is also a possibility of ending up with - // "bogus" configuration variables that were set by a dependent - // during up-negotiation but, due to changes to the overall - // configuration, such a dependent were never re-visited. - // - // The way we are going to deal with this is by detecting such - // bogus variables based on the confirmed flag, cleaning them - // out, and doing another retry. Here we clear the confirmed - // flag and the detection happens in collect_build_postponed() - // after we have processed everything postponed (since that's - // the only time we can be certain there could no longer be a - // re-visit). - // - for (package_configuration& cfg: cfgs) - for (config_variable_value& v: cfg) - if (v.dependent) - v.confirmed = false; - - pc->dependency_configurations = move (cfgs); - } - catch (merge_configuration& e) - { - // If this is not "our problem", then keep looking. - // - if (e.depth != pcd) - throw; - - postponed_configuration shadow (move (*pc)); - - // Restore the state from snapshot. - // - // Note: postponed_cfgs is re-assigned. - // - s.restore (*this, - postponed_repo, - postponed_alts, - postponed_deps, - postponed_cfgs); - - pc = &postponed_cfgs[ci]; - - assert (!pc->negotiated); - - // Drop any accumulated configuration (which could be carried - // over from retry_configuration logic). - // - pc->dependency_configurations.clear (); - - l5 ([&]{trace << "cfg-negotiation of " << *pc << " failed due " - << "to non-negotiated clusters, force-merging " - << "based on shadow cluster " << shadow;}); - - // Pre-merge into this cluster those non-negotiated clusters - // which were merged into the shadow cluster. - // - for (size_t id: shadow.merged_ids) - { - postponed_configuration* c (postponed_cfgs.find (id)); - - if (c != nullptr) - { - // Otherwise we would be handling the exception in the - // higher stack frame. - // - assert (!c->negotiated); - - l5 ([&]{trace << "force-merge " << *c << " into " << *pc;}); - - pc->merge (move (*c)); + // Bail out if the selected package version is held and the package is not + // specified on the command line nor is being upgraded/deorphaned via its + // dependents recursively. + // + const shared_ptr<selected_package>& sp (p.selected); - // Mark configuration as the one being merged from for - // subsequent erasing from the list. - // - c->dependencies.clear (); - } - } + if (sp != nullptr && sp->hold_version && + hold_pkg == nullptr && dep_pkg == nullptr && + !p.upgrade && !p.deorphan) + { + l5 ([&]{trace << "replacement of " << what << " version " + << p.available_name_version_db () << " is denied since " + << "it is already built to hold version and it is not " + << "specified on command line nor is being upgraded or " + << "deorphaned";}); - // Erase clusters which we have merged from. Also re-translate - // the current cluster address into index which may change as a - // result of the merge. - // - auto i (postponed_cfgs.begin ()); - auto j (postponed_cfgs.before_begin ()); // Precedes iterator i. + return nullopt; + } - for (size_t k (0); i != postponed_cfgs.end (); ) - { - if (!i->dependencies.empty ()) - { - if (&*i == pc) - ci = k; + transaction t (db); - ++i; - ++j; - ++k; - } - else - i = postponed_cfgs.erase_after (j); - } - - pc->set_shadow_cluster (move (shadow)); - } - } - } + // Collect the repository fragments to search the available packages in. + // + config_repo_fragments rfs; - // Note that we only get here if we didn't make any progress on the - // previous loop (the only "progress" path ends with return). + // Add a repository fragment to the specified list, suppressing duplicates. + // + auto add = [] (shared_ptr<repository_fragment>&& rf, + vector<shared_ptr<repository_fragment>>& rfs) + { + if (find (rfs.begin (), rfs.end (), rf) == rfs.end ()) + rfs.push_back (move (rf)); + }; - // Now, try to collect the dependency alternative-related - // postponements. - // - if (!postponed_alts.empty ()) + // If the package is specified as build-to-hold on the command line, then + // collect the root repository fragment from its database. Otherwise, + // collect the repository fragments its dependent packages come from. + // + if (hold_pkg != nullptr) + { + add (db.find<repository_fragment> (empty_string), rfs[db]); + } + else + { + // Collect the repository fragments the new dependents come from. + // + if (p.required_by_dependents) + { + for (const package_version_key& dvk: p.required_by) { - // Sort the postponments in the unprocessed dependencies count - // descending order. - // - // The idea here is to preferably handle those postponed packages - // first, which have a higher probability to affect the dependency - // alternative selection for other packages. - // - spas.assign (postponed_alts.begin (), postponed_alts.end ()); - - std::sort (spas.begin (), spas.end (), - [] (build_package* x, build_package* y) - { - size_t xt (x->available->dependencies.size () - - x->dependencies->size ()); - - size_t yt (y->available->dependencies.size () - - y->dependencies->size ()); - - if (xt != yt) - return xt > yt ? -1 : 1; - - // Also factor the package name and configuration path - // into the ordering to achieve a stable result. - // - int r (x->name ().compare (y->name ())); - return r != 0 - ? r - : x->db.get ().config.compare (y->db.get ().config); - }); - - // Calculate the maximum number of the enabled dependency - // alternatives. - // - size_t max_enabled_count (0); - - for (build_package* p: spas) + if (dvk.version) // Real package? { - assert (p->postponed_dependency_alternatives); - - size_t n (p->postponed_dependency_alternatives->size ()); - - if (max_enabled_count < n) - max_enabled_count = n; - } + const build_package* d (pkgs.entered_build (dvk.db, dvk.name)); - assert (max_enabled_count != 0); // Wouldn't be here otherwise. + // Must be collected as a package build (see + // build_package::required_by for details). + // + assert (d != nullptr && + d->action && + *d->action == build_package::build && + d->available != nullptr); - // Try to select a dependency alternative with the lowest index, - // preferring postponed packages with the longer tail of unprocessed - // dependencies (see above for the reasoning). - // - for (size_t i (1); i <= max_enabled_count && !prog; ++i) - { - for (build_package* p: spas) + for (const package_location& pl: d->available->locations) { - prs.clear (); - pas.clear (); - - size_t ndep (p->dependencies->size ()); - - build_package_refs dep_chain; - - l5 ([&]{trace << "index " << i << " collect alt-postponed " - << p->available_name_version_db ();}); - - collect_build_prerequisites (o, - *p, - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - &prs, - &pas, - i, - postponed_deps, - postponed_cfgs, - postponed_poss); - - prog = (pas.find (p) == pas.end () || - ndep != p->dependencies->size ()); - - // Save the potential new postponements. - // - if (prog) - { - postponed_alts.erase (p); - postponed_alts.insert (pas.begin (), pas.end ()); - } + const lazy_shared_ptr<repository_fragment>& lrf ( + pl.repository_fragment); - size_t npr (postponed_repo.size ()); - postponed_repo.insert (prs.begin (), prs.end ()); - - // Note that not collecting any alternative-relative - // postponements but producing new repository-related - // postponements is progress nevertheless. - // - // Note that we don't need to check for new configuration- - // related postponements here since if they are present, then - // this package wouldn't be in pas and so prog would be true - // (see above for details). + // Note that here we also handle dependents fetched/unpacked + // using the existing archive/directory adding the root + // repository fragments from their configurations. // - if (!prog) - prog = (npr != postponed_repo.size ()); - - if (prog) - break; + if (!rep_masked_fragment (lrf)) + add (lrf.load (), rfs[lrf.database ()]); } } - - if (prog) - continue; } - - assert (!prog); - - // If we still have any non-negotiated clusters and non-replace - // postponed positions, then it's possible one of them is the cross- - // dependent pathological case where we will never hit it unless we - // force the re-evaluation to earlier position (similar to the - // single-dependent case, which we handle accurately). For example: - // - // tex: depends: libbar(c) - // depends: libfoo(c) - // - // tix: depends: libbar(c) - // depends: tex(c) - // - // Here tex and tix are existing dependent and we are upgrading tex. - // - // While it would be ideal to handle such cases accurately, it's not - // trivial. So for now we resort to the following heuristics: when - // left with no other option, we treat the first encountered non- - // replace position as replace and see if that helps move things - // forward. - // - if (!postponed_cfgs.negotiated () && - find_if (postponed_poss.begin (), postponed_poss.end (), - [] (const auto& v) {return !v.second.replace;}) != - postponed_poss.end () && - !postponed_poss.replace) - { - l5 ([&]{trace << "non-negotiated clusters left and non-replace " - << "postponed positions are present, overriding first " - << "encountered non-replace position to replace";}); - - postponed_poss.replace = true; - prog = true; - continue; // Go back to negotiating skipped cluster. - } - - // Finally, erase the bogus postponements and re-collect from scratch, - // if any (see postponed_dependencies for details). - // - // Note that we used to re-collect such postponements in-place but - // re-doing from scratch feels more correct (i.e., we may end up doing - // it earlier which will affect dependency alternatives). - // - postponed_deps.cancel_bogus (trace, false /* initial_collection */); } - // Check if any negotiatiated configurations ended up with any bogus - // variables (see retry_configuration catch block for background). + // Collect the repository fragments the existing dependents come from. // - // Note that we could potentially end up yo-yo'ing: we remove a bogus - // and that causes the original dependent to get re-visited which in - // turn re-introduces the bogus. In other words, one of the bogus - // variables which we have removed are actually the cause of no longer - // needing the dependent that introduced it. Feels like the correct - // outcome of this should be keeping the bogus variable that triggered - // yo-yo'ing. Of course, there could be some that we should keep and - // some that we should drop and figuring this out would require retrying - // all possible combinations. An alternative solution would be to detect - // yo-yo'ing, print the bogus variables involved, and ask the user to - // choose (with an override) which ones to keep. Let's go with this for - // now. + // Note that all the existing dependents are already in the map (since + // collect_dependents() has already been called) and are either + // reconfigure adjustments or non-collected recursively builds. // + if (sp != nullptr) { - // On the first pass see if we have anything bogus. - // - bool bogus (false); - for (postponed_configuration& pcfg: postponed_cfgs) + for (database& ddb: db.dependent_configs ()) { - if (pcfg.negotiated && *pcfg.negotiated) // Negotiated. + for (const auto& pd: query_dependents (ddb, nm, db)) { - for (package_configuration& cfg: pcfg.dependency_configurations) - { - for (config_variable_value& v: cfg) - { - if (v.dependent && !v.confirmed) - { - bogus = true; - break; - } - } - if (bogus) break; - } - if (bogus) break; - } - } - - if (bogus) - { - // On the second pass calculate the checksum of all the negotiated - // clusters. - // - sha256 cs; - for (postponed_configuration& pcfg: postponed_cfgs) - { - if (pcfg.negotiated && *pcfg.negotiated) - { - for (package_configuration& cfg: pcfg.dependency_configurations) - { - for (config_variable_value& v: cfg) - { - if (v.dependent) - to_checksum (cs, v); - } - } - } - } - - bool cycle; - { - string s (cs.string ()); - if (find (postponed_cfgs_history.begin (), - postponed_cfgs_history.end (), - s) == postponed_cfgs_history.end ()) - { - postponed_cfgs_history.push_back (move (s)); - cycle = false; - } - else - cycle = true; - } - - // On the third pass we either retry or diagnose. - // - diag_record dr; - if (cycle) - { - dr << - fail << "unable to remove bogus configuration values without " - << "causing configuration refinement cycle" << - info << "consider manually specifying one or more of the " - << "following variables as user configuration"; - } + const build_package* d (pkgs.entered_build (ddb, pd.name)); - for (postponed_configuration& pcfg: postponed_cfgs) - { - optional<package_key> dept; // Bogus dependent. + // See collect_dependents() for details. + // + assert (d != nullptr && d->action); - if (pcfg.negotiated && *pcfg.negotiated) + if ((*d->action == build_package::adjust && + (d->flags & build_package::adjust_reconfigure) != 0) || + (*d->action == build_package::build && !d->dependencies)) { - for (package_configuration& cfg: pcfg.dependency_configurations) - { - // Note that the entire dependency configuration may end up - // being "bogus" (i.e., it does not contain any configuration - // variables with a confirmed dependent). But that will be - // handled naturally: we will either no longer have this - // dependency in the cluster and thus never call its - // skeleton's dependent_config() or this call will be no-op - // since it won't find any dependent variables. - // - for (config_variable_value& v: cfg) - { - if (v.dependent && !v.confirmed) - { - if (!dept) - dept = move (v.dependent); - - if (cycle) - dr << "\n " << v.serialize_cmdline (); - else - v.undefine (); - } - } - } + shared_ptr<selected_package> p ( + ddb.load<selected_package> (pd.name)); - if (dept) - { - if (cycle) - break; - else - throw retry_configuration {pcfg.depth, move (*dept)}; - } + add_dependent_repo_fragments (ddb, p, rfs); } - - if (dept) - break; } } } - - // If any postponed_{repo,alts} builds remained, then perform the - // diagnostics run. Naturally we shouldn't have any postponed_cfgs - // without one of the former. - // - if (!postponed_repo.empty ()) - { - build_package_refs dep_chain; - - collect_build_prerequisites (o, - **postponed_repo.begin (), - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - nullptr, - nullptr, - 0, - postponed_deps, - postponed_cfgs, - postponed_poss); - - assert (false); // Can't be here. - } - - if (!postponed_alts.empty ()) - { - build_package_refs dep_chain; - - collect_build_prerequisites (o, - **postponed_alts.begin (), - fdb, - rpt_depts, - apc, - false /* initial_collection */, - replaced_vers, - dep_chain, - nullptr, - nullptr, - 0, - postponed_deps, - postponed_cfgs, - postponed_poss); - - assert (false); // Can't be here. - } - - // While the assumption is that we shouldn't leave any non-negotiated - // clusters, we can potentially miss some corner cases in the above - // "skip configuration" logic. Let's thus trace the non-negotiated - // clusters before the assertion. - // -#ifndef NDEBUG - for (const postponed_configuration& cfg: postponed_cfgs) - { - if (!cfg.negotiated || !*cfg.negotiated) - trace << "unexpected non-negotiated cluster " << cfg; - } - - assert (postponed_cfgs.negotiated ()); -#endif - - l5 ([&]{trace << "end" << trace_suffix;}); } - // Order the previously-collected package with the specified name - // returning its positions. - // - // If buildtime is nullopt, then search for the specified package build in - // only the specified configuration. Otherwise, treat the package as a - // dependency and use the custom search function to find its build - // configuration. Failed that, search for it recursively (see - // package_map::find_dependency() for details). - // - // Recursively order the package dependencies being ordered failing if a - // dependency cycle is detected. If reorder is true, then reorder this - // package to be considered as "early" as possible. - // - iterator - order (database& db, - const package_name& name, - optional<bool> buildtime, - const function<find_database_function>& fdb, - bool reorder = true) - { - package_refs chain; - return order (db, name, buildtime, chain, fdb, reorder); - } - - // If a configured package is being up/down-graded then that means - // all its dependents could be affected and we have to reconfigure - // them. This function examines every package that is already on - // the list and collects and orders all its dependents. We also need - // to make sure the dependents are ok with the up/downgrade. - // - // Should we reconfigure just the direct depends or also include - // indirect, recursively? Consider this plauisible scenario as an - // example: We are upgrading a package to a version that provides - // an additional API. When its direct dependent gets reconfigured, - // it notices this new API and exposes its own extra functionality - // that is based on it. Now it would make sense to let its own - // dependents (which would be our original package's indirect ones) - // to also notice this. + // Query the dependency available packages from all the collected + // repository fragments and select the most appropriate one. Note that + // this code is inspired by the evaluate_dependency() function + // implementation, which documents the below logic in great detail. // - void - collect_order_dependents (const repointed_dependents& rpt_depts) - { - // For each package on the list we want to insert all its dependents - // before it so that they get configured after the package on which - // they depend is configured (remember, our build order is reverse, - // with the last package being built first). This applies to both - // packages that are already on the list as well as the ones that - // we add, recursively. - // - for (auto i (begin ()); i != end (); ++i) - { - const build_package& p (*i); - - // Prune if this is not a configured package being up/down-graded - // or reconfigured. - // - assert (p.action); + optional<version_constraint> c (constraint != nullptr + ? *constraint + : optional<version_constraint> ()); - // Dropped package may have no dependents. - // - if (*p.action != build_package::drop && p.reconfigure ()) - collect_order_dependents (i, rpt_depts); - } - } - - void - collect_order_dependents (iterator pos, - const repointed_dependents& rpt_depts) + if (!c && p.upgrade && !*p.upgrade) { - tracer trace ("collect_order_dependents"); - - assert (pos != end ()); - - build_package& p (*pos); - - database& pdb (p.db); - const shared_ptr<selected_package>& sp (p.selected); - - const package_name& n (sp->name); - - // See if we are up/downgrading this package. In particular, the - // available package could be NULL meaning we are just adjusting. - // - int ud (p.available != nullptr - ? sp->version.compare (p.available_version ()) - : 0); - - for (database& ddb: pdb.dependent_configs ()) - { - for (auto& pd: query_dependents_cache (ddb, n, pdb)) - { - package_name& dn (pd.name); - auto i (map_.find (ddb, dn)); - - // Make sure the up/downgraded package still satisfies this - // dependent. But first "prune" if the dependent is being dropped or - // this is a replaced prerequisite of the repointed dependent. - // - // Note that the repointed dependents are always collected and have - // all their collected prerequisites ordered (including new and old - // ones). See collect_build_prerequisites() and order() for details. - // - bool check (ud != 0 && pd.constraint); - - if (i != map_.end () && i->second.position != end ()) - { - build_package& dp (i->second.package); - - // Skip the droped dependent. - // - if (dp.action && *dp.action == build_package::drop) - continue; - - repointed_dependents::const_iterator j ( - rpt_depts.find (package_key {ddb, dn})); - - if (j != rpt_depts.end ()) - { - const map<package_key, bool>& prereqs_flags (j->second); - - auto k (prereqs_flags.find (package_key {pdb, n})); - - if (k != prereqs_flags.end () && !k->second) - continue; - } + assert (sp != nullptr); // See build_package::upgrade. - // There is one tricky aspect: the dependent could be in the - // process of being reconfigured or up/downgraded as well. In this - // case all we need to do is detect this situation and skip the - // test since all the (new) constraints of this package have been - // satisfied in collect_build(). - // - if (check) - check = !dp.dependencies; - } + c = patch_constraint (sp); - if (check) - { - const version& av (p.available_version ()); - const version_constraint& c (*pd.constraint); - - if (!satisfies (av, c)) - { - diag_record dr (fail); - - dr << "unable to " << (ud < 0 ? "up" : "down") << "grade " - << "package " << *sp << pdb << " to "; - - // Print both (old and new) package names in full if the system - // attribution changes. - // - if (p.system != sp->system ()) - dr << p.available_name_version (); - else - dr << av; // Can't be the wildcard otherwise would satisfy. - - dr << info << "because package " << dn << ddb << " depends on (" - << n << " " << c << ")"; - - string rb; - if (!p.user_selection ()) - { - for (const package_key& pk: p.required_by) - rb += (rb.empty () ? " " : ", ") + pk.string (); - } - - if (!rb.empty ()) - dr << info << "package " << p.available_name_version () - << " required by" << rb; - - dr << info << "explicitly request up/downgrade of package " - << dn; - - dr << info << "or explicitly specify package " << n - << " version to manually satisfy these constraints"; - } - - // Add this contraint to the list for completeness. - // - p.constraints.emplace_back (ddb, dn.string (), c); - } - - auto adjustment = [&dn, &ddb, &n, &pdb] () -> build_package - { - shared_ptr<selected_package> dsp (ddb.load<selected_package> (dn)); + assert (c); // See build_package::upgrade. + } - // A system package cannot be a dependent. - // - assert (!dsp->system ()); + available_packages afs (find_available (nm, c, rfs)); - return build_package { - build_package::adjust, - ddb, - move (dsp), - nullptr, // No available pkg/repo fragment. - nullptr, - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - false, // System. - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {package_key {pdb, n}}, // Required by (dependency). - false, // Required by dependents. - build_package::adjust_reconfigure}; - }; + using available = pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>>; - // We can have three cases here: the package is already on the - // list, the package is in the map (but not on the list) and it - // is in neither. - // - // If the existing entry is pre-entered, is an adjustment, or is a - // build that is not supposed to be built (not in the list), then we - // merge it into the new adjustment entry. Otherwise (is a build in - // the list), we just add the reconfigure adjustment flag to it. - // - if (i != map_.end ()) - { - build_package& dp (i->second.package); - iterator& dpos (i->second.position); + available ra; - if (!dp.action || // Pre-entered. - *dp.action != build_package::build || // Non-build. - dpos == end ()) // Build not in the list. - { - build_package bp (adjustment ()); - bp.merge (move (dp)); - dp = move (bp); - } - else // Build in the list. - dp.flags |= build_package::adjust_reconfigure; - - // It may happen that the dependent is already in the list but is - // not properly ordered against its dependencies that get into the - // list via another dependency path. Thus, we check if the - // dependent is to the right of its dependency and, if that's the - // case, reinsert it in front of the dependency. - // - if (dpos != end ()) - { - for (auto i (pos); i != end (); ++i) - { - if (i == dpos) - { - erase (dpos); - dpos = insert (pos, dp); - break; - } - } - } - else - dpos = insert (pos, dp); - } - else - { - // Don't move dn since it is used by adjustment(). - // - i = map_.emplace (package_key {ddb, dn}, - data_type {end (), adjustment ()}).first; - - i->second.position = insert (pos, i->second.package); - } + // Version to deorphan. + // + const version* dov (p.deorphan ? &sp->version : nullptr); - // Recursively collect our own dependents inserting them before us. - // - // Note that we cannot end up with an infinite recursion for - // configured packages due to a dependency cycle (see order() for - // details). - // - collect_order_dependents (i->second.position, rpt_depts); - } - } - } + optional<version_constraint> dopc; // Patch constraint for the above. + optional<version_constraint> domc; // Minor constraint for the above. - void - clear () - { - build_package_list::clear (); - map_.clear (); - } + bool orphan_best_match (p.deorphan && constraint == nullptr && !p.upgrade); - void - clear_order () + if (orphan_best_match) { - build_package_list::clear (); + // Note that non-zero iteration makes a version non-standard, so we + // reset it to 0 to produce the patch/minor constraints. + // + version v (dov->epoch, + dov->upstream, + dov->release, + dov->revision, + 0 /* iteration */); - for (auto& p: map_) - p.second.position = end (); + dopc = patch_constraint (nm, v, true /* quiet */); + domc = minor_constraint (nm, v, true /* quiet */); } - // Verify that builds ordering is consistent across all the data - // structures and the ordering expectations are fulfilled (real build - // actions are all ordered, etc). - // - void - verify_ordering () const - { - for (const auto& b: map_) - { - const build_package& bp (b.second.package); - - auto i (find_if (begin (), end (), - [&bp] (const build_package& p) {return &p == &bp;})); - - // List ordering must properly be reflected in the tree entries. - // - assert (i == b.second.position); - - // Pre-entered builds must never be ordered and the real build actions - // (builds, adjustments, etc) must all be ordered. - // - // Note that the later was not the case until we've implemented - // re-collection from scratch after the package version replacement - // (see replaced_versions for details). Before that the whole - // dependency trees from the being replaced dependent stayed in the - // map. - // - assert (bp.action.has_value () == (i != end ())); - } - } + available deorphan_latest_iteration; + available deorphan_later_revision; + available deorphan_later_patch; + available deorphan_later_minor; + available deorphan_latest_available; - private: - // Return the list of existing dependents that has a configuration clause - // for the specified dependency. Skip dependents which are being built and - // require recursive recollection or dropped (present in the map) or - // expected to be built or dropped (present in rpt_depts or replaced_vers). - // - // Optionally, specify the function which can verify the dependent build - // and decide whether to override the default behavior and still add the - // dependent package to the resulting list, returning true in this case. + // Return true if a version satisfies all the dependency constraints. + // Otherwise, save all the being built unsatisfied dependents into the + // resulting list, suppressing duplicates. // - struct existing_dependent + auto satisfactory = [&p, &unsatisfied_dpts] (const version& v) { - reference_wrapper<database> db; - shared_ptr<selected_package> selected; - pair<size_t, size_t> dependency_position; - }; - - using verify_dependent_build_function = bool (const package_key&, - pair<size_t, size_t>); - - vector<existing_dependent> - query_existing_dependents ( - tracer& trace, - database& db, - const package_name& name, - const replaced_versions& replaced_vers, - const repointed_dependents& rpt_depts, - const function<verify_dependent_build_function>& vdb = nullptr) - { - vector<existing_dependent> r; - - lazy_shared_ptr<selected_package> sp (db, name); + bool r (true); - for (database& ddb: db.dependent_configs ()) + for (const auto& c: p.constraints) { - for (auto& pd: query_dependents (ddb, name, db)) + if (!satisfies (v, c.value)) { - shared_ptr<selected_package> dsp ( - ddb.load<selected_package> (pd.name)); + r = false; - auto i (dsp->prerequisites.find (sp)); - assert (i != dsp->prerequisites.end ()); - - const auto& pos (i->second.config_position); - - if (pos.first != 0) // Has config clause? + if (c.dependent.version && !c.selected_dependent) { - package_key pk (ddb, pd.name); - - if (rpt_depts.find (pk) != rpt_depts.end ()) - { - l5 ([&]{trace << "skip repointed existing dependent " << pk - << " of dependency " << name << db;}); - continue; - } - - // Ignore dependent which is already being built or dropped. - // - const build_package* p (entered_build (pk)); + package_key pk (c.dependent.db, c.dependent.name); - if (p != nullptr && p->action) - { - bool build; - if (((build = *p->action == build_package::build) && - (p->system || p->recollect_recursively (rpt_depts))) || - *p->action == build_package::drop) - { - if (!build || !vdb || !vdb (pk, pos)) - { - l5 ([&]{trace << "skip being " - << (build ? "built" : "dropped") - << " existing dependent " << pk - << " of dependency " << name << db;}); - continue; - } - } - } - - // Ignore dependent which is expected to be built or dropped. - // - auto vi (replaced_vers.find (pk)); - if (vi != replaced_vers.end () && !vi->second.replaced) - { - bool build (vi->second.available != nullptr); - - l5 ([&]{trace << "skip expected to be " - << (build ? "built" : "dropped") - << " existing dependent " << pk - << " of dependency " << name << db;}); - - continue; - } - - r.push_back (existing_dependent {ddb, move (dsp), pos}); + if (find (unsatisfied_dpts.begin (), + unsatisfied_dpts.end (), + pk) == unsatisfied_dpts.end ()) + unsatisfied_dpts.push_back (move (pk)); } } } return r; - } + }; - // Update the existing dependent object (previously obtained with the - // query_existing_dependents() call) with the new dependency position and - // collect the dependency referred by this position. Return the pointer to - // the collected build package object. - // - const build_package* - replace_existing_dependent_dependency ( - tracer& trace, - const pkg_build_options& o, - existing_dependent& ed, - pair<size_t, size_t> pos, - const function<find_database_function>& fdb, - const repointed_dependents& rpt_depts, - const function<add_priv_cfg_function>& apc, - bool initial_collection, - replaced_versions& replaced_vers, - postponed_configurations& postponed_cfgs) + for (available& af: afs) { - // The repointed dependent cannot be returned by - // query_existing_dependents(). Note that the repointed dependent - // references both old and new prerequisites. - // - assert (rpt_depts.find (package_key (ed.db, ed.selected->name)) == - rpt_depts.end ()); - - shared_ptr<selected_package> dsp; - database* pdb (nullptr); - const version_constraint* vc (nullptr); - - // Find the dependency for this earlier dependency position. We know - // it must be there since it's with configuration. - // - for (const auto& p: ed.selected->prerequisites) - { - if (p.second.config_position == pos) - { - pdb = &p.first.database (); - - dsp = p.first.load (); - - l5 ([&]{trace << "replace dependency at index " - << ed.dependency_position.first - << " of existing dependent " << *ed.selected - << ed.db << " with dependency " << *dsp - << *pdb << " at index " << pos.first;}); - - if (p.second.constraint) - vc = &*p.second.constraint; - } - } + shared_ptr<available_package>& ap (af.first); - assert (dsp != nullptr); + if (ap->stub ()) + continue; - package_key pk (*pdb, dsp->name); + const version& av (ap->version); - // Adjust the existing dependent entry. + // Skip if the available package version doesn't satisfy all the + // constraints (note: must be checked first since has a byproduct). // - ed.dependency_position = pos; + if (!satisfactory (av)) + continue; - // Collect the package build for this dependency. + // Don't offer to replace to the same version. // - pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>> rp ( - find_available_fragment (o, pk.db, dsp)); - - bool system (dsp->system ()); - - package_key dpk (ed.db, ed.selected->name); - - build_package p { - build_package::build, - pk.db, - move (dsp), - move (rp.first), - move (rp.second), - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - nullopt, // Hold package. - nullopt, // Hold version. - {}, // Constraints. - system, // System. - false, // Keep output directory. - false, // Disfigure (from-scratch reconf). - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {dpk}, // Required by (dependent). - true, // Required by dependents. - build_package::adjust_reconfigure}; - - if (vc != nullptr) - p.constraints.emplace_back (dpk.db, dpk.name.string (), *vc); + if (av == ver) + continue; - // Note: not recursive. + // Don't repeatedly offer the same adjustments for the same command + // line. // - collect_build (o, - move (p), - fdb, - rpt_depts, - apc, - initial_collection, - replaced_vers, - postponed_cfgs); - - return entered_build (pk); - } - - private: - struct package_ref - { - database& db; - const package_name& name; - - bool - operator== (const package_ref& v) + if (cmdline_adjs.tried_earlier (db, nm, av)) { - return name == v.name && db == v.db; - } - }; - using package_refs = small_vector<package_ref, 16>; - - iterator - order (database& db, - const package_name& name, - optional<bool> buildtime, - package_refs& chain, - const function<find_database_function>& fdb, - bool reorder) - { - package_map::iterator mi; - - if (buildtime) - { - database* ddb (fdb (db, name, *buildtime)); + l5 ([&]{trace << "replacement " << package_version_key (db, nm, av) + << " tried earlier for same command line, skipping";}); - mi = ddb != nullptr - ? map_.find (*ddb, name) - : map_.find_dependency (db, name, *buildtime); + continue; } - else - mi = map_.find (db, name); - - // Every package that we order should have already been collected. - // - assert (mi != map_.end ()); - - build_package& p (mi->second.package); - assert (p.action); // Can't order just a pre-entered package. - - database& pdb (p.db); - - // Make sure there is no dependency cycle. + // If we aim to upgrade to the latest version and it tends to be less + // then the selected one, then what we currently have is the best that + // we can get. Thus, we use the selected version as a replacement, + // unless it doesn't satisfy all the constraints or we are deorphaning. // - package_ref cp {pdb, name}; + if (constraint == nullptr && sp != nullptr) { - auto i (find (chain.begin (), chain.end (), cp)); + const version& sv (sp->version); + if (av < sv && !sp->system () && !p.deorphan) + { + // Only consider the selected package if its version is satisfactory + // for its new dependents (note: must be checked first since has a + // byproduct), differs from the version being replaced, and was + // never used for the same command line (see above for details). + // + if (satisfactory (sv) && sv != ver) + { + if (!cmdline_adjs.tried_earlier (db, nm, sv)) + { + ra = make_available_fragment (o, db, sp); + break; + } + else + l5 ([&]{trace << "selected package replacement " + << package_version_key (db, nm, sp->version) + << " tried earlier for same command line, " + << "skipping";}); + } + } + } - if (i != chain.end ()) + if (orphan_best_match) + { + if (av == *dov) { - diag_record dr (fail); - dr << "dependency cycle detected involving package " << name << pdb; + ra = move (af); + break; + } - auto nv = [this] (const package_ref& cp) - { - auto mi (map_.find (cp.db, cp.name)); - assert (mi != map_.end ()); + if (deorphan_latest_iteration.first == nullptr && + av.compare (*dov, false /* revision */, true /* iteration */) == 0) + deorphan_latest_iteration = af; - build_package& p (mi->second.package); + if (deorphan_later_revision.first == nullptr && + av.compare (*dov, true /* revision */) == 0 && + av.compare (*dov, false /* revision */, true /* iteration */) > 0) + deorphan_later_revision = af; - assert (p.action); // See above. + if (deorphan_later_patch.first == nullptr && + dopc && satisfies (av, *dopc) && + av.compare (*dov, true /* revision */) > 0) // Patch is greater? + deorphan_later_patch = af; - // We cannot end up with a dependency cycle for actions other than - // build since these packages are configured and we would fail on - // a previous run while building them. - // - assert (p.available != nullptr); + if (deorphan_later_minor.first == nullptr && + domc && satisfies (av, *domc) && + av.compare (*dov, true /* revision */) > 0 && + deorphan_later_patch.first == nullptr) + deorphan_later_minor = af; - return p.available_name_version_db (); - }; + if (deorphan_latest_available.first == nullptr) + deorphan_latest_available = move (af); - // Note: push_back() can invalidate the iterator. - // - size_t j (i - chain.begin ()); + if (av.compare (*dov, false /* revision */, true /* iteration */) < 0) + { + assert (deorphan_latest_iteration.first != nullptr || + deorphan_later_revision.first != nullptr || + deorphan_later_patch.first != nullptr || + deorphan_later_minor.first != nullptr || + deorphan_latest_available.first != nullptr); - for (chain.push_back (cp); j != chain.size () - 1; ++j) - dr << info << nv (chain[j]) << " depends on " << nv (chain[j + 1]); + break; } } - - // If this package is already in the list, then that would also - // mean all its prerequisites are in the list and we can just - // return its position. Unless we want it reordered. - // - iterator& pos (mi->second.position); - if (pos != end ()) + else { - if (reorder) - erase (pos); - else - return pos; + ra = move (af); + break; } + } - // Order all the prerequisites of this package and compute the - // position of its "earliest" prerequisite -- this is where it - // will be inserted. - // - const shared_ptr<selected_package>& sp (p.selected); - const shared_ptr<available_package>& ap (p.available); + shared_ptr<available_package>& rap (ra.first); - bool build (*p.action == build_package::build); + if (rap == nullptr && orphan_best_match) + { + if (deorphan_latest_iteration.first != nullptr) + ra = move (deorphan_latest_iteration); + else if (deorphan_later_revision.first != nullptr) + ra = move (deorphan_later_revision); + else if (deorphan_later_patch.first != nullptr) + ra = move (deorphan_later_patch); + else if (deorphan_later_minor.first != nullptr) + ra = move (deorphan_later_minor); + else if (deorphan_latest_available.first != nullptr) + ra = move (deorphan_latest_available); + } - // Package build must always have the available package associated. - // - assert (!build || ap != nullptr); + t.commit (); - // Unless this package needs something to be before it, add it to - // the end of the list. - // - iterator i (end ()); + // Bail out if no appropriate replacement is found and return the + // command line adjustment object otherwise. + // + if (rap == nullptr) + return nullopt; - // Figure out if j is before i, in which case set i to j. The goal - // here is to find the position of our "earliest" prerequisite. - // - auto update = [this, &i] (iterator j) - { - for (iterator k (j); i != j && k != end ();) - if (++k == i) - i = j; - }; + optional<cmdline_adjustment> r; - // Similar to collect_build(), we can prune if the package is already - // configured, right? While in collect_build() we didn't need to add - // prerequisites of such a package, it doesn't mean that they actually - // never ended up in the map via another dependency path. For example, - // some can be a part of the initial selection. And in that case we must - // order things properly. - // - // Also, if the package we are ordering is not a system one and needs to - // be disfigured during the plan execution, then we must order its - // (current) dependencies that also need to be disfigured. - // - // And yet, if the package we are ordering is a repointed dependent, - // then we must order not only its unamended and new prerequisites but - // also its replaced prerequisites, which can also be disfigured. - // - bool src_conf (sp != nullptr && - sp->state == package_state::configured && - sp->substate != package_substate::system); + lazy_shared_ptr<repository_fragment>& raf (ra.second); - auto disfigure = [] (const build_package& p) + if (hold_pkg != nullptr || dep_pkg != nullptr) // Specified on command line? + { + if (hold_pkg != nullptr) { - return p.action && (*p.action == build_package::drop || - p.reconfigure ()); - }; + r = cmdline_adjustment (hold_pkg->db, + hold_pkg->name (), + move (rap), + move (raf)); - bool order_disfigured (src_conf && disfigure (p)); - - chain.push_back (cp); - - // Order the build dependencies. - // - if (build && !p.system) - { - // So here we are going to do things differently depending on - // whether the package is already configured or not. If it is and - // not as a system package, then that means we can use its - // prerequisites list. Otherwise, we use the manifest data. - // - if (src_conf && - sp->version == p.available_version () && - (p.config_vars.empty () || - !has_buildfile_clause (ap->dependencies))) + if (constraint != nullptr) { - for (const auto& p: sp->prerequisites) - { - database& db (p.first.database ()); - const package_name& name (p.first.object_id ()); - - // The prerequisites may not necessarily be in the map. - // - // Note that for the repointed dependent we also order its new and - // replaced prerequisites here, since they all are in the selected - // package prerequisites set. - // - auto i (map_.find (db, name)); - if (i != map_.end () && i->second.package.action) - update (order (db, - name, - nullopt /* buildtime */, - chain, - fdb, - false /* reorder */)); - } - - // We just ordered them among other prerequisites. - // - order_disfigured = false; + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " + << r->version << " by overwriting constraint " + << cmdline_adjs.to_string (*r) << " on command line";}); } else { - // The package prerequisites builds must already be collected and - // thus the resulting dependency list is complete. - // - assert (p.dependencies && - p.dependencies->size () == ap->dependencies.size ()); - - // We are iterating in reverse so that when we iterate over - // the dependency list (also in reverse), prerequisites will - // be built in the order that is as close to the manifest as - // possible. - // - for (const dependency_alternatives_ex& das: - reverse_iterate (*p.dependencies)) - { - // The specific dependency alternative must already be selected, - // unless this is a toolchain build-time dependency or all the - // alternatives are disabled in which case the alternatives list - // is empty. - // - if (das.empty ()) - continue; - - assert (das.size () == 1); - - for (const dependency& d: das.front ()) - { - // Note that for the repointed dependent we only order its new - // and unamended prerequisites here. Its replaced prerequisites - // will be ordered below. - // - update (order (pdb, - d.name, - das.buildtime, - chain, - fdb, - false /* reorder */)); - } - } + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " + << r->version << " by adding constraint " + << cmdline_adjs.to_string (*r) << " on command line";}); } } - - // Order the dependencies being disfigured. - // - if (order_disfigured) + else // dep_pkg != nullptr { - for (const auto& p: sp->prerequisites) - { - database& db (p.first.database ()); - const package_name& name (p.first.object_id ()); - - // The prerequisites may not necessarily be in the map. - // - auto i (map_.find (db, name)); + r = cmdline_adjustment (*dep_pkg->db, dep_pkg->name, rap->version); - // Note that for the repointed dependent we also order its replaced - // and potentially new prerequisites here (see above). The latter is - // redundant (we may have already ordered them above) but harmless, - // since we do not reorder. - // - if (i != map_.end () && disfigure (i->second.package)) - update (order (db, - name, - nullopt /* buildtime */, - chain, - fdb, - false /* reorder */)); + if (constraint != nullptr) + { + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " + << r->version << " by overwriting constraint " + << cmdline_adjs.to_string (*r) << " on command line";}); + } + else + { + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " + << r->version << " by adding constraint " + << cmdline_adjs.to_string (*r) << " on command line";}); } } - - chain.pop_back (); - - return pos = insert (i, p); } - - private: - struct data_type - { - iterator position; // Note: can be end(), see collect_build(). - build_package package; - }; - - class package_map: public map<package_key, data_type> + else // The package is not specified on the command line. { - public: - using base_type = map<package_key, data_type>; - - using base_type::find; - - iterator - find (database& db, const package_name& pn) - { - return find (package_key {db, pn}); - } + // If the package is configured as system, then since it is not + // specified by the user (both hold_pkg and dep_pkg are NULL) we may + // only build it as system. Thus we wouldn't be here (see above). + // + assert (sp == nullptr || !sp->system ()); - // Try to find a package build in the dependency configurations (see - // database::dependency_configs() for details). Return the end iterator - // if no build is found and issue diagnostics and fail if multiple - // builds (in multiple configurations) are found. + // Similar to the collect lambda in collect_build_prerequisites(), issue + // the warning if we are forcing an up/down-grade. // - iterator - find_dependency (database& db, const package_name& pn, bool buildtime) + if (sp != nullptr && (sp->hold_package || verb >= 2)) { - iterator r (end ()); + const version& av (rap->version); + const version& sv (sp->version); - linked_databases ldbs (db.dependency_configs (pn, buildtime)); + int ud (sv.compare (av)); - for (database& ldb: ldbs) + if (ud != 0) { - iterator i (find (ldb, pn)); - if (i != end ()) + for (const auto& c: p.constraints) { - if (r == end ()) - r = i; - else - fail << "building package " << pn << " in multiple " - << "configurations" << - info << r->first.db.get().config_orig << - info << ldb.config_orig << - info << "use --config-* to select package configuration"; + if (c.dependent.version && !satisfies (sv, c.value)) + { + warn << "package " << c.dependent << " dependency on (" + << nm << ' ' << c.value << ") is forcing " + << (ud < 0 ? "up" : "down") << "grade of " << *sp << db + << " to " << av; + + break; + } } } - - return r; } - }; - package_map map_; - }; - - // Return a patch version constraint for the selected package if it has a - // standard version, otherwise, if requested, issue a warning and return - // nullopt. - // - // Note that the function may also issue a warning and return nullopt if the - // selected package minor version reached the limit (see - // standard-version.cxx for details). - // - static optional<version_constraint> - patch_constraint (const shared_ptr<selected_package>& sp, bool quiet = false) - { - const package_name& nm (sp->name); - const version& sv (sp->version); - - // Note that we don't pass allow_stub flag so the system wildcard version - // will (naturally) not be patched. - // - string vs (sv.string ()); - optional<standard_version> v (parse_standard_version (vs)); - if (!v) - { - if (!quiet) - warn << "unable to patch " << package_string (nm, sv) << - info << "package is not using semantic/standard version"; + // For the selected built-to-hold package create the build-to-hold + // package spec and the dependency spec otherwise. + // + if (sp != nullptr && sp->hold_package) + { + r = cmdline_adjustment (db, + nm, + move (rap), + move (raf), + p.upgrade, + p.deorphan); - return nullopt; - } + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " << r->version + << " by adding package spec " + << cmdline_adjs.to_string (*r) + << " to command line";}); + } + else + { + r = cmdline_adjustment (db, nm, rap->version, p.upgrade, p.deorphan); - try - { - return version_constraint ("~" + vs); + l5 ([&]{trace << "replace " << what << " version " + << p.available_name_version () << " with " << r->version + << " by adding package spec " + << cmdline_adjs.to_string (*r) + << " to command line";}); + } } - // Note that the only possible reason for invalid_argument exception to - // be thrown is that minor version reached the 99999 limit (see - // standard-version.cxx for details). - // - catch (const invalid_argument&) - { - if (!quiet) - warn << "unable to patch " << package_string (nm, sv) << - info << "minor version limit reached"; - return nullopt; - } + return r; } - // List of dependency packages (specified with ? on the command line). + // Try to replace some of the being built, potentially indirect, dependents + // of the specified dependency with a different available version, + // satisfactory for all its new and existing dependents (if any). Return the + // command line adjustment if such a replacement is deduced and nullopt + // otherwise. It is assumed that the dependency replacement has been + // (unsuccessfully) tried by using the try_replace_dependency() call and its + // resulting list of the dependents, unsatisfied by some of the dependency + // available versions, is also passed to the function call as the + // unsatisfied_dpts argument. // - // If configuration is not specified for a system dependency package (db is - // NULL), then the dependency is assumed to be specified for all current - // configurations and their explicitly linked configurations, recursively, - // including private configurations that can potentially be created during - // this run. + // Specifically, try to replace the dependents in the following order by + // calling try_replace_dependency() for them: // - // The selected package is not NULL if the database is not NULL and the - // dependency package is present in this database. + // - Immediate dependents unsatisfied with the specified dependency. For the + // sake of tracing and documentation, we (naturally) call them unsatisfied + // dependents. // - struct dependency_package - { - database* db; // Can only be NULL if system. - package_name name; - optional<version_constraint> constraint; // nullopt if unspecified. - shared_ptr<selected_package> selected; - bool system; - bool patch; // Only for an empty version. - bool keep_out; - bool disfigure; - optional<dir_path> checkout_root; - bool checkout_purge; - strings config_vars; // Only if not system. - }; - using dependency_packages = vector<dependency_package>; - - // Evaluate a dependency package and return a new desired version. If the - // result is absent (nullopt), then there are no user expectations regarding - // this dependency. If the result is a NULL available_package, then it is - // either no longer used and can be dropped, or no changes to the dependency - // are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository fragment it must come - // from, and the system flag. + // - Immediate dependents satisfied with the dependency but applying the + // version constraint which has prevented us from picking a version which + // would be satisfactory to the unsatisfied dependents. Note that this + // information is only available for the being built unsatisfied + // dependents (added by collect_build() rather than collect_dependents()). + // We call them conflicting dependents. // - // If the package version that satisfies explicitly specified dependency - // version constraint can not be found in the dependents repositories, then - // return the "no changes are necessary" result if ignore_unsatisfiable - // argument is true and fail otherwise. The common approach is to pass true - // for this argument until the execution plan is finalized, assuming that - // the problematic dependency might be dropped. + // - Immediate dependents which apply constraint to this dependency, + // incompatible with constraints of some other dependents (both new and + // existing). We call them unsatisfiable dependents. // - struct evaluate_result - { - // The system flag is meaningless if the unused flag is true. - // - reference_wrapper<database> db; - shared_ptr<available_package> available; - lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; - bool unused; - bool system; - }; - - struct dependent_constraint - { - database& db; - shared_ptr<selected_package> package; - optional<version_constraint> constraint; - - dependent_constraint (database& d, - shared_ptr<selected_package> p, - optional<version_constraint> c) - : db (d), package (move (p)), constraint (move (c)) {} - }; - - using dependent_constraints = vector<dependent_constraint>; - - static optional<evaluate_result> - evaluate_dependency (database&, - const shared_ptr<selected_package>&, - const optional<version_constraint>& desired, - bool desired_sys, - database& desired_db, - const shared_ptr<selected_package>& desired_db_sp, - bool patch, - bool explicitly, - const config_repo_fragments&, - const dependent_constraints&, - bool ignore_unsatisfiable); - - // If there are no user expectations regarding this dependency, then we give - // no up/down-grade recommendation, unless there are no dependents in which - // case we recommend to drop the dependency. + // - Immediate dependents from unsatisfied_dpts argument. We call them + // constraining dependents. // - // Note that the user expectations are only applied for dependencies that - // have dependents in the current configurations. + // - Dependents of all the above types of dependents, discovered by + // recursively calling try_replace_dependent() for them. // - static optional<evaluate_result> - evaluate_dependency (database& db, - const shared_ptr<selected_package>& sp, - const dependency_packages& deps, - bool no_move, - bool ignore_unsatisfiable) + optional<cmdline_adjustment> + try_replace_dependent (const common_options& o, + const build_package& p, // Dependency. + const vector<unsatisfied_constraint>* ucs, + const build_packages& pkgs, + const cmdline_adjustments& cmdline_adjs, + const vector<package_key>& unsatisfied_dpts, + vector<build_package>& hold_pkgs, + dependency_packages& dep_pkgs, + set<const build_package*>& visited_dpts) { - tracer trace ("evaluate_dependency"); + tracer trace ("try_replace_dependent"); - assert (sp != nullptr && !sp->hold_package); - - const package_name& nm (sp->name); - - auto no_change = [&db] () - { - return evaluate_result {db, - nullptr /* available */, - nullptr /* repository_fragment */, - false /* unused */, - false /* system */}; - }; - - // Only search for the user expectations regarding this dependency if it - // has dependents in the current configurations, unless --no-move is - // specified. - // - // In the no-move mode consider the user-specified configurations not as a - // dependency new location, but as the current location of the dependency - // to which the expectations are applied. Note that multiple package specs - // for the same dependency in different configurations can be specified on - // the command line. + // Bail out if the dependent has already been visited and add it to the + // visited set otherwise. // - linked_databases cur_dbs; - dependency_packages::const_iterator i (deps.end ()); - - if (!no_move) - { - // Collect the current configurations which contain dependents for this - // dependency and assume no expectations if there is none. - // - for (database& cdb: current_configs) - { - if (!query_dependents (cdb, nm, db).empty ()) - cur_dbs.push_back (cdb); - } + if (!visited_dpts.insert (&p).second) + return nullopt; - // Search for the user expectations regarding this dependency by - // matching the package name and configuration type, if configuration is - // specified, preferring entries with configuration specified and fail - // if there are multiple candidates. - // - if (!cur_dbs.empty ()) - { - for (dependency_packages::const_iterator j (deps.begin ()); - j != deps.end (); - ++j) - { - if (j->name == nm && (j->db == nullptr || j->db->type == db.type)) - { - if (i == deps.end () || i->db == nullptr) - { - i = j; - } - else if (j->db != nullptr) - { - fail << "multiple " << db.type << " configurations specified " - << "for dependency package " << nm << - info << i->db->config_orig << - info << j->db->config_orig; - } - } - } - } - } - else - { - for (dependency_packages::const_iterator j (deps.begin ()); - j != deps.end (); - ++j) - { - if (j->name == nm && (j->db == nullptr || *j->db == db)) - { - if (i == deps.end () || i->db == nullptr) - i = j; + using constraint_type = build_package::constraint_type; - if (i->db != nullptr) - break; - } - } - } + const shared_ptr<available_package>& ap (p.available); + assert (ap != nullptr); // By definition. - bool user_exp (i != deps.end ()); - bool copy_dep (user_exp && i->db != nullptr && *i->db != db); + const version& av (ap->version); - // Collect the dependents for checking the version constraints, using - // their repository fragments for discovering available dependency package - // versions, etc. - // - // Note that if dependency needs to be copied, then we only consider its - // dependents in the current configurations which potentially can be - // repointed to it. Note that configurations of such dependents must - // contain the new dependency configuration in their dependency tree. + // List of the dependents which we have (unsuccessfully) tried to replace + // together with the lists of the constraining dependents. // - linked_databases dep_dbs; - - if (copy_dep) - { - for (database& db: i->db->dependent_configs ()) - { - if (find (cur_dbs.begin (), cur_dbs.end (), db) != cur_dbs.end ()) - dep_dbs.push_back (db); - } + vector<pair<package_key, vector<package_key>>> dpts; - // Bail out if no dependents can be repointed to the dependency. - // - if (dep_dbs.empty ()) - { - l5 ([&]{trace << *sp << db << ": can't repoint";}); - return no_change (); - } - } - else - dep_dbs = db.dependent_configs (); - - // Collect the dependents but bail out if the dependency is used but there - // are no user expectations regarding it. + // Try to replace a dependent, unless we have already tried to replace it. // - vector<pair<database&, package_dependent>> pds; - - for (database& ddb: dep_dbs) + auto try_replace = [&o, + &p, + &pkgs, + &cmdline_adjs, + &hold_pkgs, + &dep_pkgs, + &visited_dpts, + &dpts, + &trace] (package_key dk, const char* what) + -> optional<cmdline_adjustment> { - auto ds (query_dependents (ddb, nm, db)); - - if (!ds.empty ()) + if (find_if (dpts.begin (), dpts.end (), + [&dk] (const auto& v) {return v.first == dk;}) == + dpts.end ()) { - if (!user_exp) - return nullopt; - - for (auto& d: ds) - pds.emplace_back (ddb, move (d)); - } - } - - // Bail out if the dependency is unused. - // - if (pds.empty ()) - { - l5 ([&]{trace << *sp << db << ": unused";}); - - return evaluate_result {db, - nullptr /* available */, - nullptr /* repository_fragment */, - true /* unused */, - false /* system */}; - } + const build_package* d (pkgs.entered_build (dk)); - // The requested dependency database, version constraint, and system flag. - // - assert (i != deps.end ()); + // Always come from the dependency's constraints member. + // + assert (d != nullptr); - database& ddb (i->db != nullptr ? *i->db : db); - const optional<version_constraint>& dvc (i->constraint); // May be nullopt. - bool dsys (i->system); + // Skip the visited dependents since, by definition, we have already + // tried to replace them. + // + if (find (visited_dpts.begin (), visited_dpts.end (), d) == + visited_dpts.end ()) + { + l5 ([&]{trace << "try to replace " << what << ' ' + << d->available_name_version_db () << " of dependency " + << p.available_name_version_db () << " with some " + << "other version";}); - // The selected package in the desired database which we copy over. - // - // It is the current dependency package, if we don't copy, and may or may - // not exist otherwise. - // - shared_ptr<selected_package> dsp (db == ddb - ? sp - : ddb.find<selected_package> (nm)); + vector<package_key> uds; - // If a package in the desired database is already selected and matches - // the user expectations then no package change is required. - // - if (dsp != nullptr && dvc) - { - const version& sv (dsp->version); - bool ssys (dsp->system ()); + if (optional<cmdline_adjustment> a = try_replace_dependency ( + o, + *d, + pkgs, + hold_pkgs, + dep_pkgs, + cmdline_adjs, + uds, + what)) + { + return a; + } - if (ssys == dsys && - (ssys ? sv == *dvc->min_version : satisfies (sv, dvc))) - { - l5 ([&]{trace << *dsp << ddb << ": unchanged";}); - return no_change (); + dpts.emplace_back (move (dk), move (uds)); + } } - } - - // Build a set of repository fragments the dependent packages come from. - // Also cache the dependents and the constraints they apply to this - // dependency. - // - config_repo_fragments repo_frags; - dependent_constraints dpt_constrs; - - for (auto& pd: pds) - { - database& ddb (pd.first); - package_dependent& dep (pd.second); - - shared_ptr<selected_package> p (ddb.load<selected_package> (dep.name)); - - add_dependent_repo_fragments ( - ddb, - available_package_id (p->name, p->version), - repo_frags); - - dpt_constrs.emplace_back (ddb, move (p), move (dep.constraint)); - } - - return evaluate_dependency (db, - sp, - dvc, - dsys, - ddb, - dsp, - i->patch, - true /* explicitly */, - repo_frags, - dpt_constrs, - ignore_unsatisfiable); - } - - struct config_selected_package - { - database& db; - const shared_ptr<selected_package>& package; - - config_selected_package (database& d, - const shared_ptr<selected_package>& p) - : db (d), package (p) {} - - bool - operator== (const config_selected_package& v) const - { - return package->name == v.package->name && db == v.db; - } - - bool - operator< (const config_selected_package& v) const - { - int r (package->name.compare (v.package->name)); - return r != 0 ? (r < 0) : (db < v.db); - } - }; - - static optional<evaluate_result> - evaluate_dependency (database& db, - const shared_ptr<selected_package>& sp, - const optional<version_constraint>& dvc, - bool dsys, - database& ddb, - const shared_ptr<selected_package>& dsp, - bool patch, - bool explicitly, - const config_repo_fragments& rfs, - const dependent_constraints& dpt_constrs, - bool ignore_unsatisfiable) - { - tracer trace ("evaluate_dependency"); - const package_name& nm (sp->name); - - auto no_change = [&db] () - { - return evaluate_result {db, - nullptr /* available */, - nullptr /* repository_fragment */, - false /* unused */, - false /* system */}; + return nullopt; }; - // Build the list of available packages for the potential up/down-grade - // to, in the version-descending order. If patching, then we constrain the - // choice with the latest patch version and place no constraints if - // upgrading. For a system package we will try to find the available - // package that matches the user-specified system version (preferable for - // the configuration negotiation machinery) and, if fail, fallback to - // picking the latest one just to make sure the package is recognized. + // Try to replace unsatisfied dependents. // - optional<version_constraint> c; - - if (!dvc) + for (const constraint_type& c: p.constraints) { - assert (!dsys); // The version can't be empty for the system package. + const package_version_key& dvk (c.dependent); - if (patch) + if (dvk.version && !c.selected_dependent && !satisfies (av, c.value)) { - c = patch_constraint (sp, ignore_unsatisfiable); - - if (!c) + if (optional<cmdline_adjustment> a = try_replace ( + package_key (dvk.db, dvk.name), "unsatisfied dependent")) { - l5 ([&]{trace << *sp << db << ": non-patchable";}); - return no_change (); + return a; } } } - else if (!dsys || !wildcard (*dvc)) - c = dvc; - - vector<pair<shared_ptr<available_package>, - lazy_shared_ptr<repository_fragment>>> afs ( - find_available (nm, c, rfs)); - if (afs.empty () && dsys && c) - afs = find_available (nm, nullopt, rfs); - - // Go through up/down-grade candidates and pick the first one that - // satisfies all the dependents. Collect (and sort) unsatisfied dependents - // per the unsatisfiable version in case we need to print them. + // Try to replace conflicting dependents. // - using sp_set = set<config_selected_package>; - - vector<pair<version, sp_set>> unsatisfiable; - - bool stub (false); - - assert (!dsys || - (ddb.system_repository && - ddb.system_repository->find (nm) != nullptr)); - - for (auto& af: afs) + if (ucs != nullptr) { - shared_ptr<available_package>& ap (af.first); - const version& av (!dsys ? ap->version : *ap->system_version (ddb)); - - // If we aim to upgrade to the latest version and it tends to be less - // then the selected one, then what we currently have is the best that - // we can get, and so we return the "no change" result. - // - // Note that we also handle a package stub here. - // - if (!dvc && dsp != nullptr && av < dsp->version) + for (const unsatisfied_constraint& uc: *ucs) { - assert (!dsys); // Version can't be empty for the system package. - - // For the selected system package we still need to pick a source - // package version to downgrade to. - // - if (!dsp->system ()) - { - l5 ([&]{trace << *dsp << ddb << ": best";}); - return no_change (); - } - - // We can not upgrade the (system) package to a stub version, so just - // skip it. - // - if (ap->stub ()) - { - stub = true; - continue; - } - } - - // Check if the version satisfies all the dependents and collect - // unsatisfied ones. - // - bool satisfactory (true); - sp_set unsatisfied_dependents; + const package_version_key& dvk (uc.constraint.dependent); - for (const auto& dp: dpt_constrs) - { - if (!satisfies (av, dp.constraint)) + if (dvk.version) { - satisfactory = false; - - // Continue to collect dependents of the unsatisfiable version if - // we need to print them before failing. - // - if (ignore_unsatisfiable) - break; - - unsatisfied_dependents.emplace (dp.db, dp.package); + if (optional<cmdline_adjustment> a = try_replace ( + package_key (dvk.db, dvk.name), "conflicting dependent")) + { + return a; + } } } - - if (!satisfactory) - { - if (!ignore_unsatisfiable) - unsatisfiable.emplace_back (av, move (unsatisfied_dependents)); - - // If the dependency is expected to be configured as system, then bail - // out, as an available package version will always resolve to the - // system one (see above). - // - if (dsys) - break; - - continue; - } - - // If the best satisfactory version and the desired system flag perfectly - // match the ones of the selected package, then no package change is - // required. Otherwise, recommend an up/down-grade. - // - if (dsp != nullptr && av == dsp->version && dsp->system () == dsys) - { - l5 ([&]{trace << *dsp << ddb << ": unchanged";}); - return no_change (); - } - - l5 ([&]{trace << *sp << db << ": update to " - << package_string (nm, av, dsys) << ddb;}); - - return evaluate_result { - ddb, move (ap), move (af.second), false /* unused */, dsys}; - } - - // If we aim to upgrade to the latest version, then what we currently have - // is the only thing that we can get, and so returning the "no change" - // result, unless we need to upgrade a package configured as system. - // - if (!dvc && dsp != nullptr && !dsp->system ()) - { - assert (!dsys); // Version cannot be empty for the system package. - - l5 ([&]{trace << *dsp << ddb << ": only";}); - return no_change (); - } - - // If the version satisfying the desired dependency version constraint is - // unavailable or unsatisfiable for some dependents then we fail, unless - // requested not to do so. In the latter case we return the "no change" - // result. - // - if (ignore_unsatisfiable) - { - l5 ([&]{trace << package_string (nm, dvc, dsys) << ddb - << (unsatisfiable.empty () - ? ": no source" - : ": unsatisfiable");}); - - return no_change (); - } - - // If there are no unsatisfiable versions then the package is not present - // (or is not available in source) in its dependents' repositories. - // - if (unsatisfiable.empty ()) - { - diag_record dr (fail); - - if (!dvc && patch) - { - // Otherwise, we should have bailed out earlier (see above). - // - assert (dsp != nullptr && dsp->system ()); - - // Patch (as any upgrade) of a system package is always explicit, so - // we always fail and never treat the package as being up to date. - // - assert (explicitly); - - fail << "patch version for " << *sp << db << " is not available " - << "from its dependents' repositories"; - } - else if (!stub) - fail << package_string (nm, dsys ? nullopt : dvc) << ddb - << " is not available from its dependents' repositories"; - else // The only available package is a stub. - { - // Note that we don't advise to "build" the package as a system one as - // it is already as such (see above). - // - assert (!dvc && !dsys && dsp != nullptr && dsp->system ()); - - fail << package_string (nm, dvc) << ddb << " is not available in " - << "source from its dependents' repositories"; - } } - // Issue the diagnostics and fail. + // Try to replace unsatisfiable dependents. // - diag_record dr (fail); - dr << "package " << nm << ddb << " doesn't satisfy its dependents"; - - // Print the list of unsatisfiable versions together with dependents they - // don't satisfy: up to three latest versions with no more than five - // dependents each. - // - size_t nv (0); - for (const auto& u: unsatisfiable) + for (const constraint_type& c1: p.constraints) { - dr << info << package_string (nm, u.first) << " doesn't satisfy"; + const package_version_key& dvk (c1.dependent); - const sp_set& ps (u.second); - - size_t i (0), n (ps.size ()); - for (auto p (ps.begin ()); i != n; ++p) + if (dvk.version && !c1.selected_dependent) { - dr << (i == 0 ? " " : ", ") << *p->package << p->db; - - if (++i == 5 && n != 6) // Printing 'and 1 more' looks stupid. - break; - } - - if (i != n) - dr << " and " << n - i << " more"; - - if (++nv == 3 && unsatisfiable.size () != 4) - break; - } - - if (nv != unsatisfiable.size ()) - dr << info << "and " << unsatisfiable.size () - nv << " more"; - - dr << endf; - } - - // List of dependent packages whose immediate/recursive dependencies must be - // upgraded (specified with -i/-r on the command line). - // - struct recursive_package - { - database& db; - package_name name; - bool upgrade; // true -- upgrade, false -- patch. - bool recursive; // true -- recursive, false -- immediate. - }; - using recursive_packages = vector<recursive_package>; + const version_constraint& v1 (c1.value); - // Recursively check if immediate dependencies of this dependent must be - // upgraded or patched. Return true if it must be upgraded, false if - // patched, and nullopt otherwise. - // - static optional<bool> - upgrade_dependencies (database& db, - const package_name& nm, - const recursive_packages& rs, - bool recursion = false) - { - auto i (find_if (rs.begin (), rs.end (), - [&nm, &db] (const recursive_package& i) -> bool - { - return i.name == nm && i.db == db; - })); - - optional<bool> r; - - if (i != rs.end () && i->recursive >= recursion) - { - r = i->upgrade; + bool unsatisfiable (false); + for (const constraint_type& c2: p.constraints) + { + const version_constraint& v2 (c2.value); - if (*r) // Upgrade (vs patch)? - return r; - } + if (!satisfies (v1, v2) && !satisfies (v2, v1)) + { + unsatisfiable = true; + break; + } + } - for (database& ddb: db.dependent_configs ()) - { - for (auto& pd: query_dependents_cache (ddb, nm, db)) - { - // Note that we cannot end up with an infinite recursion for - // configured packages due to a dependency cycle (see order() for - // details). - // - if (optional<bool> u = upgrade_dependencies (ddb, pd.name, rs, true)) + if (unsatisfiable) { - if (!r || *r < *u) // Upgrade wins patch. + if (optional<cmdline_adjustment> a = try_replace ( + package_key (dvk.db, dvk.name), "unsatisfiable dependent")) { - r = u; - - if (*r) // Upgrade (vs patch)? - return r; + return a; } } } } - return r; - } - - // Evaluate a package (not necessarily dependency) and return a new desired - // version. If the result is absent (nullopt), then no changes to the - // package are necessary. Otherwise, the result is available_package to - // upgrade/downgrade to as well as the repository fragment it must come - // from. - // - // If the system package cannot be upgraded to the source one, not being - // found in the dependents repositories, then return nullopt if - // ignore_unsatisfiable argument is true and fail otherwise (see the - // evaluate_dependency() function description for details). - // - static optional<evaluate_result> - evaluate_recursive (database& db, - const shared_ptr<selected_package>& sp, - const recursive_packages& recs, - bool ignore_unsatisfiable) - { - tracer trace ("evaluate_recursive"); - - assert (sp != nullptr); - - // Build a set of repository fragment the dependent packages come from. - // Also cache the dependents and the constraints they apply to this - // dependency. - // - config_repo_fragments repo_frags; - dependent_constraints dpt_constrs; - - // Only collect repository fragments (for best version selection) of - // (immediate) dependents that have a hit (direct or indirect) in recs. - // Note, however, that we collect constraints from all the dependents. + // Try to replace constraining dependents. // - optional<bool> upgrade; - - for (database& ddb: db.dependent_configs ()) + for (const auto& dk: unsatisfied_dpts) { - for (auto& pd: query_dependents_cache (ddb, sp->name, db)) + if (optional<cmdline_adjustment> a = try_replace ( + dk, "constraining dependent")) { - shared_ptr<selected_package> p (ddb.load<selected_package> (pd.name)); - - dpt_constrs.emplace_back (ddb, p, move (pd.constraint)); - - if (optional<bool> u = upgrade_dependencies (ddb, pd.name, recs)) - { - if (!upgrade || *upgrade < *u) // Upgrade wins patch. - upgrade = u; - } - else - continue; - - // While we already know that the dependency upgrade is required, we - // continue to iterate over dependents, collecting the repository - // fragments and the constraints. - // - add_dependent_repo_fragments ( - ddb, - available_package_id (p->name, p->version), - repo_frags); + return a; } } - if (!upgrade) + // Try to replace dependents of the above dependents, recursively. + // + for (const auto& dep: dpts) { - l5 ([&]{trace << *sp << db << ": no hit";}); - return nullopt; - } + const build_package* d (pkgs.entered_build (dep.first)); - // Recommends the highest possible version. - // - optional<evaluate_result> r ( - evaluate_dependency (db, - sp, - nullopt /* desired */, - false /* desired_sys */, - db, - sp, - !*upgrade /* patch */, - false /* explicitly */, - repo_frags, - dpt_constrs, - ignore_unsatisfiable)); + assert (d != nullptr); - // Translate the "no change" result into nullopt. - // - assert (!r || !r->unused); - return r && r->available == nullptr ? nullopt : r; + if (optional<cmdline_adjustment> a = try_replace_dependent ( + o, + *d, + nullptr /* unsatisfied_constraints */, + pkgs, + cmdline_adjs, + dep.second, + hold_pkgs, + dep_pkgs, + visited_dpts)) + { + return a; + } + } + + return nullopt; } - // Return false if the plan execution was noop. + // Return false if the plan execution was noop. If unsatisfied dependents + // are specified then we are in the simulation mode. // static bool execute_plan (const pkg_build_options&, build_package_list&, - bool simulate, + unsatisfied_dependents* simulate, const function<find_database_function>&); using pkg_options = pkg_build_pkg_options; @@ -8528,13 +2938,14 @@ namespace bpkg dr << fail << "both --immediate|-i and --recursive|-r specified"; // The --immediate or --recursive option can only be specified with an - // explicit --upgrade or --patch. + // explicit --upgrade, --patch, or --deorphan. // if (const char* n = (o.immediate () ? "--immediate" : o.recursive () ? "--recursive" : nullptr)) { - if (!o.upgrade () && !o.patch ()) - dr << fail << n << " requires explicit --upgrade|-u or --patch|-p"; + if (!o.upgrade () && !o.patch () && !o.deorphan ()) + dr << fail << n << " requires explicit --upgrade|-u, --patch|-p, or " + << "--deorphan"; } if (((o.upgrade_immediate () ? 1 : 0) + @@ -8544,6 +2955,10 @@ namespace bpkg dr << fail << "multiple --(upgrade|patch)-(immediate|recursive) " << "specified"; + if (o.deorphan_immediate () && o.deorphan_recursive ()) + dr << fail << "both --deorphan-immediate and --deorphan-recursive " + << "specified"; + if (multi_config ()) { if (const char* opt = o.config_name_specified () ? "--config-name" : @@ -8570,13 +2985,16 @@ namespace bpkg dst.recursive (src.recursive ()); // If -r|-i was specified at the package level, then so should - // -u|-p. + // -u|-p and --deorphan. // if (!(dst.upgrade () || dst.patch ())) { dst.upgrade (src.upgrade ()); dst.patch (src.patch ()); } + + if (!dst.deorphan ()) + dst.deorphan (src.deorphan ()); } if (!(dst.upgrade_immediate () || dst.upgrade_recursive () || @@ -8588,6 +3006,12 @@ namespace bpkg dst.patch_recursive (src.patch_recursive ()); } + if (!(dst.deorphan_immediate () || dst.deorphan_recursive ())) + { + dst.deorphan_immediate (src.deorphan_immediate ()); + dst.deorphan_recursive (src.deorphan_recursive ()); + } + dst.dependency (src.dependency () || dst.dependency ()); dst.keep_out (src.keep_out () || dst.keep_out ()); dst.disfigure (src.disfigure () || dst.disfigure ()); @@ -8631,19 +3055,22 @@ namespace bpkg static bool compare_options (const pkg_options& x, const pkg_options& y) { - return x.keep_out () == y.keep_out () && - x.disfigure () == y.disfigure () && - x.dependency () == y.dependency () && - x.upgrade () == y.upgrade () && - x.patch () == y.patch () && - x.immediate () == y.immediate () && - x.recursive () == y.recursive () && - x.upgrade_immediate () == y.upgrade_immediate () && - x.upgrade_recursive () == y.upgrade_recursive () && - x.patch_immediate () == y.patch_immediate () && - x.patch_recursive () == y.patch_recursive () && - x.checkout_root () == y.checkout_root () && - x.checkout_purge () == y.checkout_purge (); + return x.keep_out () == y.keep_out () && + x.disfigure () == y.disfigure () && + x.dependency () == y.dependency () && + x.upgrade () == y.upgrade () && + x.patch () == y.patch () && + x.deorphan () == y.deorphan () && + x.immediate () == y.immediate () && + x.recursive () == y.recursive () && + x.upgrade_immediate () == y.upgrade_immediate () && + x.upgrade_recursive () == y.upgrade_recursive () && + x.patch_immediate () == y.patch_immediate () && + x.patch_recursive () == y.patch_recursive () && + x.deorphan_immediate () == y.deorphan_immediate () && + x.deorphan_recursive () == y.deorphan_recursive () && + x.checkout_root () == y.checkout_root () && + x.checkout_purge () == y.checkout_purge (); } int @@ -8681,7 +3108,11 @@ namespace bpkg << "specified" << info << "run 'bpkg help pkg-build' for more information"; - if (!args.more () && !o.upgrade () && !o.patch ()) + if (o.sys_no_query () && o.sys_install ()) + fail << "both --sys-no-query and --sys-install specified" << + info << "run 'bpkg help pkg-build' for more information"; + + if (!args.more () && !o.upgrade () && !o.patch () && !o.deorphan ()) fail << "package name argument expected" << info << "run 'bpkg help pkg-build' for more information"; @@ -8704,6 +3135,10 @@ namespace bpkg ? empty_string : '[' + config_dirs[0].representation () + ']')); + // Command line as a dependent. + // + package_version_key cmd_line (mdb, "command line"); + current_configs.push_back (mdb); if (config_dirs.size () != 1) @@ -8729,6 +3164,8 @@ namespace bpkg if (!current (db)) current_configs.push_back (db); } + + t.commit (); } validate_options (o, ""); // Global package options. @@ -8739,7 +3176,7 @@ namespace bpkg // will modify the cached instance, which means our list will always "see" // their updated state. // - // Also note that rep_fetch() must be called in session. + // Also note that rep_fetch() and pkg_fetch() must be called in session. // session ses; @@ -8850,7 +3287,13 @@ namespace bpkg if (a.find ('=') == string::npos) fail << "unexpected group argument '" << a << "'"; - cvs.push_back (move (trim (a))); + trim (a); + + if (a[0] == '!') + fail << "global override in package-specific configuration " + << "variable '" << a << "'"; + + cvs.push_back (move (a)); } } @@ -9126,6 +3569,14 @@ namespace bpkg string () /* reason for "fetching ..." */); } + // Now, as repo_configs is filled and the repositories are fetched mask + // the repositories, if any. + // + if (o.mask_repository_specified () || o.mask_repository_uuid_specified ()) + rep_mask (o.mask_repository (), + o.mask_repository_uuid (), + current_configs); + // Expand the package specs into individual package args, parsing them // into the package scheme, name, and version constraint components, and // also saving associated options and configuration variables. @@ -9147,6 +3598,12 @@ namespace bpkg string value; pkg_options options; strings config_vars; + + // If schema is sys then this member indicates whether the constraint + // came from the system package manager (not NULL) or user/fallback + // (NULL). + // + const system_package_status* system_status; }; auto arg_parsed = [] (const pkg_arg& a) {return !a.name.empty ();}; @@ -9215,16 +3672,19 @@ namespace bpkg const pkg_options& o (a.options); - add_bool ("--keep-out", o.keep_out ()); - add_bool ("--disfigure", o.disfigure ()); - add_bool ("--upgrade", o.upgrade ()); - add_bool ("--patch", o.patch ()); - add_bool ("--immediate", o.immediate ()); - add_bool ("--recursive", o.recursive ()); - add_bool ("--upgrade-immediate", o.upgrade_immediate ()); - add_bool ("--upgrade-recursive", o.upgrade_recursive ()); - add_bool ("--patch-immediate", o.patch_immediate ()); - add_bool ("--patch-recursive", o.patch_recursive ()); + add_bool ("--keep-out", o.keep_out ()); + add_bool ("--disfigure", o.disfigure ()); + add_bool ("--upgrade", o.upgrade ()); + add_bool ("--patch", o.patch ()); + add_bool ("--deorphan", o.deorphan ()); + add_bool ("--immediate", o.immediate ()); + add_bool ("--recursive", o.recursive ()); + add_bool ("--upgrade-immediate", o.upgrade_immediate ()); + add_bool ("--upgrade-recursive", o.upgrade_recursive ()); + add_bool ("--patch-immediate", o.patch_immediate ()); + add_bool ("--patch-recursive", o.patch_recursive ()); + add_bool ("--deorphan-immediate", o.deorphan_immediate ()); + add_bool ("--deorphan-recursive", o.deorphan_recursive ()); if (o.checkout_root_specified ()) add_string ("--checkout-root", o.checkout_root ().string ()); @@ -9262,23 +3722,141 @@ namespace bpkg return r; }; - // Add the system package authoritative information to the database's - // system repository, unless it already contains authoritative information - // for this package. + // Figure out the system package version unless explicitly specified and + // add the system package authoritative information to the database's + // system repository unless the database is NULL or it already contains + // authoritative information for this package. Return the figured out + // system package version as constraint. // // Note that it is assumed that all the possible duplicates are handled // elsewhere/later. // - auto add_system_package = [] (database& db, - const package_name& nm, - const version& v) + auto add_system_package = [&o] (database* db, + const package_name& nm, + optional<version_constraint> vc, + const system_package_status* sps, + vector<shared_ptr<available_package>>* stubs) + -> pair<version_constraint, const system_package_status*> { - assert (db.system_repository); + if (!vc) + { + assert (sps == nullptr); + + // See if we should query the system package manager. + // + if (!sys_pkg_mgr) + sys_pkg_mgr = o.sys_no_query () + ? nullptr + : make_consumption_system_package_manager (o, + host_triplet, + o.sys_distribution (), + o.sys_architecture (), + o.sys_install (), + !o.sys_no_fetch (), + o.sys_yes (), + o.sys_sudo ()); + if (*sys_pkg_mgr != nullptr) + { + system_package_manager& spm (**sys_pkg_mgr); + + // First check the cache. + // + optional<const system_package_status*> os (spm.status (nm, nullptr)); + + available_packages aps; + if (!os) + { + // If no cache hit, then collect the available packages for the + // mapping information. + // + aps = find_available_all (current_configs, nm); + + // If no source/stub for the package (and thus no mapping), issue + // diagnostics consistent with other such places unless explicitly + // allowed by the user. + // + if (aps.empty ()) + { + if (!o.sys_no_stub ()) + fail << "unknown package " << nm << + info << "consider specifying --sys-no-stub or " << nm << "/*"; + + // Add the stub package to the imaginary system repository (like + // the user-specified case below). + // + if (stubs != nullptr) + stubs->push_back (make_shared<available_package> (nm)); + } + } - const system_package* sp (db.system_repository->find (nm)); + // This covers both our diagnostics below as well as anything that + // might be issued by status(). + // + auto df = make_diag_frame ( + [&nm] (diag_record& dr) + { + dr << info << "specify " << nm << "/* if package is not " + << "installed with system package manager"; + + dr << info << "specify --sys-no-query to disable system " + << "package manager interactions"; + }); + + if (!os) + { + os = spm.status (nm, &aps); + assert (os); + } + + if ((sps = *os) != nullptr) + vc = version_constraint (sps->version); + else + { + diag_record dr (fail); + + dr << "no installed " << (o.sys_install () ? "or available " : "") + << "system package for " << nm; + + if (!o.sys_install ()) + dr << info << "specify --sys-install to try to install it"; + } + } + else + vc = version_constraint (wildcard_version); + } + else + { + // The system package may only have an exact/wildcard version + // specified. + // + assert (vc->min_version == vc->max_version); + + // For system packages not associated with a specific repository + // location add the stub package to the imaginary system repository + // (see below for details). + // + if (stubs != nullptr) + stubs->push_back (make_shared<available_package> (nm)); + } + + if (db != nullptr) + { + assert (db->system_repository); - if (sp == nullptr || !sp->authoritative) - db.system_repository->insert (nm, v, true /* authoritative */); + const system_package* sp (db->system_repository->find (nm)); + + // Note that we don't check for the version match here since that's + // handled by check_dup() lambda at a later stage, which covers both + // db and no-db cases consistently. + // + if (sp == nullptr || !sp->authoritative) + db->system_repository->insert (nm, + *vc->min_version, + true /* authoritative */, + sps); + } + + return make_pair (move (*vc), sps); }; // Create the parsed package argument. Issue diagnostics and fail if the @@ -9290,15 +3868,23 @@ namespace bpkg package_name nm, optional<version_constraint> vc, pkg_options os, - strings vs) -> pkg_arg + strings vs, + vector<shared_ptr<available_package>>* stubs = nullptr) + -> pkg_arg { assert (!vc || !vc->empty ()); // May not be empty if present. if (db == nullptr) assert (sc == package_scheme::sys && os.dependency ()); - pkg_arg r { - db, sc, move (nm), move (vc), string (), move (os), move (vs)}; + pkg_arg r {db, + sc, + move (nm), + move (vc), + string () /* value */, + move (os), + move (vs), + nullptr /* system_status */}; // Verify that the package database is specified in the multi-config // mode, unless this is a system dependency package. @@ -9317,17 +3903,16 @@ namespace bpkg { case package_scheme::sys: { - if (!r.constraint) - r.constraint = version_constraint (wildcard_version); - - // The system package may only have an exact/wildcard version - // specified. - // - assert (r.constraint->min_version == r.constraint->max_version); + assert (stubs != nullptr); - if (db != nullptr) - add_system_package (*db, r.name, *r.constraint->min_version); + auto sp (add_system_package (db, + r.name, + move (r.constraint), + nullptr /* system_package_status */, + stubs)); + r.constraint = move (sp.first); + r.system_status = sp.second; break; } case package_scheme::none: break; // Nothing to do. @@ -9349,7 +3934,8 @@ namespace bpkg nullopt /* constraint */, move (v), move (os), - move (vs)}; + move (vs), + nullptr /* system_status */}; }; vector<pkg_arg> pkg_args; @@ -9412,13 +3998,6 @@ namespace bpkg parse_package_version_constraint ( s, sys, version_flags (sc), version_only (sc))); - // For system packages not associated with a specific repository - // location add the stub package to the imaginary system - // repository (see above for details). - // - if (sys && vc) - stubs.push_back (make_shared<available_package> (n)); - pkg_options& o (ps.options); // Disregard the (main) database for a system dependency with @@ -9435,7 +4014,8 @@ namespace bpkg move (n), move (vc), move (o), - move (ps.config_vars))); + move (ps.config_vars), + &stubs)); } else // Add unparsed. pkg_args.push_back (arg_raw (ps.db, @@ -9669,7 +4249,8 @@ namespace bpkg move (n), move (vc), ps.options, - ps.config_vars)); + ps.config_vars, + &stubs)); } } } @@ -9691,6 +4272,32 @@ namespace bpkg dependency_packages dep_pkgs; recursive_packages rec_pkgs; + // Note that the command line adjustments which resolve the unsatisfied + // dependent issue (see unsatisfied_dependents for details) may + // potentially be sub-optimal, since we do not perform the full + // backtracking by trying all the possible adjustments and picking the + // most optimal combination. Instead, we keep collecting adjustments until + // either the package builds collection succeeds or there are no more + // adjustment combinations to try (and we don't try all of them). As a + // result we, for example, may end up with some redundant constraints on + // the command line just because the respective dependents have been + // evaluated first. Generally, dropping all the redundant adjustments can + // potentially be quite time-consuming, since we would need to try + // dropping all their possible combinations. We, however, will implement + // the refinement for only the common case (adjustments are independent), + // trying to drop just one adjustment per the refinement cycle iteration + // and wait and see how it goes. + // + cmdline_adjustments cmdline_adjs (hold_pkgs, dep_pkgs); + + // If both are present, then we are in the command line adjustments + // refinement cycle, where cmdline_refine_adjustment is the adjustment + // being currently dropped and cmdline_refine_index is its index on the + // stack (as it appears at the beginning of the cycle). + // + optional<cmdline_adjustment> cmdline_refine_adjustment; + optional<size_t> cmdline_refine_index; + { // Check if the package is a duplicate. Return true if it is but // harmless. @@ -9742,7 +4349,7 @@ namespace bpkg !compare_options (a.options, pa.options) || a.config_vars != pa.config_vars)) fail << "duplicate package " << pa.name << - info << "first mentioned as " << arg_string (r.first->second) << + info << "first mentioned as " << arg_string (a) << info << "second mentioned as " << arg_string (pa); return !r.second; @@ -9750,6 +4357,120 @@ namespace bpkg transaction t (mdb); + // Return the available package that matches the specified orphan best + // (see evaluate_dependency() description for details). Also return the + // repository fragment the package comes from. Return a pair of NULLs if + // no suitable package has been found. + // + auto find_orphan_match = + [] (const shared_ptr<selected_package>& sp, + const lazy_shared_ptr<repository_fragment>& root) + { + using available = pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>>; + + assert (sp != nullptr); + + const package_name& n (sp->name); + const version& v (sp->version); + optional<version_constraint> vc {version_constraint (v)}; + + // Note that non-zero iteration makes a version non-standard, so we + // reset it to 0 to produce the patch/minor constraints. + // + version vr (v.epoch, + v.upstream, + v.release, + v.revision, + 0 /* iteration */); + + optional<version_constraint> pc ( + patch_constraint (n, vr, true /* quiet */)); + + optional<version_constraint> mc ( + minor_constraint (n, vr, true /* quiet */)); + + // Note: explicit revision makes query_available() to always consider + // revisions (but not iterations) regardless of the revision argument + // value. + // + optional<version_constraint> verc { + version_constraint (version (v.epoch, + v.upstream, + v.release, + v.revision ? v.revision : 0, + 0 /* iteration */))}; + + optional<version_constraint> vlc { + version_constraint (version (v.epoch, + v.upstream, + v.release, + nullopt, + 0 /* iteration */))}; + + // Find the latest available non-stub package, optionally matching a + // constraint and considering revision. If a package is found, then + // cache it together with the repository fragment it comes from and + // return true. + // + available find_result; + const version* find_version (nullptr); + auto find = [&n, + &root, + &find_result, + &find_version] (const optional<version_constraint>& c, + bool revision = false) -> bool + { + available r ( + find_available_one (n, c, root, false /* prereq */, revision)); + + const shared_ptr<available_package>& ap (r.first); + + if (ap != nullptr && !ap->stub ()) + { + find_result = move (r); + find_version = &find_result.first->version; + return true; + } + else + return false; + }; + + if (// Same version, revision, and iteration. + // + find (vc, true) || + // + // Latest iteration of same version and revision. + // + find (verc) || + // + // Later revision of same version. + // + (find (vlc) && + find_version->compare (v, + false /* revision */, + true /* iteration */) > 0) || + // + // Later patch of same version. + // + (pc && find (pc) && + find_version->compare (v, true /* revision */) > 0) || + // + // Later minor of same version. + // + (mc && find (mc) && + find_version->compare (v, true /* revision */) > 0) || + // + // Latest available version, including earlier. + // + find (nullopt)) + { + return find_result; + } + + return available (); + }; + // Here is what happens here: for unparsed package args we are going to // try and guess whether we are dealing with a package archive, package // directory, or package name/version by first trying it as an archive, @@ -9774,6 +4495,7 @@ namespace bpkg // lazy_shared_ptr<repository_fragment> af; shared_ptr<available_package> ap; + bool existing (false); // True if build as an archive or directory. if (!arg_parsed (pa)) { @@ -9796,22 +4518,16 @@ namespace bpkg pkg_verify (o, a, true /* ignore_unknown */, + false /* ignore_toolchain */, false /* expand_values */, true /* load_buildfiles */, - true /* complete_depends */, + true /* complete_values */, diag ? 2 : 1)); // This is a package archive. // l4 ([&]{trace << "archive '" << a << "': " << arg_string (pa);}); - // Supporting this would complicate things a bit, but we may add - // support for it one day. - // - if (pa.options.dependency ()) - fail << "package archive '" << a - << "' may not be built as a dependency"; - pa = arg_package (pdb, package_scheme::none, m.name, @@ -9822,6 +4538,9 @@ namespace bpkg af = root; ap = make_shared<available_package> (move (m)); ap->locations.push_back (package_location {root, move (a)}); + + existing_packages.push_back (make_pair (ref (*pdb), ap)); + existing = true; } } catch (const invalid_path&) @@ -9862,10 +4581,15 @@ namespace bpkg o, d, true /* ignore_unknown */, + false /* ignore_toolchain */, true /* load_buildfiles */, [&o, &d, &pvi] (version& v) { - pvi = package_version (o, d); + // Note that we also query subprojects since the package + // information will be used for the subsequent + // package_iteration() call. + // + pvi = package_version (o, d, b_info_flags::subprojects); if (pvi.version) v = move (*pvi.version); @@ -9877,13 +4601,6 @@ namespace bpkg l4 ([&]{trace << "directory '" << d << "': " << arg_string (pa);}); - // Supporting this would complicate things a bit, but we may - // add support for it one day. - // - if (pa.options.dependency ()) - fail << "package directory '" << d - << "' may not be built as a dependency"; - // Fix-up the package version to properly decide if we need to // upgrade/downgrade the package. // @@ -9908,6 +4625,9 @@ namespace bpkg ap = make_shared<available_package> (move (m)); af = root; ap->locations.push_back (package_location {root, move (d)}); + + existing_packages.push_back (make_pair (ref (*pdb), ap)); + existing = true; } } catch (const invalid_path&) @@ -9929,6 +4649,7 @@ namespace bpkg // shared_ptr<selected_package> sp; bool patch (false); + bool deorphan (false); if (ap == nullptr) { @@ -9968,12 +4689,13 @@ namespace bpkg lazy_shared_ptr<repository_fragment> root (*pdb, empty_string); - // Either get the user-specified version or the latest allowed - // for a source code package. For a system package we will try - // to find the available package that matches the user-specified - // system version (preferable for the configuration negotiation - // machinery) and, if fail, fallback to picking the latest one - // just to make sure the package is recognized. + // Get the user-specified version, the latest allowed version, + // or the orphan best match for a source code package. For a + // system package we will try to find the available package that + // matches the user-specified system version (preferable for the + // configuration negotiation machinery) and, if fail, fallback + // to picking the latest one just to make sure the package is + // recognized. // optional<version_constraint> c; @@ -10003,10 +4725,48 @@ namespace bpkg else if (!sys || !wildcard (*pa.constraint)) c = pa.constraint; - auto rp (find_available_one (pa.name, c, root)); + if (pa.options.deorphan ()) + { + if (!sys) + { + if (sp == nullptr) + sp = pdb->find<selected_package> (pa.name); - if (rp.first == nullptr && sys && c) - rp = find_available_one (pa.name, nullopt, root); + if (sp != nullptr && orphan_package (*pdb, sp)) + deorphan = true; + } + + // If the package is not an orphan, its version is not + // constrained and upgrade/patch is not requested, then just + // skip the package. + // + if (!deorphan && + !pa.constraint && + !pa.options.upgrade () && + !pa.options.patch ()) + { + ++i; + continue; + } + } + + pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>> rp ( + deorphan && + !pa.constraint && + !pa.options.upgrade () && + !pa.options.patch () + ? find_orphan_match (sp, root) + : find_available_one (pa.name, c, root)); + + if (rp.first == nullptr && sys) + { + available_packages aps ( + find_available_all (repo_configs, pa.name)); + + if (!aps.empty ()) + rp = move (aps.front ()); + } ap = move (rp.first); af = move (rp.second); @@ -10026,22 +4786,55 @@ namespace bpkg continue; // Save (both packages to hold and dependencies) as dependents for - // recursive upgrade. + // recursive upgrade/deorphaning. // { - optional<bool> u; - optional<bool> r; + // Recursive/immediate upgrade/patch. + // + optional<bool> r; // true -- recursive, false -- immediate. + optional<bool> u; // true -- upgrade, false -- patch. + + // Recursive/immediate deorphaning. + // + optional<bool> d; // true -- recursive, false -- immediate. const auto& po (pa.options); - if (po.upgrade_immediate ()) { u = true; r = false; } - else if (po.upgrade_recursive ()) { u = true; r = true; } - else if ( po.patch_immediate ()) { u = false; r = false; } - else if ( po.patch_recursive ()) { u = false; r = true; } - else if ( po.immediate ()) { u = po.upgrade (); r = false; } - else if ( po.recursive ()) { u = po.upgrade (); r = true; } + // Note that, for example, --upgrade-immediate wins over the + // --upgrade --recursive options pair. + // + if (po.immediate ()) + { + if (po.upgrade () || po.patch ()) + { + r = false; + u = po.upgrade (); + } + + if (po.deorphan ()) + d = false; + } + else if (po.recursive ()) + { + if (po.upgrade () || po.patch ()) + { + r = true; + u = po.upgrade (); + } + + if (po.deorphan ()) + d = true; + } + + if (po.upgrade_immediate ()) { u = true; r = false; } + else if (po.upgrade_recursive ()) { u = true; r = true; } + else if ( po.patch_immediate ()) { u = false; r = false; } + else if ( po.patch_recursive ()) { u = false; r = true; } - if (r) + if (po.deorphan_immediate ()) { d = false; } + else if (po.deorphan_recursive ()) { d = true; } + + if (r || d) { l4 ([&]{trace << "stash recursive package " << arg_string (pa);}); @@ -10050,7 +4843,9 @@ namespace bpkg // configuration. // if (pdb != nullptr) - rec_pkgs.push_back (recursive_package {*pdb, pa.name, *u, *r}); + rec_pkgs.push_back (recursive_package {*pdb, pa.name, + r, u && *u, + d}); } } @@ -10062,42 +4857,70 @@ namespace bpkg bool sys (arg_sys (pa)); - // Make sure that the package is known. - // - auto apr (find_available (repo_configs, - pa.name, - !sys ? pa.constraint : nullopt)); + if (pdb != nullptr) + sp = pdb->find<selected_package> (pa.name); - if (apr.empty ()) + // Make sure that the package is known. Only allow to unhold an + // unknown orphaned selected package (with the view that there is + // a good chance it will get dropped; and if not, such an unhold + // should be harmless). + // + if (!existing && + find_available (repo_configs, + pa.name, + !sys ? pa.constraint : nullopt).empty ()) { - diag_record dr (fail); - dr << "unknown package " << arg_string (pa, false /* options */); - check_any_available (repo_configs, t, &dr); + // Don't fail if the selected package is held and satisfies the + // constraints, if specified. Note that we may still fail later + // with the "not available from its dependents' repositories" + // error if the dependency is requested to be deorphaned and all + // its dependents are orphaned. + // + if (!(sp != nullptr && + sp->hold_package && + (!pa.constraint || satisfies (sp->version, pa.constraint)))) + { + string n (arg_string (pa, false /* options */)); + + diag_record dr (fail); + dr << "unknown package " << n; + if (sys) + { + // Feels like we can't end up here if the version was specified + // explicitly. + // + dr << info << "consider specifying " << n << "/*"; + } + else + check_any_available (repo_configs, t, &dr); + } } if (pdb != nullptr) - { - // Save before the name move. - // - sp = pdb->find<selected_package> (pa.name); - pkg_confs.emplace_back (*pdb, pa.name); - } + + bool hold_version (pa.constraint.has_value ()); dep_pkgs.push_back ( dependency_package {pdb, move (pa.name), move (pa.constraint), + hold_version, move (sp), sys, - pa.options.patch (), + existing, + (pa.options.upgrade () || pa.options.patch () + ? pa.options.upgrade () + : optional<bool> ()), + pa.options.deorphan (), pa.options.keep_out (), pa.options.disfigure (), (pa.options.checkout_root_specified () ? move (pa.options.checkout_root ()) : optional<dir_path> ()), pa.options.checkout_purge (), - move (pa.config_vars)}); + move (pa.config_vars), + pa.system_status}); continue; } @@ -10123,6 +4946,8 @@ namespace bpkg bool found (true); bool sys_advise (false); + bool sys (arg_sys (pa)); + // If the package is not available from the repository we can try to // create it from the orphaned selected package. Meanwhile that // doesn't make sense for a system package. The only purpose to @@ -10130,7 +4955,7 @@ namespace bpkg // package is not in the repository then there is no dependent for it // (otherwise the repository would be broken). // - if (!arg_sys (pa)) + if (!sys) { // If we failed to find the requested package we can still check if // the package name is present in the repositories and if that's the @@ -10155,17 +4980,18 @@ namespace bpkg // if (pa.constraint) { - for (;;) + for (;;) // Breakout loop. { if (ap != nullptr) // Must be that version, see above. break; // Otherwise, our only chance is that the already selected object - // satisfies the version constraint. + // satisfies the version constraint, unless we are deorphaning. // - if (sp != nullptr && - !sp->system () && - satisfies (sp->version, pa.constraint)) + if (sp != nullptr && + !sp->system () && + satisfies (sp->version, pa.constraint) && + !deorphan) break; // Derive ap from sp below. found = false; @@ -10173,13 +4999,10 @@ namespace bpkg } } // - // No explicit version was specified by the user (not relevant for a - // system package, see above). + // No explicit version was specified by the user. // else { - assert (!arg_sys (pa)); - if (ap != nullptr) { assert (!ap->stub ()); @@ -10188,14 +5011,17 @@ namespace bpkg // we have a newer version, we treat it as an upgrade request; // otherwise, why specify the package in the first place? We just // need to check if what we already have is "better" (i.e., - // newer). + // newer), unless we are deorphaning. // - if (sp != nullptr && !sp->system () && ap->version < sp->version) + if (sp != nullptr && + !sp->system () && + ap->version < sp->version && + !deorphan) ap = nullptr; // Derive ap from sp below. } else { - if (sp == nullptr || sp->system ()) + if (sp == nullptr || sp->system () || deorphan) found = false; // Otherwise, derive ap from sp below. @@ -10216,15 +5042,30 @@ namespace bpkg if (!sys_advise) { - dr << "unknown package " << pa.name; + // Note that if the package is not system and its version was + // explicitly specified, then we can only be here if no version of + // this package is available in source from the repository + // (otherwise we would advise to configure it as a system package; + // see above). Thus, let's not print it's version constraint in + // this case. + // + // Also note that for a system package we can't end up here if the + // version was specified explicitly. + // + string n (package_string (pa.name, nullopt /* vc */, sys)); + + dr << "unknown package " << n; // Let's help the new user out here a bit. // - check_any_available (*pdb, t, &dr); + if (sys) + dr << info << "consider specifying " << n << "/*"; + else + check_any_available (*pdb, t, &dr); } else { - assert (!arg_sys (pa)); + assert (!sys); dr << arg_string (pa, false /* options */) << " is not available in source"; @@ -10241,7 +5082,7 @@ namespace bpkg // if (ap == nullptr) { - assert (sp != nullptr && sp->system () == arg_sys (pa)); + assert (sp != nullptr && sp->system () == sys); auto rp (make_available_fragment (o, *pdb, sp)); ap = move (rp.first); @@ -10257,8 +5098,16 @@ namespace bpkg bool keep_out (pa.options.keep_out () && sp != nullptr && sp->external ()); + bool replace ((existing && sp != nullptr) || deorphan); + // Finally add this package to the list. // + optional<bool> upgrade (sp != nullptr && + !pa.constraint && + (pa.options.upgrade () || pa.options.patch ()) + ? pa.options.upgrade () + : optional<bool> ()); + // @@ Pass pa.configure_only() when support for package-specific // --configure-only is added. // @@ -10276,7 +5125,7 @@ namespace bpkg true, // Hold package. pa.constraint.has_value (), // Hold version. {}, // Constraints. - arg_sys (pa), + sys, keep_out, pa.options.disfigure (), false, // Configure-only. @@ -10285,9 +5134,11 @@ namespace bpkg : optional<dir_path> ()), pa.options.checkout_purge (), move (pa.config_vars), - {package_key {mdb, ""}}, // Required by (command line). + upgrade, + deorphan, + {cmd_line}, // Required by (command line). false, // Required by dependents. - 0}; // State flags. + replace ? build_package::build_replace : uint16_t (0)}; l4 ([&]{trace << "stash held package " << p.available_name_version_db ();}); @@ -10299,7 +5150,7 @@ namespace bpkg // if (pa.constraint) p.constraints.emplace_back ( - mdb, "command line", move (*pa.constraint)); + move (*pa.constraint), cmd_line.db, cmd_line.name.string ()); pkg_confs.emplace_back (p.db, p.name ()); @@ -10314,7 +5165,7 @@ namespace bpkg // command line option to enable this behavior. // if (hold_pkgs.empty () && dep_pkgs.empty () && - (o.upgrade () || o.patch ())) + (o.upgrade () || o.patch () || o.deorphan ())) { for (database& cdb: current_configs) { @@ -10348,7 +5199,26 @@ namespace bpkg continue; } - auto apr (find_available_one (name, pc, root)); + bool deorphan (false); + + if (o.deorphan ()) + { + // If the package is not an orphan and upgrade/patch is not + // requested, then just skip the package. + // + if (orphan_package (cdb, sp)) + deorphan = true; + else if (!o.upgrade () && !o.patch ()) + continue; + } + + // In the deorphan mode with no upgrade/patch requested pick the + // version that matches the orphan best. Otherwise, pick the patch + // or the latest available version, as requested. + // + auto apr (deorphan && !o.upgrade () && !o.patch () + ? find_orphan_match (sp, root) + : find_available_one (name, pc, root)); shared_ptr<available_package> ap (move (apr.first)); if (ap == nullptr || ap->stub ()) @@ -10356,11 +5226,13 @@ namespace bpkg diag_record dr (fail); dr << name << " is not available"; - if (ap != nullptr) + if (ap != nullptr) // Stub? + { dr << " in source" << info << "consider building it as " - << package_string (name, version (), true /* system */) - << " if it is available from the system"; + << package_string (name, version (), true /* system */) + << " if it is available from the system"; + } // Let's help the new user out here a bit. // @@ -10377,40 +5249,53 @@ namespace bpkg // build_package p { build_package::build, - cdb, - move (sp), - move (ap), - move (apr.second), - nullopt, // Dependencies. - nullopt, // Dependencies alternatives. - nullopt, // Package skeleton. - nullopt, // Postponed dependency alternatives. - false, // Recursive collection. - true, // Hold package. - false, // Hold version. - {}, // Constraints. - false, // System package. - keep_out, - o.disfigure (), - false, // Configure-only. - nullopt, // Checkout root. - false, // Checkout purge. - strings (), // Configuration variables. - {package_key {mdb, ""}}, // Required by (command line). - false, // Required by dependents. - 0}; // State flags. + cdb, + move (sp), + move (ap), + move (apr.second), + nullopt, // Dependencies. + nullopt, // Dependencies alternatives. + nullopt, // Package skeleton. + nullopt, // Postponed dependency alternatives. + false, // Recursive collection. + true, // Hold package. + false, // Hold version. + {}, // Constraints. + false, // System package. + keep_out, + o.disfigure (), + false, // Configure-only. + nullopt, // Checkout root. + false, // Checkout purge. + strings (), // Configuration variables. + (o.upgrade () || o.patch () + ? o.upgrade () + : optional<bool> ()), + deorphan, + {cmd_line}, // Required by (command line). + false, // Required by dependents. + deorphan ? build_package::build_replace : uint16_t (0)}; l4 ([&]{trace << "stash held package " << p.available_name_version_db ();}); hold_pkgs.push_back (move (p)); - // If there are also -i|-r, then we are also upgrading dependencies - // of all held packages. + // If there are also -i|-r, then we are also upgrading and/or + // deorphaning dependencies of all held packages. // if (o.immediate () || o.recursive ()) - rec_pkgs.push_back ( - recursive_package {cdb, name, o.upgrade (), o.recursive ()}); + { + rec_pkgs.push_back (recursive_package { + cdb, name, + (o.upgrade () || o.patch () + ? o.recursive () + : optional<bool> ()), + o.upgrade (), + (o.deorphan () + ? o.recursive () + : optional<bool> ())}); + } } } } @@ -10472,6 +5357,9 @@ namespace bpkg // dependencies to up/down-grade, and unused dependencies to drop. We call // this the plan. // + // Note: for the sake of brevity we also assume the package replacement + // wherever we mention the package up/down-grade in this description. + // // The way we do it is tricky: we first create the plan based on build-to- // holds (i.e., the user selected). Next, to decide whether we need to // up/down-grade or drop any dependecies we need to take into account an @@ -10510,6 +5398,11 @@ namespace bpkg // grade order where any subsequent entry does not affect the decision of // the previous ones. // + // Note that we also need to rebuild the plan from scratch on adding a new + // up/down-grade/drop if any dependency configuration negotiation has been + // performed, since any package replacement may affect the already + // negotiated configurations. + // // Package managers are an easy, already solved problem, right? // build_packages pkgs; @@ -10524,13 +5417,18 @@ namespace bpkg shared_ptr<available_package> available; lazy_shared_ptr<bpkg::repository_fragment> repository_fragment; - bool system; + bool system; + bool existing; // Build as an existing archive or directory. + optional<bool> upgrade; + bool deorphan; }; vector<dep> deps; + existing_dependencies existing_deps; + deorphaned_dependencies deorphaned_deps; - replaced_versions replaced_vers; - postponed_dependencies postponed_deps; - postponed_positions postponed_poss; + replaced_versions replaced_vers; + postponed_dependencies postponed_deps; + unacceptable_alternatives unacceptable_alts; // Map the repointed dependents to the replacement flags (see // repointed_dependents for details), unless --no-move is specified. @@ -10597,12 +5495,12 @@ namespace bpkg // during the package collection) because we want to enter them before // collect_build_postponed() and they could be the dependents that have // the config clauses. In a sense, change to replaced_vers, - // postponed_deps, or postponed_poss maps should not affect the deps + // postponed_deps, or unacceptable_alts maps should not affect the deps // list. But not the other way around: a dependency erased from the deps // list could have caused an entry in the replaced_vers, postponed_deps, - // and/or postponed_poss maps. And so we clean replaced_vers, - // postponed_deps, and postponed_poss on scratch_exe (scratch during the - // plan execution). + // and/or unacceptable_alts maps. And so we clean replaced_vers, + // postponed_deps, and unacceptable_alts on scratch_exe (scratch during + // the plan execution). // for (bool refine (true), scratch_exe (true), scratch_col (false); refine; ) @@ -10716,8 +5614,17 @@ namespace bpkg // Also, if a dependency package already has selected package that // is held, then we need to unhold it. // - auto enter = [&mdb, &pkgs] (database& db, const dependency_package& p) + auto enter = [&pkgs, &cmd_line] (database& db, + const dependency_package& p) { + // Note that we don't set the upgrade and deorphan flags based on + // the --upgrade, --patch, and --deorphan options since an option + // presense doesn't necessarily means that the respective flag needs + // to be set (the package may not be selected, may not be patchable + // and/or an orphan, etc). The proper flags will be provided by + // evaluate_dependency() if/when any upgrade/deorphan recommendation + // is given. + // build_package bp { nullopt, // Action. db, @@ -10730,7 +5637,7 @@ namespace bpkg nullopt, // Postponed dependency alternatives. false, // Recursive collection. false, // Hold package. - p.constraint.has_value (), // Hold version. + p.hold_version, {}, // Constraints. p.system, p.keep_out, @@ -10739,13 +5646,16 @@ namespace bpkg p.checkout_root, p.checkout_purge, p.config_vars, - {package_key {mdb, ""}}, // Required by (command line). + nullopt, // Upgrade. + false, // Deorphan. + {cmd_line}, // Required by (command line). false, // Required by dependents. 0}; // State flags. if (p.constraint) - bp.constraints.emplace_back ( - mdb, "command line", *p.constraint); + bp.constraints.emplace_back (*p.constraint, + cmd_line.db, + cmd_line.name.string ()); pkgs.enter (p.name, move (bp)); }; @@ -10759,17 +5669,18 @@ namespace bpkg // The system package may only have an exact/wildcard version // specified. // - add_system_package (db, + add_system_package (&db, p.name, - (p.constraint - ? *p.constraint->min_version - : wildcard_version)); - + p.constraint, + p.system_status, + nullptr /* stubs */); enter (db, p); }; // Private configurations that were created during collection of the - // package builds. + // package builds. The list contains the private configuration paths, + // relative to the containing configuration directories (.bpkg/host/, + // etc), together with the containing configuration databases. // // Note that the private configurations are linked to their parent // configurations right after being created, so that the subsequent @@ -10784,14 +5695,14 @@ namespace bpkg // back, re-link these configurations and persist the changes using // the new transaction. // - private_configs priv_cfgs; + vector<pair<database&, dir_path>> priv_cfgs; // Add a newly created private configuration to the private // configurations and the dependency databases lists and pre-enter // builds of system dependencies with unspecified configuration for // this configuration. // - const function<add_priv_cfg_function> add_priv_cfg ( + const function<build_packages::add_priv_cfg_function> add_priv_cfg ( [&priv_cfgs, &dep_dbs, &dep_pkgs, &enter_system_dependency] (database& pdb, dir_path&& cfg) { @@ -10809,10 +5720,13 @@ namespace bpkg } }); - postponed_packages postponed_repo; - postponed_packages postponed_alts; - postponed_configurations postponed_cfgs; - strings postponed_cfgs_history; + postponed_packages postponed_repo; + postponed_packages postponed_alts; + postponed_packages postponed_recs; + postponed_existing_dependencies postponed_edeps; + postponed_configurations postponed_cfgs; + strings postponed_cfgs_history; + unsatisfied_dependents unsatisfied_depts; try { @@ -10824,12 +5738,14 @@ namespace bpkg { replaced_vers.clear (); postponed_deps.clear (); - postponed_poss.clear (); + unacceptable_alts.clear (); scratch_exe = false; } - else if (scratch_col) + else { + assert (scratch_col); // See the scratch definition above. + // Reset to detect bogus entries. // for (auto& rv: replaced_vers) @@ -10841,12 +5757,6 @@ namespace bpkg pd.second.with_config = false; } - for (auto& pd: postponed_poss) - { - pd.second.skipped = false; - pd.second.reevaluated = false; - } - scratch_col = false; } @@ -10879,14 +5789,8 @@ namespace bpkg // specify packages on the command line does not matter). // for (const build_package& p: hold_pkgs) - pkgs.collect_build (o, - p, - find_prereq_database, - rpt_depts, - add_priv_cfg, - true /* initial_collection */, - replaced_vers, - postponed_cfgs); + pkgs.collect_build ( + o, p, replaced_vers, postponed_cfgs, unsatisfied_depts); // Collect all the prerequisites of the user selection. // @@ -10899,25 +5803,7 @@ namespace bpkg auto i (postponed_deps.find (pk)); - if (i == postponed_deps.end ()) - { - pkgs.collect_build_prerequisites ( - o, - p.db, - p.name (), - find_prereq_database, - rpt_depts, - add_priv_cfg, - true /* initial_collection */, - replaced_vers, - postponed_repo, - postponed_alts, - 0 /* max_alt_index */, - postponed_deps, - postponed_cfgs, - postponed_poss); - } - else + if (i != postponed_deps.end ()) { // Even though the user selection may have a configuration, we // treat it as a dependent without any configuration because @@ -10928,6 +5814,37 @@ namespace bpkg l5 ([&]{trace << "dep-postpone user-specified " << pk;}); } + else + { + const postponed_configuration* pcfg ( + postponed_cfgs.find_dependency (pk)); + + if (pcfg != nullptr) + { + l5 ([&]{trace << "dep-postpone user-specified " << pk + << " since already in cluster " << *pcfg;}); + } + else + { + pkgs.collect_build_prerequisites ( + o, + p.db, + p.name (), + find_prereq_database, + add_priv_cfg, + rpt_depts, + replaced_vers, + postponed_repo, + postponed_alts, + 0 /* max_alt_index */, + postponed_recs, + postponed_edeps, + postponed_deps, + postponed_cfgs, + unacceptable_alts, + unsatisfied_depts); + } + } } // Note that we need to collect unheld after prerequisites, not to @@ -10966,9 +5883,12 @@ namespace bpkg replaced_vers, postponed_repo, postponed_alts, + postponed_recs, + postponed_edeps, postponed_deps, postponed_cfgs, - postponed_poss, + unacceptable_alts, + unsatisfied_depts, find_prereq_database, add_priv_cfg); } @@ -11005,7 +5925,7 @@ namespace bpkg // Marking upgraded dependencies as "required by command line" // may seem redundant as they should already be pre-entered as // such (see above). But remember dependencies upgraded with - // -i|-r? Note that the required_by data member should never be + // -i|-r? Note that the required_by data member should never be // empty, as it is used in prompts/diagnostics. // build_package p { @@ -11029,65 +5949,104 @@ namespace bpkg nullopt, // Checkout root. false, // Checkout purge. strings (), // Configuration variables. - {package_key {mdb, ""}}, // Required by (command line). + d.upgrade, + d.deorphan, + {cmd_line}, // Required by (command line). false, // Required by dependents. - 0}; // State flags. + (d.existing || d.deorphan + ? build_package::build_replace + : uint16_t (0))}; - build_package_refs dep_chain; + package_key pk {ddb, d.name}; - // Note: recursive. + // Similar to the user-selected packages, collect non- + // recursively the dependencies for which recursive collection + // is postponed (see above for details). // - pkgs.collect_build (o, - move (p), - find_prereq_database, - rpt_depts, - add_priv_cfg, - true /* initial_collection */, - replaced_vers, - postponed_cfgs, - &dep_chain, - &postponed_repo, - &postponed_alts, - &postponed_deps, - &postponed_poss); - } - } + auto i (postponed_deps.find (pk)); + if (i != postponed_deps.end ()) + { + i->second.wout_config = true; - // Erase the bogus postponements and re-collect from scratch, if any - // (see postponed_dependencies for details). - // - // Note that we used to re-collect such postponements in-place but - // re-doing from scratch feels more correct (i.e., we may end up - // doing it earlier which will affect dependency alternatives). - // - postponed_deps.cancel_bogus (trace, true /* initial_collection */); + // Note: not recursive. + // + pkgs.collect_build ( + o, move (p), replaced_vers, postponed_cfgs, unsatisfied_depts); - // Now remove all the dependencies postponed during the initial - // collection since all this information is already in - // postponed_cfgs. - // - for (auto i (postponed_deps.begin ()); i != postponed_deps.end (); ) - { - if (i->second.initial_collection) - i = postponed_deps.erase (i); - else - ++i; + l5 ([&]{trace << "dep-postpone user-specified dependency " + << pk;}); + } + else + { + const postponed_configuration* pcfg ( + postponed_cfgs.find_dependency (pk)); + + if (pcfg != nullptr) + { + // Note: not recursive. + // + pkgs.collect_build (o, + move (p), + replaced_vers, + postponed_cfgs, + unsatisfied_depts); + + l5 ([&]{trace << "dep-postpone user-specified dependency " + << pk << " since already in cluster " + << *pcfg;}); + } + else + { + build_package_refs dep_chain; + + // Note: recursive. + // + pkgs.collect_build (o, + move (p), + replaced_vers, + postponed_cfgs, + unsatisfied_depts, + &dep_chain, + find_prereq_database, + add_priv_cfg, + &rpt_depts, + &postponed_repo, + &postponed_alts, + &postponed_recs, + &postponed_edeps, + &postponed_deps, + &unacceptable_alts); + } + } + } } // Handle the (combined) postponed collection. // - if (!postponed_repo.empty () || - !postponed_alts.empty () || - postponed_deps.has_bogus () || + if (find_if (postponed_recs.begin (), postponed_recs.end (), + [] (const build_package* p) + { + // Note that we check for the dependencies presence + // rather than for the recursive_collection flag + // (see collect_build_postponed() for details). + // + return !p->dependencies; + }) != postponed_recs.end () || + !postponed_repo.empty () || + !postponed_alts.empty () || + postponed_deps.has_bogus () || !postponed_cfgs.empty ()) pkgs.collect_build_postponed (o, replaced_vers, postponed_repo, postponed_alts, + postponed_recs, + postponed_edeps, postponed_deps, postponed_cfgs, postponed_cfgs_history, - postponed_poss, + unacceptable_alts, + unsatisfied_depts, find_prereq_database, rpt_depts, add_priv_cfg); @@ -11096,12 +6055,6 @@ namespace bpkg // (see replaced_versions for details). // replaced_vers.cancel_bogus (trace, true /* scratch */); - - // Erase the bogus existing dependent re-evaluation postponements - // and re-collect from scratch, if any (see postponed_positions for - // details). - // - postponed_poss.cancel_bogus (trace); } catch (const scratch_collection& e) { @@ -11111,7 +6064,7 @@ namespace bpkg l5 ([&]{trace << "collection failed due to " << e.description << (e.package != nullptr - ? " (" + e.package->string () + ")" + ? " (" + e.package->string () + ')' : empty_string) << ", retry from scratch";}); @@ -11132,6 +6085,9 @@ namespace bpkg continue; } + set<package_key> depts ( + pkgs.collect_dependents (rpt_depts, unsatisfied_depts)); + // Now that we have collected all the package versions that we need to // build, arrange them in the "dependency order", that is, with every // package on the list only possibly depending on the ones after @@ -11142,29 +6098,33 @@ namespace bpkg // dependencies between the specified packages). // // The order of dependency upgrades/downgrades/drops is not really - // deterministic. We, however, do them before hold_pkgs so that they - // appear (e.g., on the plan) last. + // deterministic. We, however, do upgrades/downgrades before hold_pkgs + // so that they appear (e.g., on the plan) after the packages being + // built to hold. We handle drops last, though, so that the unused + // packages are likely get purged before the package fetches, so that + // the disk space they occupy can be reused. // for (const dep& d: deps) - pkgs.order (d.db, - d.name, - nullopt /* buildtime */, - find_prereq_database, - false /* reorder */); + { + if (d.available != nullptr) + pkgs.order (d.db, + d.name, + find_prereq_database, + false /* reorder */); + } for (const build_package& p: reverse_iterate (hold_pkgs)) - pkgs.order (p.db, - p.name (), - nullopt /* buildtime */, - find_prereq_database); + pkgs.order (p.db, p.name (), find_prereq_database); for (const auto& rd: rpt_depts) pkgs.order (rd.first.db, rd.first.name, - nullopt /* buildtime */, find_prereq_database, - false /* reorder */); + false /* reorder */); + // Order the existing dependents which have participated in + // negotiation of the configuration of their dependencies. + // for (const postponed_configuration& cfg: postponed_cfgs) { for (const auto& d: cfg.dependents) @@ -11172,23 +6132,27 @@ namespace bpkg if (d.second.existing) { const package_key& p (d.first); - - pkgs.order (p.db, - p.name, - nullopt /* buildtime */, - find_prereq_database); + pkgs.order (p.db, p.name, find_prereq_database); } } } - // Collect and order all the dependents that we will need to - // reconfigure because of the up/down-grades of packages that are now - // on the list. + // Order the existing dependents whose dependencies are being + // up/down-graded or reconfigured. + // + for (const package_key& p: depts) + pkgs.order (p.db, p.name, find_prereq_database, false /* reorder */); + + // Order the re-collected packages (deviated dependents, etc). // - pkgs.collect_order_dependents (rpt_depts); + for (build_package* p: postponed_recs) + { + assert (p->recursive_collection); - // And, finally, make sure all the packages that we need to unhold - // are on the list. + pkgs.order (p->db, p->name (), find_prereq_database); + } + + // Make sure all the packages that we need to unhold are on the list. // for (const dependency_package& p: dep_pkgs) { @@ -11202,9 +6166,8 @@ namespace bpkg if (sp != nullptr && sp->hold_package) pkgs.order (db, p.name, - nullopt /* buildtime */, find_prereq_database, - false /* reorder */); + false /* reorder */); }; if (p.db != nullptr) @@ -11218,6 +6181,43 @@ namespace bpkg } } + // And, finally, order the package drops. + // + for (const dep& d: deps) + { + if (d.available == nullptr) + pkgs.order (d.db, + d.name, + find_prereq_database, + false /* reorder */); + } + + // Make sure all the postponed dependencies of existing dependents + // have been collected and fail if that's not the case. + // + for (const auto& pd: postponed_edeps) + { + const build_package* p (pkgs.entered_build (pd.first)); + assert (p != nullptr && p->available != nullptr); + + if (!p->recursive_collection) + { + // Feels like this shouldn't happen but who knows. + // + diag_record dr (fail); + dr << "package " << p->available_name_version_db () << " is not " + << "built due to its configured dependents deviation in " + << "dependency resolution" << + info << "deviated dependents:"; + + for (const package_key& d: pd.second) + dr << ' ' << d; + + dr << info << "please report in " + << "https://github.com/build2/build2/issues/302"; + } + } + #ifndef NDEBUG pkgs.verify_ordering (); #endif @@ -11266,7 +6266,7 @@ namespace bpkg changed = execute_plan (o, bl, - true /* simulate */, + &unsatisfied_depts, find_prereq_database); if (changed) @@ -11290,20 +6290,31 @@ namespace bpkg // value covers both the "no change is required" and the "no // recommendation available" cases. // - auto eval_dep = [&dep_pkgs, &rec_pkgs, &o] ( - database& db, - const shared_ptr<selected_package>& sp, - bool ignore_unsatisfiable = true) -> optional<evaluate_result> + auto eval_dep = [&dep_pkgs, + &rec_pkgs, + &o, + &existing_deps, + &deorphaned_deps, + &pkgs, + cache = upgrade_dependencies_cache {}] ( + database& db, + const shared_ptr<selected_package>& sp, + bool ignore_unsatisfiable = true) mutable + -> optional<evaluate_result> { optional<evaluate_result> r; // See if there is an optional dependency upgrade recommendation. // if (!sp->hold_package) - r = evaluate_dependency (db, + r = evaluate_dependency (o, + db, sp, dep_pkgs, o.no_move (), + existing_deps, + deorphaned_deps, + pkgs, ignore_unsatisfiable); // If none, then see for the recursive dependency upgrade @@ -11313,7 +6324,15 @@ namespace bpkg // configured as such for a reason. // if (!r && !sp->system () && !rec_pkgs.empty ()) - r = evaluate_recursive (db, sp, rec_pkgs, ignore_unsatisfiable); + r = evaluate_recursive (o, + db, + sp, + rec_pkgs, + existing_deps, + deorphaned_deps, + pkgs, + ignore_unsatisfiable, + cache); // Translate the "no change" result to nullopt. // @@ -11348,11 +6367,12 @@ namespace bpkg bool s (false); database& db (i->db); + const package_name& nm (i->name); // Here we scratch if evaluate changed its mind or if the resulting // version doesn't match what we expect it to be. // - if (auto sp = db.find<selected_package> (i->name)) + if (auto sp = db.find<selected_package> (nm)) { const version& dv (target_version (db, i->available, i->system)); @@ -11368,12 +6388,25 @@ namespace bpkg if (s) { scratch_exe = true; // Rebuild the plan from scratch. + + package_key pk (db, nm); + + auto j (find (existing_deps.begin (), existing_deps.end (), pk)); + if (j != existing_deps.end ()) + existing_deps.erase (j); + + deorphaned_deps.erase (pk); + i = deps.erase (i); } else ++i; } + if (scratch_exe) + l5 ([&]{trace << "one of dependency evaluation decisions has " + << "changed, re-collecting from scratch";}); + // If the execute_plan() call was noop, there are no user expectations // regarding any dependency, and no upgrade is requested, then the // only possible refinement outcome can be recommendations to drop @@ -11401,8 +6434,13 @@ namespace bpkg // make sure that the unsatisfiable dependency, if left, is // reported. // - auto need_refinement = [&eval_dep, &deps, &rec_pkgs, &dep_dbs, &o] ( - bool diag = false) -> bool + auto need_refinement = [&eval_dep, + &deps, + &rec_pkgs, + &dep_dbs, + &existing_deps, + &deorphaned_deps, + &o] (bool diag = false) -> bool { // Examine the new dependency set for any up/down-grade/drops. // @@ -11433,11 +6471,25 @@ namespace bpkg continue; if (!diag) + { deps.push_back (dep {er->db, sp->name, move (er->available), move (er->repository_fragment), - er->system}); + er->system, + er->existing, + er->upgrade, + er->orphan.has_value ()}); + + if (er->existing) + existing_deps.emplace_back (er->db, sp->name); + + if (er->orphan) + { + deorphaned_deps[package_key (er->db, sp->name)] = + move (*er->orphan); + } + } r = true; } @@ -11449,8 +6501,19 @@ namespace bpkg refine = need_refinement (); + // If no further refinement is necessary, then perform the + // diagnostics run. Otherwise, if any dependency configuration + // negotiation has been performed during the current plan refinement + // iteration, then rebuild the plan from scratch (see above for + // details). Also rebuild it from from scratch if any unsatisfied + // dependents have been ignored, since their unsatisfied constraints + // are now added to the dependencies' build_package::constraints + // lists. + // if (!refine) need_refinement (true /* diag */); + else if (!postponed_cfgs.empty () || !unsatisfied_depts.empty ()) + scratch_exe = true; } // Note that we prevent building multiple instances of the same @@ -11546,10 +6609,10 @@ namespace bpkg // that the build-time dependency configuration type (host or // build2) differs from the dependent configuration type (target // is a common case) and doesn't work well, for example, for the - // self-hosted configurations. For them it can fail - // erroneously. We can potentially fix that by additionally - // storing the build-time flag for a prerequisite. However, let's - // first see if it ever becomes a problem. + // self-hosted configurations. For them it can fail erroneously. + // We can potentially fix that by additionally storing the + // build-time flag for a prerequisite. However, let's first see if + // it ever becomes a problem. // prerequisites r; const package_prerequisites& prereqs (sp->prerequisites); @@ -11833,6 +6896,190 @@ namespace bpkg t.commit (); } + + if (!refine) + { + // Cleanup the package build collecting state, preparing for the + // re-collection from the very beginning. + // + auto prepare_recollect = [&refine, + &scratch_exe, + &deps, + &existing_deps, + &deorphaned_deps] () + { + refine = true; + scratch_exe = true; + + deps.clear (); + existing_deps.clear (); + deorphaned_deps.clear (); + }; + + // Issue diagnostics and fail if any existing dependents are not + // satisfied with their dependencies. + // + // But first, try to resolve the first encountered unsatisfied + // constraint by replacing the collected unsatisfactory dependency + // or some of its dependents with some other available package + // version. This version, while not being the best possible choice, + // must be satisfactory for all its new and existing dependents. If + // succeed, punch the replacement version into the command line and + // recollect from the very beginning (see unsatisfied_dependents for + // details). + // + if (!unsatisfied_depts.empty ()) + { + if (!cmdline_refine_index) // Not command line adjustments refinement? + { + const unsatisfied_dependent& dpt (unsatisfied_depts.front ()); + + assert (!dpt.ignored_constraints.empty ()); + + const ignored_constraint& ic (dpt.ignored_constraints.front ()); + + const build_package* p (pkgs.entered_build (ic.dependency)); + assert (p != nullptr); // The dependency must be collected. + + l5 ([&]{trace << "try to replace unsatisfactory dependency " + << p->available_name_version_db () << " with some " + << "other version";}); + + optional<cmdline_adjustment> a; + vector<package_key> unsatisfied_dpts; + set<const build_package*> visited_dpts; + + if ((a = try_replace_dependency (o, + *p, + pkgs, + hold_pkgs, + dep_pkgs, + cmdline_adjs, + unsatisfied_dpts, + "unsatisfactory dependency")) || + (a = try_replace_dependent (o, + *p, + &ic.unsatisfied_constraints, + pkgs, + cmdline_adjs, + unsatisfied_dpts, + hold_pkgs, + dep_pkgs, + visited_dpts)) || + !cmdline_adjs.empty ()) + { + if (a) + { + cmdline_adjs.push (move (*a)); + } + else + { + cmdline_adjustment a (cmdline_adjs.pop ()); + + l5 ([&]{trace << "cannot replace any package, rolling back " + << "latest command line adjustment (" + << cmdline_adjs.to_string (a) << ')';}); + } + + prepare_recollect (); + } + else + unsatisfied_depts.diag (pkgs); // Issue the diagnostics and fail. + } + else // We are in the command line adjustments refinement cycle. + { + // Since we have failed to collect, then the currently dropped + // command line adjustment is essential. Thus, push it back to + // the stack, drop the next one, and retry. If this is the last + // adjustment in the stack, then we assume that no further + // refinement is possible and we just recollect, assuming that + // this recollection will be successful. + // + assert (cmdline_refine_adjustment); // Wouldn't be here otherwise. + + l5 ([&]{trace << "attempt to refine command line adjustments by " + << "rolling back adjustment " + << cmdline_adjs.to_string ( + *cmdline_refine_adjustment) + << " failed, pushing it back";}); + + cmdline_adjs.push (move (*cmdline_refine_adjustment)); + + // Index of the being previously dropped adjustment must be + // valid. + // + assert (*cmdline_refine_index != cmdline_adjs.size ()); + + if (++(*cmdline_refine_index) != cmdline_adjs.size ()) + { + cmdline_refine_adjustment = cmdline_adjs.pop (true /* front */); + + l5 ([&]{trace << "continue with command line adjustments " + << "refinement cycle by rolling back adjustment " + << cmdline_adjs.to_string ( + *cmdline_refine_adjustment);}); + } + else + { + cmdline_refine_adjustment = nullopt; + + l5 ([&]{trace << "cannot further refine command line " + << "adjustments, performing final collection";}); + } + + prepare_recollect (); + } + } + // + // If the collection was successful, then see if we still need to + // perform the command line adjustments refinement. + // + else if (cmdline_adjs.tried () && + (!cmdline_refine_index || + *cmdline_refine_index != cmdline_adjs.size ())) + { + // If some command line adjustment is currently being dropped, + // that means that this adjustment is redundant. + // + bool initial (!cmdline_refine_index); + + if (!initial) + { + assert (cmdline_refine_adjustment); + + l5 ([&]{trace << "command line adjustment " + << cmdline_adjs.to_string ( + *cmdline_refine_adjustment) + << " is redundant, dropping it";}); + + cmdline_refine_adjustment = nullopt; + cmdline_refine_index = nullopt; + } + + // We cannot remove all the adjustments during the refinement. + // Otherwise, we shouldn't be failing in the first place. + // + assert (!cmdline_adjs.empty ()); + + // If there is just a single adjustment left, then there is + // nothing to refine anymore. + // + if (cmdline_adjs.size () != 1) + { + cmdline_refine_adjustment = cmdline_adjs.pop (true /* front */); + cmdline_refine_index = 0; + + l5 ([&]{trace << (initial ? "start" : "re-start") << " command " + << "line adjustments refinement cycle by rolling " + << "back first adjustment (" + << cmdline_adjs.to_string ( + *cmdline_refine_adjustment) + << ')';}); + + prepare_recollect (); + } + } + } } } @@ -11850,10 +7097,11 @@ namespace bpkg bool update_dependents (false); // We need the plan and to ask for the user's confirmation only if some - // implicit action (such as building prerequisite or reconfiguring - // dependent package) is to be taken or there is a selected package which - // version must be changed. But if the user explicitly requested it with - // --plan, then we print it as long as it is not empty. + // implicit action (such as building prerequisite, reconfiguring dependent + // package, or installing system/distribution packages) is to be taken or + // there is a selected package which version must be changed. But if the + // user explicitly requested it with --plan, then we print it as long as + // it is not empty. // string plan; sha256 csum; @@ -11864,6 +7112,31 @@ namespace bpkg o.plan_specified () || o.rebuild_checksum_specified ()) { + // Map the main system/distribution packages that need to be installed + // to the system packages which caused their installation (see + // build_package::system_install() for details). + // + using package_names = vector<reference_wrapper<const package_name>>; + using system_map = map<string, package_names>; + + system_map sys_map; + + // Iterate in the reverse order as we will do for printing the action + // lines. This way a sys-install action line will be printed right + // before the bpkg action line of a package which appears first in the + // sys-install action's 'required by' list. + // + for (const build_package& p: reverse_iterate (pkgs)) + { + if (const system_package_status* s = p.system_install ()) + { + package_names& ps (sys_map[s->system_name]); + + if (find (ps.begin (), ps.end (), p.name ()) == ps.end ()) + ps.push_back (p.name ()); + } + } + // Start the transaction since we may query available packages for // skeleton initializations. // @@ -11871,200 +7144,262 @@ namespace bpkg bool first (true); // First entry in the plan. - for (build_package& p: reverse_iterate (pkgs)) + // Print the bpkg package action lines. + // + // Also print the sys-install action lines for system/distribution + // packages which require installation by the system package manager. + // Print them before the respective system package action lines, but + // only once per (main) system/distribution package. For example: + // + // sys-install libssl1.1/1.1.1l (required by sys:libssl, sys:libcrypto) + // configure sys:libssl/1.1.1 (required by foo) + // configure sys:libcrypto/1.1.1 (required by bar) + // + for (auto i (pkgs.rbegin ()); i != pkgs.rend (); ) { + build_package& p (*i); assert (p.action); - database& pdb (p.db); - const shared_ptr<selected_package>& sp (p.selected); - string act; - if (*p.action == build_package::drop) - { - act = "drop " + sp->string (pdb) + " (unused)"; - need_prompt = true; - } - else + const system_package_status* s; + system_map::iterator j; + + if ((s = p.system_install ()) != nullptr && + (j = sys_map.find (s->system_name)) != sys_map.end ()) { - // Print configuration variables. - // - // The idea here is to only print configuration for those packages - // for which we call pkg_configure*() in execute_plan(). - // - package_skeleton* cfg (nullptr); + act = "sys-install "; + act += s->system_name; + act += '/'; + act += s->system_version; + act += " (required by "; - string cause; - if (*p.action == build_package::adjust) + bool first (true); + for (const package_name& n: j->second) { - assert (sp != nullptr && (p.reconfigure () || p.unhold ())); + if (first) + first = false; + else + act += ", "; - // This is a dependent needing reconfiguration. - // - // This is an implicit reconfiguration which requires the plan to - // be printed. Will flag that later when composing the list of - // prerequisites. - // - if (p.reconfigure ()) - { - act = "reconfigure"; - cause = "dependent of"; + act += "sys:"; + act += n.string (); + } - if (!o.configure_only ()) - update_dependents = true; - } + act += ')'; - // This is a held package needing unhold. - // - if (p.unhold ()) - { - if (act.empty ()) - act = "unhold"; - else - act += "/unhold"; - } + need_prompt = true; - act += ' ' + sp->name.string (); + // Make sure that we print this sys-install action just once. + // + sys_map.erase (j); - const string& s (pdb.string); - if (!s.empty ()) - act += ' ' + s; + // Note that we don't increment i in order to re-iterate this pkgs + // entry. + } + else + { + ++i; - // This is an adjustment and so there is no available package - // specified for the build package object and thus the skeleton - // cannot be present. - // - assert (p.available == nullptr && !p.skeleton); + database& pdb (p.db); + const shared_ptr<selected_package>& sp (p.selected); - // We shouldn't be printing configurations for plain unholds. - // - if (p.reconfigure ()) - { - // Since there is no available package specified we need to find - // it (or create a transient one). - // - cfg = &p.init_skeleton (o, find_available (o, pdb, sp)); - } + if (*p.action == build_package::drop) + { + act = "drop " + sp->string (pdb) + " (unused)"; + need_prompt = true; } else { - assert (p.available != nullptr); // This is a package build. - - // Even if we already have this package selected, we have to - // make sure it is configured and updated. + // Print configuration variables. + // + // The idea here is to only print configuration for those packages + // for which we call pkg_configure*() in execute_plan(). // - if (sp == nullptr) + package_skeleton* cfg (nullptr); + + string cause; + if (*p.action == build_package::adjust) { - act = p.system ? "configure" : "new"; + assert (sp != nullptr && (p.reconfigure () || p.unhold ())); - // For a new non-system package the skeleton must already be - // initialized. + // This is a dependent needing reconfiguration. // - assert (p.system || p.skeleton.has_value ()); - - // Initialize the skeleton if it is not initialized yet. + // This is an implicit reconfiguration which requires the plan + // to be printed. Will flag that later when composing the list + // of prerequisites. // - cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); - } - else if (sp->version == p.available_version ()) - { - // If this package is already configured and is not part of the - // user selection (or we are only configuring), then there is - // nothing we will be explicitly doing with it (it might still - // get updated indirectly as part of the user selection update). + if (p.reconfigure ()) + { + act = "reconfigure"; + cause = "dependent of"; + + if (!o.configure_only ()) + update_dependents = true; + } + + // This is a held package needing unhold. // - if (!p.reconfigure () && - sp->state == package_state::configured && - (!p.user_selection () || - o.configure_only () || - p.configure_only ())) - continue; + if (p.unhold ()) + { + if (act.empty ()) + act = "unhold"; + else + act += "/unhold"; + } - act = p.system - ? "reconfigure" - : (p.reconfigure () - ? (o.configure_only () || p.configure_only () - ? "reconfigure" - : "reconfigure/update") - : "update"); + act += ' ' + sp->name.string (); + const string& s (pdb.string); + if (!s.empty ()) + act += ' ' + s; + + // This is an adjustment and so there is no available package + // specified for the build package object and thus the skeleton + // cannot be present. + // + assert (p.available == nullptr && !p.skeleton); + + // We shouldn't be printing configurations for plain unholds. + // if (p.reconfigure ()) { - // Initialize the skeleton if it is not initialized yet. + // Since there is no available package specified we need to + // find it (or create a transient one). // - cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + cfg = &p.init_skeleton (o, + true /* load_old_dependent_config */, + find_available (o, pdb, sp)); } } else { - act = p.system - ? "reconfigure" - : sp->version < p.available_version () - ? "upgrade" - : "downgrade"; - - // For a non-system package up/downgrade the skeleton must - // already be initialized. - // - assert (p.system || p.skeleton.has_value ()); + assert (p.available != nullptr); // This is a package build. + + bool replace (p.replace ()); - // Initialize the skeleton if it is not initialized yet. + // Even if we already have this package selected, we have to + // make sure it is configured and updated. // - cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + if (sp == nullptr) + { + act = p.system ? "configure" : "new"; - need_prompt = true; - } + // For a new non-system package the skeleton must already be + // initialized. + // + assert (p.system || p.skeleton.has_value ()); - if (p.unhold ()) - act += "/unhold"; + // Initialize the skeleton if it is not initialized yet. + // + cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + } + else if (sp->version == p.available_version ()) + { + // If this package is already configured and is not part of + // the user selection (or we are only configuring), then there + // is nothing we will be explicitly doing with it (it might + // still get updated indirectly as part of the user selection + // update). + // + if (!p.reconfigure () && + sp->state == package_state::configured && + (!p.user_selection () || + o.configure_only () || + p.configure_only ())) + continue; - act += ' ' + p.available_name_version_db (); - cause = p.required_by_dependents ? "required by" : "dependent of"; + act = p.system + ? "reconfigure" + : (p.reconfigure () + ? (o.configure_only () || p.configure_only () + ? (replace ? "replace" : "reconfigure") + : (replace ? "replace/update" : "reconfigure/update")) + : "update"); - if (p.configure_only ()) - update_dependents = true; - } + if (p.reconfigure ()) + { + // Initialize the skeleton if it is not initialized yet. + // + cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + } + } + else + { + act += p.system + ? "reconfigure" + : (sp->version < p.available_version () + ? (replace ? "replace/upgrade" : "upgrade") + : (replace ? "replace/downgrade" : "downgrade")); + + // For a non-system package up/downgrade the skeleton must + // already be initialized. + // + assert (p.system || p.skeleton.has_value ()); - // Also list dependents for the newly built user-selected - // dependencies. - // - bool us (p.user_selection ()); - string rb; - if (!us || (!p.user_selection (hold_pkgs) && sp == nullptr)) - { - // Note: if we are ever tempted to truncate this, watch out for - // the --rebuild-checksum functionality which uses this. But then - // it's not clear this information is actually important: can a - // dependent-dependency structure change without any of the - // package versions changing? Doesn't feel like it should. + // Initialize the skeleton if it is not initialized yet. + // + cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + + need_prompt = true; + } + + if (p.unhold ()) + act += "/unhold"; + + act += ' ' + p.available_name_version_db (); + cause = p.required_by_dependents ? "required by" : "dependent of"; + + if (p.configure_only ()) + update_dependents = true; + } + + // Also list dependents for the newly built user-selected + // dependencies. // - for (const package_key& pk: p.required_by) + bool us (p.user_selection ()); + string rb; + if (!us || (!p.user_selection (hold_pkgs) && sp == nullptr)) { - // Skip the command-line dependent. + // Note: if we are ever tempted to truncate this, watch out for + // the --rebuild-checksum functionality which uses this. But + // then it's not clear this information is actually important: + // can a dependent-dependency structure change without any of + // the package versions changing? Doesn't feel like it should. + // + for (const package_version_key& pvk: p.required_by) + { + // Skip the command-line, etc dependents and don't print the + // package version (which is not always available; see + // build_package::required_by for details). + // + if (pvk.version) // Is it a real package? + { + rb += (rb.empty () ? " " : ", ") + + pvk.string (true /* ignore_version */); + } + } + + // If not user-selected, then there should be another (implicit) + // reason for the action. // - if (!pk.name.empty ()) - rb += (rb.empty () ? " " : ", ") + pk.string (); + assert (!rb.empty ()); } - // If not user-selected, then there should be another (implicit) - // reason for the action. - // - assert (!rb.empty ()); - } + if (!rb.empty ()) + act += " (" + cause + rb + ')'; - if (!rb.empty ()) - act += " (" + cause + rb + ')'; + if (cfg != nullptr && !cfg->empty_print ()) + { + ostringstream os; + cfg->print_config (os, o.print_only () ? " " : " "); + act += '\n'; + act += os.str (); + } - if (cfg != nullptr && !cfg->empty_print ()) - { - ostringstream os; - cfg->print_config (os, o.print_only () ? " " : " "); - act += '\n'; - act += os.str (); + if (!us) + need_prompt = true; } - - if (!us) - need_prompt = true; } if (first) @@ -12126,13 +7461,14 @@ namespace bpkg // Ok, we have "all systems go". The overall action plan is as follows. // - // 1. disfigure up/down-graded, reconfigured [left to right] - // 2. purge up/down-graded [right to left] - // 3.a fetch/unpack new, up/down-graded - // 3.b checkout new, up/down-graded - // 4. configure all - // 5. unhold unheld - // 6. build user selection [right to left] + // 1. sys-install not installed system/distribution + // 2. disfigure up/down-graded, reconfigured [left to right] + // 3. purge up/down-graded [right to left] + // 4.a fetch/unpack new, up/down-graded, replaced + // 4.b checkout new, up/down-graded, replaced + // 5. configure all + // 6. unhold unheld + // 7. build user selection [right to left] // // Note that for some actions, e.g., purge or fetch, the order is not // really important. We will, however, do it right to left since that @@ -12148,7 +7484,7 @@ namespace bpkg // bool noop (!execute_plan (o, pkgs, - false /* simulate */, + nullptr /* simulate */, find_prereq_database)); if (o.configure_only ()) @@ -12163,6 +7499,10 @@ namespace bpkg // First add the user selection. // + // Only update user-selected packages which are specified on the command + // line as build to hold. Note that the dependency package will be updated + // implicitly via their dependents, if the latter are updated. + // for (const build_package& p: reverse_iterate (pkgs)) { assert (p.action); @@ -12174,7 +7514,7 @@ namespace bpkg const shared_ptr<selected_package>& sp (p.selected); if (!sp->system () && // System package doesn't need update. - p.user_selection ()) + p.user_selection (hold_pkgs)) upkgs.push_back (pkg_command_vars {db.config_orig, !multi_config () && db.main (), sp, @@ -12193,10 +7533,15 @@ namespace bpkg database& db (p.db); + // Note: don't update the re-evaluated and re-collected dependents + // unless they are reconfigured. + // if ((*p.action == build_package::adjust && p.reconfigure ()) || (*p.action == build_package::build && - (p.flags & (build_package::build_repoint | - build_package::build_reevaluate)) != 0)) + ((p.flags & build_package::build_repoint) != 0 || + ((p.flags & (build_package::build_reevaluate | + build_package::build_recollect)) != 0 && + p.reconfigure ())))) upkgs.push_back (pkg_command_vars {db.config_orig, !multi_config () && db.main (), p.selected, @@ -12219,13 +7564,16 @@ namespace bpkg static bool execute_plan (const pkg_build_options& o, build_package_list& build_pkgs, - bool simulate, + unsatisfied_dependents* simulate, const function<find_database_function>& fdb) { tracer trace ("execute_plan"); l4 ([&]{trace << "simulate: " << (simulate ? "yes" : "no");}); + // If unsatisfied dependents are specified then we are in the simulation + // mode and thus simulate can be used as bool. + bool r (false); uint16_t verb (!simulate ? bpkg::verb : 0); @@ -12236,6 +7584,40 @@ namespace bpkg size_t prog_i, prog_n, prog_percent; + // sys-install + // + // Install the system/distribution packages required by the respective + // system packages (see build_package::system_install() for details). + // + if (!simulate && o.sys_install ()) + { + // Collect the names of all the system packages being managed by the + // system package manager (as opposed to user/fallback), suppressing + // duplicates. + // + vector<package_name> ps; + + for (build_package& p: build_pkgs) + { + if (p.system_status () && + find (ps.begin (), ps.end (), p.name ()) == ps.end ()) + { + ps.push_back (p.name ()); + } + } + + // Install the system/distribution packages. + // + if (!ps.empty ()) + { + // Otherwise, we wouldn't get any package statuses. + // + assert (sys_pkg_mgr && *sys_pkg_mgr != nullptr); + + (*sys_pkg_mgr)->install (ps); + } + } + // disfigure // // Note: similar code in pkg-drop. @@ -12262,10 +7644,9 @@ namespace bpkg // On the package reconfiguration we will try to resolve dependencies to // the same prerequisites (see pkg_configure() for details). For that, we - // will save prerequisites before disfiguring the dependents. Note, - // though, that this is not required for dependents with the collected - // prerequisites builds since the dependency alternatives are already - // selected for them. + // will save prerequisites before disfiguring a package. Note, though, + // that this is not required for the recursively collected packages since + // the dependency alternatives are already selected for them. // map<const build_package*, vector<package_name>> previous_prerequisites; @@ -12279,6 +7660,8 @@ namespace bpkg database& pdb (p.db); shared_ptr<selected_package>& sp (p.selected); + assert (sp != nullptr); // Shouldn't be here otherwise. + // Each package is disfigured in its own transaction, so that we // always leave the configuration in a valid state. // @@ -12290,7 +7673,7 @@ namespace bpkg bool external (false); if (!simulate) { - external = sp != nullptr && sp->external () && p.external (); + external = (sp->external () && p.external ()); // Reset the keep_out flag if the package being unpacked is not // external. @@ -12299,31 +7682,62 @@ namespace bpkg p.keep_out = false; } - if (*p.action != build_package::drop && - !p.dependencies && - !sp->prerequisites.empty ()) + // Save prerequisites before disfiguring the package. + // + // Note that we add the prerequisites list to the map regardless if + // there are any prerequisites or not to, in particular, indicate the + // package reconfiguration mode to the subsequent + // pkg_configure_prerequisites() call (see the function documentation + // for details). + // + if (*p.action != build_package::drop && !p.dependencies && !p.system) { vector<package_name>& ps (previous_prerequisites[&p]); - ps.reserve (sp->prerequisites.size ()); + if (!sp->prerequisites.empty ()) + { + ps.reserve (sp->prerequisites.size ()); - for (const auto& pp: sp->prerequisites) - ps.push_back (pp.first.object_id ()); + for (const auto& pp: sp->prerequisites) + ps.push_back (pp.first.object_id ()); + } } // For an external package being replaced with another external, keep // the configuration unless requested not to with --disfigure. // - // Note that for other cases the preservation of the configuration is - // still a @@ TODO (the idea is to use our config.config.{save,load} - // machinery). Also see "parallel" logic in package_skeleton. + bool disfigure (p.disfigure || !external); + + // If the skeleton was not initialized yet (this is an existing package + // reconfiguration and no configuration was printed as a part of the + // plan, etc), then initialize it now. Whether the skeleton is newly + // initialized or not, make sure that the current configuration is + // loaded, unless the package project is not being disfigured. // + if (*p.action != build_package::drop && !p.system) + { + if (!p.skeleton) + { + // If there is no available package specified for the build package + // object, then we need to find it (or create a transient one). + // + p.init_skeleton (o, + true /* load_old_dependent_config */, + (p.available == nullptr + ? find_available (o, pdb, sp) + : nullptr)); + } + + if (disfigure) + p.skeleton->load_old_config (); + } + // Commits the transaction. // pkg_disfigure (o, pdb, t, sp, !p.keep_out /* clean */, - p.disfigure || !external /* disfigure */, + disfigure, simulate); r = true; @@ -12458,9 +7872,11 @@ namespace bpkg } // Fetch or checkout if this is a new package or if we are - // up/down-grading. + // up/down-grading or replacing. // - if (sp == nullptr || sp->version != p.available_version ()) + if (sp == nullptr || + sp->version != p.available_version () || + p.replace ()) { sp = nullptr; // For the directory case below. @@ -12482,19 +7898,22 @@ namespace bpkg for (const package_location& l: ap->locations) { - const repository_location& rl ( - l.repository_fragment.load ()->location); - - if (!basis || rl.local ()) // First or local? + if (!rep_masked_fragment (l.repository_fragment)) { - basis = rl.basis (); + const repository_location& rl ( + l.repository_fragment.load ()->location); - if (rl.directory_based ()) - break; + if (!basis || rl.local ()) // First or local? + { + basis = rl.basis (); + + if (rl.directory_based ()) + break; + } } } - assert (basis); + assert (basis); // Shouldn't be here otherwise. // All calls commit the transaction. // @@ -12621,7 +8040,7 @@ namespace bpkg // Commits the transaction. // - sp = pkg_unpack (o, pdb, af.database (), t, ap->id.name, simulate); + sp = pkg_unpack (o, pdb, t, ap->id.name, simulate); if (result) text << "unpacked " << *sp << pdb; @@ -12672,14 +8091,51 @@ namespace bpkg return true; }; - if (progress) + // On the first pass collect all the build_package's to be configured and + // calculate their configure_prerequisites_result's. + // + struct configure_package { - prog_i = 0; - prog_n = static_cast<size_t> (count_if (build_pkgs.begin (), - build_pkgs.end (), - configure_pred)); - prog_percent = 100; - } + reference_wrapper<build_package> pkg; + + // These are unused for system packages. + // + configure_prerequisites_result res; + build2::variable_overrides ovrs; + }; + vector<configure_package> configure_packages; + configure_packages.reserve (build_pkgs.size ()); + + // While at it also collect global configuration variable overrides from + // each configure_prerequisites_result::config_variables and merge them + // into configure_global_vars. + // + // @@ TODO: Note that the current global override semantics is quite + // broken in that we don't force reconfiguration of all the packages. + // +#ifndef BPKG_OUTPROC_CONFIGURE + strings configure_global_vars; +#endif + + // Return the "would be" state of packages that would be configured + // by this stage. + // + function<find_package_state_function> configured_state ( + [&configure_packages] (const shared_ptr<selected_package>& sp) + -> optional<pair<package_state, package_substate>> + { + for (const configure_package& cp: configure_packages) + { + const build_package& p (cp.pkg); + + if (p.selected == sp) + return make_pair ( + package_state::configured, + p.system ? package_substate::system : package_substate::none); + } + + return nullopt; + }); for (build_package& p: reverse_iterate (build_pkgs)) { @@ -12691,7 +8147,7 @@ namespace bpkg shared_ptr<selected_package>& sp (p.selected); const shared_ptr<available_package>& ap (p.available); - // Configure the package. + // Collect the package. // // At this stage the package is either selected, in which case it's a // source code one, or just available, in which case it is a system @@ -12703,7 +8159,6 @@ namespace bpkg assert (sp != nullptr || p.system); database& pdb (p.db); - transaction t (pdb, !simulate /* start */); // Show how we got here if things go wrong, for example selecting a @@ -12717,137 +8172,319 @@ namespace bpkg info << "while configuring " << p.name () << p.db; })); - auto prereqs = [&p, &previous_prerequisites] () - { - auto i (previous_prerequisites.find (&p)); - return i != previous_prerequisites.end () ? &i->second : nullptr; - }; - - // Note that pkg_configure() commits the transaction. - // + configure_prerequisites_result cpr; if (p.system) { + // We have no choice but to configure system packages on the first + // pass since otherwise there will be no selected package for + // pkg_configure_prerequisites() to find. Luckily they have no + // dependencies and so can be configured in any order. We will print + // their progress/result on the second pass in the proper order. + // + // Note: commits the transaction. + // sp = pkg_configure_system (ap->id.name, p.available_version (), pdb, t); } - else if (ap != nullptr) + else { - assert (*p.action == build_package::build); - - // If the package prerequisites builds are collected, then use the - // resulting package skeleton and the pre-selected dependency - // alternatives. + // Should only be called for packages whose prerequisites are saved. // - // Note that we may not collect the package prerequisites builds if - // the package is already configured but we still need to reconfigure - // it due, for example, to an upgrade of its dependency. In this case - // we pass to pkg_configure() the newly created package skeleton which - // contains the package configuration variables specified on the - // command line but (naturally) no reflection configuration variables. - // Note, however, that in this case pkg_configure() call will evaluate - // the reflect clauses itself and so the proper reflection variables - // will still end up in the package configuration. + auto prereqs = [&p, &previous_prerequisites] () + { + auto i (previous_prerequisites.find (&p)); + assert (i != previous_prerequisites.end ()); + return &i->second; + }; + + // In the simulation mode unconstrain all the unsatisfactory + // dependencies, if any, while configuring the dependent (see + // build_packages::collect_dependents() for details). // - // @@ Note that if we ever allow the user to override the alternative - // selection, this will break (and also if the user re-configures - // the package manually). Maybe that a good reason not to allow - // this? Or we could store this information in the database. + // Note: must be called at most once. // - if (p.dependencies) + auto unconstrain_deps = [simulate, + &p, + &trace, + deps = vector<package_key> ()] () mutable { - assert (p.skeleton); + if (simulate) + { + unsatisfied_dependent* ud ( + simulate->find_dependent (package_key (p.db, p.name ()))); - pkg_configure (o, - pdb, - t, - sp, - *p.dependencies, - &*p.alternatives, - move (*p.skeleton), - nullptr /* previous_prerequisites */, - p.disfigure, - simulate, - fdb); + if (ud != nullptr) + { + assert (deps.empty ()); + + deps.reserve (ud->ignored_constraints.size ()); + + for (const auto& c: ud->ignored_constraints) + { + l5 ([&]{trace << "while configuring dependent " << p.name () + << p.db << " in simulation mode unconstrain (" + << c.dependency << ' ' << c.constraint << ')';}); + + deps.emplace_back (c.dependency); + } + } + } + + return !deps.empty () ? &deps : nullptr; + }; + + if (ap != nullptr) + { + assert (*p.action == build_package::build); + + // If the package prerequisites builds are collected, then use the + // resulting package skeleton and the pre-selected dependency + // alternatives. + // + // Note that we may not collect the package prerequisites builds if + // the package is already configured but we still need to + // reconfigure it due, for example, to an upgrade of its dependency. + // In this case we pass to pkg_configure() the newly created package + // skeleton which contains the package configuration variables + // specified on the command line but (naturally) no reflection + // configuration variables. Note, however, that in this case + // pkg_configure() call will evaluate the reflect clauses itself and + // so the proper reflection variables will still end up in the + // package configuration. + // + // @@ Note that if we ever allow the user to override the + // alternative selection, this will break (and also if the user + // re-configures the package manually). Maybe that a good reason + // not to allow this? Or we could store this information in the + // database. + // + if (p.dependencies) + { + assert (p.skeleton); + + cpr = pkg_configure_prerequisites (o, + pdb, + t, + *p.dependencies, + &*p.alternatives, + move (*p.skeleton), + nullptr /* prev_prerequisites */, + simulate, + fdb, + configured_state, + unconstrain_deps ()); + } + else + { + assert (p.skeleton); // Must be initialized before disfiguring. + + cpr = pkg_configure_prerequisites (o, + pdb, + t, + ap->dependencies, + nullptr /* alternatives */, + move (*p.skeleton), + prereqs (), + simulate, + fdb, + configured_state, + unconstrain_deps ()); + } } - else + else // Existing dependent. { - assert (sp != nullptr); // See above. + // This is an adjustment of a dependent which cannot be system + // (otherwise it wouldn't be a dependent) and cannot become system + // (otherwise it would be a build). + // + assert (*p.action == build_package::adjust && !sp->system ()); - // Note that the skeleton can be present if, for example, this is a - // dependency which configuration has been negotiated but it is not - // collected recursively since it has no buildfile clauses. + // Must be in the unpacked state since it was disfigured on the + // first pass (see above). // - if (!p.skeleton) - p.init_skeleton (o); + assert (sp->state == package_state::unpacked); - pkg_configure (o, - pdb, - t, - sp, - ap->dependencies, - nullptr /* alternatives */, - move (*p.skeleton), - prereqs (), - p.disfigure, - simulate, - fdb); + // The skeleton must be initialized before disfiguring and the + // package can't be system. + // + assert (p.skeleton && p.skeleton->available != nullptr); + + const dependencies& deps (p.skeleton->available->dependencies); + + // @@ Note that on reconfiguration the dependent looses the + // potential configuration variables specified by the user on + // some previous build, which can be quite surprising. Should we + // store this information in the database? + // + // Note: this now works for external packages via package + // skeleton (which extracts user configuration). + // + cpr = pkg_configure_prerequisites (o, + pdb, + t, + deps, + nullptr /* alternatives */, + move (*p.skeleton), + prereqs (), + simulate, + fdb, + configured_state, + unconstrain_deps ()); + } + + t.commit (); + + if (verb >= 5 && !simulate && !cpr.config_variables.empty ()) + { + diag_record dr (trace); + + dr << sp->name << pdb << " configuration variables:"; + + for (const string& cv: cpr.config_variables) + dr << "\n " << cv; + } + + if (!simulate) + { +#ifndef BPKG_OUTPROC_CONFIGURE + auto& gvs (configure_global_vars); + + // Note that we keep global overrides in cpr.config_variables for + // diagnostics and skip them in var_override_function below. + // + for (const string& v: cpr.config_variables) + { + // Each package should have exactly the same set of global + // overrides by construction since we don't allow package- + // specific global overrides. + // + if (v[0] == '!') + { + if (find (gvs.begin (), gvs.end (), v) == gvs.end ()) + gvs.push_back (v); + } + } +#endif + // Add config.config.disfigure unless already disfigured (see the + // high-level pkg_configure() version for background). + // + if (ap == nullptr || !p.disfigure) + { + cpr.config_variables.push_back ( + "config.config.disfigure='config." + sp->name.variable () + "**'"); + } } } - else // Dependent. - { - // This is an adjustment of a dependent which cannot be system - // (otherwise it wouldn't be a dependent) and cannot become system - // (otherwise it would be a build). - // - assert (*p.action == build_package::adjust && - !p.system && - !sp->system ()); - // Must be in the unpacked state since it was disfigured on the first - // pass (see above). - // - assert (sp->state == package_state::unpacked); + configure_packages.push_back (configure_package {p, move (cpr), {}}); + } - // Initialize the skeleton if it is not initialized yet. - // - // Note that the skeleton can only be present here if it was - // initialized during the preparation of the plan and so this plan - // execution is not simulated (see above for details). - // - // Also note that there is no available package specified for the - // build package object here and so we need to find it (or create a - // transient one). - // - assert (p.available == nullptr && (!p.skeleton || !simulate)); + // Reuse the build state to avoid reloading the dependencies over and over + // again. This is a valid optimization since we are configuring in the + // dependency-dependent order. + // + unique_ptr<build2::context> configure_ctx; - if (!p.skeleton) - p.init_skeleton (o, find_available (o, pdb, sp)); +#ifndef BPKG_OUTPROC_CONFIGURE + if (!simulate) + { + using build2::context; + using build2::variable_override; + + function<context::var_override_function> vof ( + [&configure_packages] (context& ctx, size_t& i) + { + for (configure_package& cp: configure_packages) + { + for (const string& v: cp.res.config_variables) + { + if (v[0] == '!') // Skip global overrides (see above). + continue; + + pair<char, variable_override> p ( + ctx.parse_variable_override (v, i++, false /* buildspec */)); + + variable_override& vo (p.second); + + // @@ TODO: put absolute scope overrides into global_vars. + // + assert (!(p.first == '!' || (vo.dir && vo.dir->absolute ()))); + + cp.ovrs.push_back (move (vo)); + } + } + }); + + configure_ctx = pkg_configure_context ( + o, move (configure_global_vars), vof); + + // Only global in configure_global_vars. + // + assert (configure_ctx->var_overrides.empty ()); + } +#endif + + if (progress) + { + prog_i = 0; + prog_n = configure_packages.size (); + prog_percent = 100; + } + + for (configure_package& cp: configure_packages) + { + build_package& p (cp.pkg); + + const shared_ptr<selected_package>& sp (p.selected); + + // Configure the package (system already configured). + // + // NOTE: remember to update the preparation of the plan to be presented + // to the user if changing anything here. + // + database& pdb (p.db); - assert (p.skeleton->available != nullptr); // Can't be system. + if (!p.system) + { + const shared_ptr<available_package>& ap (p.available); - const dependencies& deps (p.skeleton->available->dependencies); + transaction t (pdb, !simulate /* start */); - // @@ Note that on reconfiguration the dependent looses the potential - // configuration variables specified by the user on some previous - // build, which can be quite surprising. Should we store this - // information in the database? + // Show how we got here if things go wrong. // - // I believe this now works for external packages via package - // skeleton (which extracts user configuration). + auto g ( + make_exception_guard ( + [&p] () + { + info << "while configuring " << p.name () << p.db; + })); + + // Note that pkg_configure() commits the transaction. // - pkg_configure (o, - pdb, - t, - sp, - deps, - nullptr /* alternatives */, - move (*p.skeleton), - prereqs (), - false /* disfigured */, - simulate, - fdb); + if (ap != nullptr) + { + pkg_configure (o, + pdb, + t, + sp, + move (cp.res), + configure_ctx, + cp.ovrs, + simulate); + } + else // Dependent. + { + pkg_configure (o, + pdb, + t, + sp, + move (cp.res), + configure_ctx, + cp.ovrs, + simulate); + } } r = true; @@ -12872,6 +8509,10 @@ namespace bpkg } } +#ifndef BPKG_OUTPROC_CONFIGURE + configure_ctx.reset (); // Free. +#endif + // Clear the progress if shown. // if (progress) |