diff options
Diffstat (limited to 'bpkg/rep-fetch.cxx')
| -rw-r--r-- | bpkg/rep-fetch.cxx | 476 |
1 files changed, 355 insertions, 121 deletions
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 038d54e..fe25b86 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -17,6 +17,8 @@ #include <bpkg/rep-remove.hxx> #include <bpkg/pkg-verify.hxx> #include <bpkg/diagnostics.hxx> +#include <bpkg/satisfaction.hxx> +#include <bpkg/package-query.hxx> #include <bpkg/manifest-utility.hxx> using namespace std; @@ -53,7 +55,8 @@ namespace bpkg database* db, const repository_location& rl, const optional<string>& dependent_trust, - bool ignore_unknown) + bool ignore_unknown, + bool ignore_toolchain) { // First fetch the repositories list and authenticate the base's // certificate. @@ -115,6 +118,18 @@ namespace bpkg authenticate_repository (co, conf, cert_pem, *cert, sm, rl); } + // If requested, verify that the packages are compatible with the current + // toolchain. + // + if (!ignore_toolchain) + { + for (const package_manifest& m: fr.packages) + { + for (const dependency_alternatives& das: m.dependencies) + toolchain_buildtime_dependency (co, das, &m.name); + } + } + return rep_fetch_data {{move (fr)}, move (cert_pem), move (cert)}; } @@ -192,6 +207,23 @@ namespace bpkg return r; } + static void + print_package_info (diag_record& dr, + const dir_path& pl, + const repository_location& rl, + const optional<string>& fragment) + { + dr << "package "; + + if (!pl.current ()) + dr << "'" << pl.string () << "' "; // Strip trailing '/'. + + dr << "in repository " << rl; + + if (fragment) + dr << ' ' << *fragment; + } + // Parse package manifests referenced by the package directory manifests. // static pair<vector<package_manifest>, vector<package_info>> @@ -199,27 +231,24 @@ namespace bpkg const dir_path& repo_dir, vector<package_manifest>&& pms, bool iu, - bool lb, + bool it, const repository_location& rl, const optional<string>& fragment) // For diagnostics. { - auto add_package_info = [&rl, &fragment] (const package_manifest& pm, - diag_record& dr) + auto prn_package_info = [&rl, &fragment] (diag_record& dr, + const package_manifest& pm) { - dr << "package "; - - if (!pm.location->current ()) - dr << "'" << pm.location->string () << "' "; // Strip trailing '/'. - - dr << "in repository " << rl; - - if (fragment) - dr << ' ' << *fragment; + print_package_info (dr, + path_cast<dir_path> (*pm.location), + rl, + fragment); }; // Verify that all the package directories contain the package manifest - // files and retrieve the package versions via the single `b info` call. - // While at it cache the manifest paths for the future use. + // files and retrieve the package versions via the single `b info` call, + // but only if the current build2 version is satisfactory for all the + // repository packages. While at it cache the manifest paths for the + // future use. // // Note that if the package is not compatible with the toolchain, not to // end up with an unfriendly build2 error message (referring a line in the @@ -229,16 +258,21 @@ namespace bpkg // since we need the package versions for that. Thus, we cache the // respective name value lists instead. // - package_version_infos pvs; + optional<package_version_infos> pvs; paths mfs; - dir_paths pds; vector<vector<manifest_name_value>> nvs; { mfs.reserve (pms.size ()); nvs.reserve (pms.size ()); + dir_paths pds; pds.reserve (pms.size ()); + // If true, then build2 version is satisfactory for all the repository + // packages. + // + bool bs (true); + for (const package_manifest& pm: pms) { assert (pm.location); @@ -251,19 +285,19 @@ namespace bpkg { diag_record dr (fail); dr << "no manifest file for "; - add_package_info (pm, dr); + prn_package_info (dr, pm); } // Provide the context if the package compatibility verification fails. // auto g ( make_exception_guard ( - [&pm, &add_package_info] () + [&pm, &prn_package_info] () { diag_record dr (info); dr << "while retrieving information for "; - add_package_info (pm, dr); + prn_package_info (dr, pm); })); try @@ -275,7 +309,17 @@ namespace bpkg // (e.g., .bpkg/tmp/6f746365314d/) and it's probably better to omit // it entirely (the above exception guard will print all we've got). // - nvs.push_back (pkg_verify (co, mp, dir_path ())); + pkg_verify_result r (pkg_verify (co, mp, it, dir_path ())); + + if (bs && + r.build2_dependency && + !satisfy_build2 (co, *r.build2_dependency)) + { + bs = false; + pds.clear (); // Won't now be used. + } + + nvs.push_back (move (r)); } catch (const manifest_parsing& e) { @@ -287,17 +331,29 @@ namespace bpkg } mfs.push_back (move (f)); - pds.push_back (move (d)); + + if (bs) + pds.push_back (move (d)); } - pvs = package_versions (co, pds); + // Note that for the directory-based repositories we also query + // subprojects since the package information will be used for the + // subsequent package_iteration() call (see below). + // + if (bs) + pvs = package_versions (co, pds, + (rl.directory_based () + ? b_info_flags::subprojects + : b_info_flags::none)); } // Parse package manifests, fixing up their versions. // pair<vector<package_manifest>, vector<package_info>> r; - r.first.reserve (pms.size ()); - r.second.reserve (pms.size ()); + r.first.reserve (pms.size ()); + + if (pvs) + r.second.reserve (pms.size ()); for (size_t i (0); i != pms.size (); ++i) { @@ -307,15 +363,18 @@ namespace bpkg try { - optional<version>& pv (pvs[i].version); - package_manifest m ( mfs[i].string (), move (nvs[i]), - [&pv] (version& v) + [&pvs, i] (version& v) { - if (pv) - v = move (*pv); + if (pvs) + { + optional<version>& pv ((*pvs)[i].version); + + if (pv) + v = move (*pv); + } }, iu); @@ -323,34 +382,19 @@ namespace bpkg // m.location = move (*pm.location); - // Load the bootstrap, root, and config/*.build buildfiles into the - // respective *-build values, if requested and if they are not already - // specified in the manifest. - // - if (lb) - try - { - load_package_buildfiles (m, pds[i], true /* err_path_relative */); - } - catch (const runtime_error& e) - { - diag_record dr (fail); - dr << e << info; - add_package_info (m, dr); - dr << endf; - } - pm = move (m); } catch (const manifest_parsing& e) { diag_record dr (fail (e.name, e.line, e.column)); dr << e.description << info; - add_package_info (pm, dr); + prn_package_info (dr, pm); } r.first.push_back (move (pm)); - r.second.push_back (move (pvs[i].info)); + + if (pvs) + r.second.push_back (move ((*pvs)[i].info)); } return r; @@ -374,11 +418,11 @@ namespace bpkg ifdstream is (fp); string s (is.read_text ()); - if (s.empty ()) + if (s.empty () && name != "build-file") fail << name << " manifest value in " << pkg / manifest_file << " references empty file " << rp << info << "repository " << rl - << (!fragment.empty () ? " " + fragment : ""); + << (!fragment.empty () ? ' ' + fragment : ""); return s; } @@ -388,7 +432,7 @@ namespace bpkg << name << " manifest value in " << pkg / manifest_file << ": " << e << info << "repository " << rl - << (!fragment.empty () ? " " + fragment : "") << endf; + << (!fragment.empty () ? ' ' + fragment : "") << endf; } } @@ -396,6 +440,7 @@ namespace bpkg rep_fetch_dir (const common_options& co, const repository_location& rl, bool iu, + bool it, bool ev, bool lb) { @@ -423,29 +468,71 @@ namespace bpkg rd, move (pms), iu, - lb, + it, rl, empty_string /* fragment */)); fr.packages = move (pmi.first); fr.package_infos = move (pmi.second); - // Expand file-referencing package manifest values. + // If requested, expand file-referencing package manifest values and load + // the buildfiles into the respective *-build values. // - if (ev) + if (ev || lb) { for (package_manifest& m: fr.packages) - m.load_files ( - [&m, &rd, &rl] (const string& n, const path& p) - { - return read_package_file (p, - n, - path_cast<dir_path> (*m.location), - rd, - rl, - empty_string /* fragment */); - }, - iu); + { + dir_path pl (path_cast<dir_path> (*m.location)); + + // Load *-file values. + // + try + { + m.load_files ( + [ev, &rd, &rl, &pl] + (const string& n, const path& p) -> optional<string> + { + // Always expand the build-file values. + // + if (ev || n == "build-file") + { + return read_package_file (p, + n, + pl, + rd, + rl, + empty_string /* fragment */); + } + else + return nullopt; + }, + iu); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, nullopt /* fragment */); + dr << endf; + } + + // Load the bootstrap, root, and config/*.build buildfiles into the + // respective *-build values, if requested and if they are not already + // specified in the manifest. + // + if (lb) + try + { + load_package_buildfiles (m, rd / pl, true /* err_path_relative */); + } + catch (const runtime_error& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, nullopt /* fragment */); + dr << endf; + } + } } return rep_fetch_data {{move (fr)}, @@ -458,6 +545,7 @@ namespace bpkg const dir_path* conf, const repository_location& rl, bool iu, + bool it, bool ev, bool lb) { @@ -587,42 +675,86 @@ namespace bpkg td, move (pms), iu, - lb, + it, rl, fr.friendly_name)); fr.packages = move (pmi.first); fr.package_infos = move (pmi.second); - // Expand file-referencing package manifest values checking out - // submodules, if required. + // If requested, expand file-referencing package manifest values + // checking out submodules, if required, and load the buildfiles into + // the respective *-build values. // - if (ev) + if (ev || lb) { for (package_manifest& m: fr.packages) - m.load_files ( - [&m, &td, &rl, &fr, &checkout_submodules] (const string& n, - const path& p) - { - // Note that this doesn't work for symlinks on Windows where git - // normally creates filesystem-agnostic symlinks that are - // indistinguishable from regular files (see fixup_worktree() - // for details). It seems like the only way to deal with that is - // to unconditionally checkout submodules on Windows. Let's not - // pessimize things for now (if someone really wants this to - // work, they can always enable real symlinks in git). - // - if (!exists (td / *m.location / p)) - checkout_submodules (); - - return read_package_file (p, - n, - path_cast<dir_path> (*m.location), - td, - rl, - fr.friendly_name); - }, - iu); + { + dir_path pl (path_cast<dir_path> (*m.location)); + + // Load *-file values. + // + try + { + m.load_files ( + [ev, &td, &rl, &pl, &fr, &checkout_submodules] + (const string& n, const path& p) -> optional<string> + { + // Always expand the build-file values. + // + if (ev || n == "build-file") + { + // Check out submodules if the referenced file doesn't exist. + // + // Note that this doesn't work for symlinks on Windows where + // git normally creates filesystem-agnostic symlinks that + // are indistinguishable from regular files (see + // fixup_worktree() for details). It seems like the only way + // to deal with that is to unconditionally checkout + // submodules on Windows. Let's not pessimize things for now + // (if someone really wants this to work, they can always + // enable real symlinks in git). + // + if (!exists (td / pl / p)) + checkout_submodules (); + + return read_package_file (p, + n, + pl, + td, + rl, + fr.friendly_name); + } + else + return nullopt; + }, + iu); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, fr.friendly_name); + dr << endf; + } + + // Load the bootstrap, root, and config/*.build buildfiles into the + // respective *-build values, if requested and if they are not + // already specified in the manifest. + // + if (lb) + try + { + load_package_buildfiles (m, td / pl, true /* err_path_relative */); + } + catch (const runtime_error& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, fr.friendly_name); + dr << endf; + } + } } np += fr.packages.size (); @@ -657,14 +789,24 @@ namespace bpkg const repository_location& rl, const optional<string>& dt, bool iu, + bool it, bool ev, bool lb) { switch (rl.type ()) { - case repository_type::pkg: return rep_fetch_pkg (co, conf, db, rl, dt, iu); - case repository_type::dir: return rep_fetch_dir (co, rl, iu, ev, lb); - case repository_type::git: return rep_fetch_git (co, conf, rl, iu, ev, lb); + case repository_type::pkg: + { + return rep_fetch_pkg (co, conf, db, rl, dt, iu, it); + } + case repository_type::dir: + { + return rep_fetch_dir (co, rl, iu, it, ev, lb); + } + case repository_type::git: + { + return rep_fetch_git (co, conf, rl, iu, it, ev, lb); + } } assert (false); // Can't be here. @@ -676,6 +818,7 @@ namespace bpkg const dir_path* conf, const repository_location& rl, bool iu, + bool it, bool ev, bool lb) { @@ -685,6 +828,7 @@ namespace bpkg rl, nullopt /* dependent_trust */, iu, + it, ev, lb); } @@ -971,8 +1115,6 @@ namespace bpkg // if (rl.directory_based ()) { - assert (!pis.empty ()); - // Note that we can't check if the external package of this upstream // version and revision is already available in the configuration // until we fetch all the repositories, as some of the available @@ -986,7 +1128,7 @@ namespace bpkg path_cast<dir_path> (rl.path () / *pm.location), pm.name, pm.version, - &pis[i], + !pis.empty () ? &pis[i] : nullptr, false /* check_external */)); if (v) @@ -1171,6 +1313,12 @@ namespace bpkg // repository fragments list, as well as its prerequisite and complement // repository sets. // + // Note that we do this in the forward compatible manner ignoring + // unrecognized manifest values and unsatisfied build2 toolchain + // constraints in the package manifests. This approach allows older + // toolchains to work with newer repositories, successfully building the + // toolchain-satisfied packages and only failing for unsatisfied ones. + // rep_fetch_data rfd ( rep_fetch (co, &db.config_orig, @@ -1178,6 +1326,7 @@ namespace bpkg rl, dependent_trust, true /* ignore_unknow */, + true /* ignore_toolchain */, false /* expand_values */, true /* load_buildfiles */)); @@ -1403,6 +1552,10 @@ namespace bpkg } } +#ifndef NDEBUG + rep_remove_verify (db, t); +#endif + // Make sure that the external packages are available from a single // directory-based repository. // @@ -1468,12 +1621,17 @@ namespace bpkg { dependencies& ds (at.package->dependencies); - // Note that the special test dependencies entry is always the last - // one, if present. + // Note that there is only one special test dependencies entry in + // the test package. // - assert (!ds.empty () && ds.back ().type); - - ds.pop_back (); + for (auto i (ds.begin ()), e (ds.end ()); i != e; ++i) + { + if (i->type) + { + ds.erase (i); + break; + } + } db.update (at.package); } @@ -1482,16 +1640,25 @@ namespace bpkg // Go through the available packages that have external tests and add // them as the special test dependencies to these test packages. // + // Note that not being able to resolve the test package for a main + // package is not an error, since the test package absence doesn't + // affect the main package building and internal testing. Dropping of an + // external test package from a repository may, however, be intentional. + // Think of a private repository crafted as a subset of some public + // repository with the external examples packages omitted. + // for (const auto& am: db.query<available_main> ()) { const shared_ptr<available_package>& p (am.package); + const package_name& n (p->id.name); + const version& v (p->version); vector<shared_ptr<repository_fragment>> rfs; for (const package_location& pl: p->locations) rfs.push_back (pl.repository_fragment.load ()); - bool module (build2_module (p->id.name)); + bool module (build2_module (n)); for (const test_dependency& td: p->tests) { @@ -1501,7 +1668,7 @@ namespace bpkg if (module && !td.buildtime) fail << "run-time " << td.type << ' ' << td.name << " for build " << "system module " - << package_string (p->id.name, p->version) << + << package_string (n, v) << info << "build system modules cannot have run-time " << td.type; vector<pair<shared_ptr<available_package>, @@ -1519,11 +1686,12 @@ namespace bpkg dependencies& ds (tp->dependencies); - if (ds.empty () || !ds.back ().type) - ds.push_back (dependency_alternatives_ex (td.type, - td.buildtime)); - - dependency_alternatives_ex& das (ds.back ()); + // Find the special test dependencies entry, if already present. + // + auto b (ds.begin ()); + auto e (ds.end ()); + auto oi (b); // Old entry location. + for (; oi != e && !oi->type; ++oi) ; // Note that since we store all the primary packages as // alternative dependencies (which must be all of the same @@ -1534,12 +1702,10 @@ namespace bpkg // `== <version>` constraints (see below), so we can use min // version of such a constraint as the primary package version. // - if (das.buildtime != td.buildtime) + if (oi != e && oi->buildtime != td.buildtime) { - // Could only be empty if we just added it, which cannot be the - // case since the build-time flag differs. - // - assert (!das.empty ()); + dependency_alternatives_ex& das (*oi); + assert (!das.empty ()); // Cannot be empty if present. const dependency_alternative& da (das[0]); @@ -1555,19 +1721,87 @@ namespace bpkg << package_string (da[0].name, *da[0].constraint->min_version) << info << (td.buildtime ? "build-time for " : "run-time for ") - << package_string (p->id.name, p->version); + << package_string (n, v); } - dependency_alternative da (nullopt /* enable */, + // Find the (new) location for the special test dependencies entry. + // + // Note that if the entry is already present, it can only be moved + // towards the end of the list. + // + auto ni (e); + + // First, find the last depends clause that explicitly specifies + // this main package but goes after the special entry current + // location, if present. Note that we only consider clauses with + // the matching buildtime flag. + // + for (auto i (oi != e ? oi + 1 : b); i != e; ++i) + { + const dependency_alternatives_ex& das (*i); + if (das.buildtime == td.buildtime) + { + bool specifies (false); + + for (const dependency_alternative& da: das) + { + for (const dependency& d: da) + { + if (d.name == n) + { + specifies = true; + break; + } + } + + if (specifies) + break; + } + + if (specifies) + ni = i; + } + } + + // Now, set ni to refer to the special test dependencies entry, + // moving or creating one, if required. + // + if (oi != e) // The entry already exists? + { + if (ni != e) // Move the entry to the new location? + { + // Move the [oi + 1, ni] range 1 position to the left and + // move the *oi element to the now vacant ni slot. + // + rotate (oi, oi + 1, ni + 1); + } + else + ni = oi; // Leave the entry at the old location. + } + else // The entry doesn't exist. + { + if (ni != e) // Create the entry right after ni? + ++ni; + else + ni = b; // Create the entry at the beginning of the list. + + ni = ds.emplace (ni, td.type, td.buildtime); // Create the entry. + } + + // Finally, add the new dependency alternative to the special + // entry. + // + dependency_alternative da (td.enable, td.reflect, nullopt /* prefer */, nullopt /* accept */, nullopt /* require */); - da.push_back ( - dependency {p->id.name, version_constraint (p->version)}); + da.push_back (dependency {n, version_constraint (v)}); + + assert (ni != ds.end ()); // Must be deduced by now. - das.push_back (move (da)); + ni->push_back (move (da)); db.update (tp); } |