diff options
Diffstat (limited to 'bpkg/rep-fetch.cxx')
| -rw-r--r-- | bpkg/rep-fetch.cxx | 549 |
1 files changed, 429 insertions, 120 deletions
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index d2931ca..d02a064 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -6,7 +6,7 @@ #include <map> #include <set> -#include <libbutl/manifest-parser.mxx> +#include <libbutl/manifest-parser.hxx> #include <bpkg/auth.hxx> #include <bpkg/fetch.hxx> @@ -15,7 +15,10 @@ #include <bpkg/package-odb.hxx> #include <bpkg/database.hxx> #include <bpkg/rep-remove.hxx> +#include <bpkg/pkg-verify.hxx> #include <bpkg/diagnostics.hxx> +#include <bpkg/satisfaction.hxx> +#include <bpkg/package-query.hxx> #include <bpkg/manifest-utility.hxx> using namespace std; @@ -52,7 +55,8 @@ namespace bpkg database* db, const repository_location& rl, const optional<string>& dependent_trust, - bool ignore_unknown) + bool ignore_unknown, + bool ignore_toolchain) { // First fetch the repositories list and authenticate the base's // certificate. @@ -114,6 +118,18 @@ namespace bpkg authenticate_repository (co, conf, cert_pem, *cert, sm, rl); } + // If requested, verify that the packages are compatible with the current + // toolchain. + // + if (!ignore_toolchain) + { + for (const package_manifest& m: fr.packages) + { + for (const dependency_alternatives& das: m.dependencies) + toolchain_buildtime_dependency (co, das, &m.name); + } + } + return rep_fetch_data {{move (fr)}, move (cert_pem), move (cert)}; } @@ -161,7 +177,8 @@ namespace bpkg M r; if (exists (f)) r = parse_manifest<M> (f, iu, rl, fragment); - else + + if (r.empty ()) r.emplace_back (repository_manifest ()); // Add the base repository. return r; @@ -190,6 +207,23 @@ namespace bpkg return r; } + static void + print_package_info (diag_record& dr, + const dir_path& pl, + const repository_location& rl, + const optional<string>& fragment) + { + dr << "package "; + + if (!pl.current ()) + dr << "'" << pl.string () << "' "; // Strip trailing '/'. + + dr << "in repository " << rl; + + if (fragment) + dr << ' ' << *fragment; + } + // Parse package manifests referenced by the package directory manifests. // static pair<vector<package_manifest>, vector<package_info>> @@ -197,35 +231,48 @@ namespace bpkg const dir_path& repo_dir, vector<package_manifest>&& pms, bool iu, + bool it, const repository_location& rl, const optional<string>& fragment) // For diagnostics. { - auto add_package_info = [&rl, &fragment] (const package_manifest& pm, - diag_record& dr) + auto prn_package_info = [&rl, &fragment] (diag_record& dr, + const package_manifest& pm) { - dr << "package "; - - if (!pm.location->current ()) - dr << "'" << pm.location->string () << "' "; // Strip trailing '/'. - - dr << "in repository " << rl; - - if (fragment) - dr << ' ' << *fragment; + print_package_info (dr, + path_cast<dir_path> (*pm.location), + rl, + fragment); }; // Verify that all the package directories contain the package manifest - // files and retrieve the package versions via the single `b info` call. - // While at it cache the manifest paths for the future use. + // files and retrieve the package versions via the single `b info` call, + // but only if the current build2 version is satisfactory for all the + // repository packages. While at it cache the manifest paths for the + // future use. + // + // Note that if the package is not compatible with the toolchain, not to + // end up with an unfriendly build2 error message (referring a line in the + // bootstrap file issued by the version module), we need to verify the + // compatibility of the package manifests prior to calling `b info`. Also + // note that we cannot create the package manifest objects at this stage, + // since we need the package versions for that. Thus, we cache the + // respective name value lists instead. // - paths mfs; - package_version_infos pvs; + optional<package_version_infos> pvs; + paths mfs; + vector<vector<manifest_name_value>> nvs; { mfs.reserve (pms.size ()); + nvs.reserve (pms.size ()); dir_paths pds; pds.reserve (pms.size ()); + // If true, then build2 version is satisfactory for all the repository + // packages. + // + bool bs (true); + for (const package_manifest& pm: pms) { assert (pm.location); @@ -238,21 +285,75 @@ namespace bpkg { diag_record dr (fail); dr << "no manifest file for "; - add_package_info (pm, dr); + prn_package_info (dr, pm); + } + + // Provide the context if the package compatibility verification fails. + // + auto g ( + make_exception_guard ( + [&pm, &prn_package_info] () + { + diag_record dr (info); + + dr << "while retrieving information for "; + prn_package_info (dr, pm); + })); + + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f.string ()); + + // Note that the package directory points to something temporary + // (e.g., .bpkg/tmp/6f746365314d/) and it's probably better to omit + // it entirely (the above exception guard will print all we've got). + // + pkg_verify_result r (pkg_verify (co, mp, it, dir_path ())); + + if (bs && + r.build2_dependency && + !satisfy_build2 (co, *r.build2_dependency)) + { + bs = false; + pds.clear (); // Won't now be used. + } + + nvs.push_back (move (r)); + } + catch (const manifest_parsing& e) + { + fail (e.name, e.line, e.column) << e.description; + } + catch (const io_error& e) + { + fail << "unable to read from " << f << ": " << e; } mfs.push_back (move (f)); - pds.push_back (move (d)); + + if (bs) + pds.push_back (move (d)); } - pvs = package_versions (co, pds); + // Note that for the directory-based repositories we also query + // subprojects since the package information will be used for the + // subsequent package_iteration() call (see below). + // + if (bs) + pvs = package_versions (co, pds, + (rl.directory_based () + ? b_info_flags::subprojects + : b_info_flags::none)); } // Parse package manifests, fixing up their versions. // pair<vector<package_manifest>, vector<package_info>> r; - r.first.reserve (pms.size ()); - r.second.reserve (pms.size ()); + r.first.reserve (pms.size ()); + + if (pvs) + r.second.reserve (pms.size ()); for (size_t i (0); i != pms.size (); ++i) { @@ -260,42 +361,40 @@ namespace bpkg assert (pm.location); - const path& f (mfs[i]); - try { - ifdstream ifs (f); - manifest_parser mp (ifs, f.string ()); - - optional<version>& pv (pvs[i].version); - package_manifest m ( - mp, - [&pv] (version& v) + mfs[i].string (), + move (nvs[i]), + [&pvs, i] (version& v) { - if (pv) - v = move (*pv); + if (pvs) + { + optional<version>& pv ((*pvs)[i].version); + + if (pv) + v = move (*pv); + } }, iu); // Save the package manifest, preserving its location. // m.location = move (*pm.location); + pm = move (m); } catch (const manifest_parsing& e) { diag_record dr (fail (e.name, e.line, e.column)); dr << e.description << info; - add_package_info (pm, dr); - } - catch (const io_error& e) - { - fail << "unable to read from " << f << ": " << e; + prn_package_info (dr, pm); } r.first.push_back (move (pm)); - r.second.push_back (move (pvs[i].info)); + + if (pvs) + r.second.push_back (move ((*pvs)[i].info)); } return r; @@ -319,11 +418,11 @@ namespace bpkg ifdstream is (fp); string s (is.read_text ()); - if (s.empty ()) + if (s.empty () && name != "build-file") fail << name << " manifest value in " << pkg / manifest_file << " references empty file " << rp << info << "repository " << rl - << (!fragment.empty () ? " " + fragment : ""); + << (!fragment.empty () ? ' ' + fragment : ""); return s; } @@ -333,7 +432,7 @@ namespace bpkg << name << " manifest value in " << pkg / manifest_file << ": " << e << info << "repository " << rl - << (!fragment.empty () ? " " + fragment : "") << endf; + << (!fragment.empty () ? ' ' + fragment : "") << endf; } } @@ -341,7 +440,9 @@ namespace bpkg rep_fetch_dir (const common_options& co, const repository_location& rl, bool iu, - bool ev) + bool it, + bool ev, + bool lb) { assert (rl.absolute ()); @@ -367,28 +468,71 @@ namespace bpkg rd, move (pms), iu, + it, rl, empty_string /* fragment */)); fr.packages = move (pmi.first); fr.package_infos = move (pmi.second); - // Expand file-referencing package manifest values. + // If requested, expand file-referencing package manifest values and load + // the buildfiles into the respective *-build values. // - if (ev) + if (ev || lb) { for (package_manifest& m: fr.packages) - m.load_files ( - [&m, &rd, &rl] (const string& n, const path& p) - { - return read_package_file (p, - n, - path_cast<dir_path> (*m.location), - rd, - rl, - empty_string /* fragment */); - }, - iu); + { + dir_path pl (path_cast<dir_path> (*m.location)); + + // Load *-file values. + // + try + { + m.load_files ( + [ev, &rd, &rl, &pl] + (const string& n, const path& p) -> optional<string> + { + // Always expand the build-file values. + // + if (ev || n == "build-file") + { + return read_package_file (p, + n, + pl, + rd, + rl, + empty_string /* fragment */); + } + else + return nullopt; + }, + iu); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, nullopt /* fragment */); + dr << endf; + } + + // Load the bootstrap, root, and config/*.build buildfiles into the + // respective *-build values, if requested and if they are not already + // specified in the manifest. + // + if (lb) + try + { + load_package_buildfiles (m, rd / pl, true /* err_path_relative */); + } + catch (const runtime_error& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, nullopt /* fragment */); + dr << endf; + } + } } return rep_fetch_data {{move (fr)}, @@ -401,14 +545,16 @@ namespace bpkg const dir_path* conf, const repository_location& rl, bool iu, - bool ev) + bool it, + bool ev, + bool lb) { - auto i (temp_dir.find (conf != nullptr ? *conf : empty_dir_path)); - assert (i != temp_dir.end ()); + auto i (tmp_dirs.find (conf != nullptr ? *conf : empty_dir_path)); + assert (i != tmp_dirs.end ()); dir_path sd (repository_state (rl)); - auto_rmdir rm (i->second / sd); + auto_rmdir rm (i->second / sd, !keep_tmp); const dir_path& td (rm.path); if (exists (td)) @@ -529,41 +675,86 @@ namespace bpkg td, move (pms), iu, + it, rl, fr.friendly_name)); fr.packages = move (pmi.first); fr.package_infos = move (pmi.second); - // Expand file-referencing package manifest values checking out - // submodules, if required. + // If requested, expand file-referencing package manifest values + // checking out submodules, if required, and load the buildfiles into + // the respective *-build values. // - if (ev) + if (ev || lb) { for (package_manifest& m: fr.packages) - m.load_files ( - [&m, &td, &rl, &fr, &checkout_submodules] (const string& n, - const path& p) - { - // Note that this doesn't work for symlinks on Windows where git - // normally creates filesystem-agnostic symlinks that are - // indistinguishable from regular files (see fixup_worktree() - // for details). It seems like the only way to deal with that is - // to unconditionally checkout submodules on Windows. Let's not - // pessimize things for now (if someone really wants this to - // work, they can always enable real symlinks in git). - // - if (!exists (td / *m.location / p)) - checkout_submodules (); - - return read_package_file (p, - n, - path_cast<dir_path> (*m.location), - td, - rl, - fr.friendly_name); - }, - iu); + { + dir_path pl (path_cast<dir_path> (*m.location)); + + // Load *-file values. + // + try + { + m.load_files ( + [ev, &td, &rl, &pl, &fr, &checkout_submodules] + (const string& n, const path& p) -> optional<string> + { + // Always expand the build-file values. + // + if (ev || n == "build-file") + { + // Check out submodules if the referenced file doesn't exist. + // + // Note that this doesn't work for symlinks on Windows where + // git normally creates filesystem-agnostic symlinks that + // are indistinguishable from regular files (see + // fixup_worktree() for details). It seems like the only way + // to deal with that is to unconditionally checkout + // submodules on Windows. Let's not pessimize things for now + // (if someone really wants this to work, they can always + // enable real symlinks in git). + // + if (!exists (td / pl / p)) + checkout_submodules (); + + return read_package_file (p, + n, + pl, + td, + rl, + fr.friendly_name); + } + else + return nullopt; + }, + iu); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, fr.friendly_name); + dr << endf; + } + + // Load the bootstrap, root, and config/*.build buildfiles into the + // respective *-build values, if requested and if they are not + // already specified in the manifest. + // + if (lb) + try + { + load_package_buildfiles (m, td / pl, true /* err_path_relative */); + } + catch (const runtime_error& e) + { + diag_record dr (fail); + dr << e << info; + print_package_info (dr, pl, rl, fr.friendly_name); + dr << endf; + } + } } np += fr.packages.size (); @@ -598,13 +789,24 @@ namespace bpkg const repository_location& rl, const optional<string>& dt, bool iu, - bool ev) + bool it, + bool ev, + bool lb) { switch (rl.type ()) { - case repository_type::pkg: return rep_fetch_pkg (co, conf, db, rl, dt, iu); - case repository_type::dir: return rep_fetch_dir (co, rl, iu, ev); - case repository_type::git: return rep_fetch_git (co, conf, rl, iu, ev); + case repository_type::pkg: + { + return rep_fetch_pkg (co, conf, db, rl, dt, iu, it); + } + case repository_type::dir: + { + return rep_fetch_dir (co, rl, iu, it, ev, lb); + } + case repository_type::git: + { + return rep_fetch_git (co, conf, rl, iu, it, ev, lb); + } } assert (false); // Can't be here. @@ -616,7 +818,9 @@ namespace bpkg const dir_path* conf, const repository_location& rl, bool iu, - bool ev) + bool it, + bool ev, + bool lb) { return rep_fetch (co, conf, @@ -624,7 +828,9 @@ namespace bpkg rl, nullopt /* dependent_trust */, iu, - ev); + it, + ev, + lb); } // Return an existing repository fragment or create a new one. Update the @@ -909,8 +1115,6 @@ namespace bpkg // if (rl.directory_based ()) { - assert (!pis.empty ()); - // Note that we can't check if the external package of this upstream // version and revision is already available in the configuration // until we fetch all the repositories, as some of the available @@ -924,7 +1128,7 @@ namespace bpkg path_cast<dir_path> (rl.path () / *pm.location), pm.name, pm.version, - &pis[i], + !pis.empty () ? &pis[i] : nullptr, false /* check_external */)); if (v) @@ -1109,6 +1313,12 @@ namespace bpkg // repository fragments list, as well as its prerequisite and complement // repository sets. // + // Note that we do this in the forward compatible manner ignoring + // unrecognized manifest values and unsatisfied build2 toolchain + // constraints in the package manifests. This approach allows older + // toolchains to work with newer repositories, successfully building the + // toolchain-satisfied packages and only failing for unsatisfied ones. + // rep_fetch_data rfd ( rep_fetch (co, &db.config_orig, @@ -1116,7 +1326,9 @@ namespace bpkg rl, dependent_trust, true /* ignore_unknow */, - false /* expand_values */)); + true /* ignore_toolchain */, + false /* expand_values */, + true /* load_buildfiles */)); // Save for subsequent certificate authentication for repository use by // its dependents. @@ -1405,12 +1617,17 @@ namespace bpkg { dependencies& ds (at.package->dependencies); - // Note that the special test dependencies entry is always the last - // one, if present. + // Note that there is only one special test dependencies entry in + // the test package. // - assert (!ds.empty () && ds.back ().type); - - ds.pop_back (); + for (auto i (ds.begin ()), e (ds.end ()); i != e; ++i) + { + if (i->type) + { + ds.erase (i); + break; + } + } db.update (at.package); } @@ -1419,33 +1636,36 @@ namespace bpkg // Go through the available packages that have external tests and add // them as the special test dependencies to these test packages. // + // Note that not being able to resolve the test package for a main + // package is not an error, since the test package absence doesn't + // affect the main package building and internal testing. Dropping of an + // external test package from a repository may, however, be intentional. + // Think of a private repository crafted as a subset of some public + // repository with the external examples packages omitted. + // for (const auto& am: db.query<available_main> ()) { const shared_ptr<available_package>& p (am.package); + const package_name& n (p->id.name); + const version& v (p->version); vector<shared_ptr<repository_fragment>> rfs; for (const package_location& pl: p->locations) rfs.push_back (pl.repository_fragment.load ()); - // @@ TMP: enable after 0.14.0 is out. - // -#if 0 - bool module (build2_module (p->id.name)); -#endif + bool module (build2_module (n)); for (const test_dependency& td: p->tests) { // Verify that the package has no runtime tests if it is a build // system module. // -#if 0 if (module && !td.buildtime) fail << "run-time " << td.type << ' ' << td.name << " for build " << "system module " - << package_string (p->id.name, p->version) << + << package_string (n, v) << info << "build system modules cannot have run-time " << td.type; -#endif vector<pair<shared_ptr<available_package>, shared_ptr<repository_fragment>>> tps ( @@ -1462,11 +1682,12 @@ namespace bpkg dependencies& ds (tp->dependencies); - if (ds.empty () || !ds.back ().type) - ds.push_back (dependency_alternatives_ex (td.type, - td.buildtime)); - - dependency_alternatives_ex& da (ds.back ()); + // Find the special test dependencies entry, if already present. + // + auto b (ds.begin ()); + auto e (ds.end ()); + auto oi (b); // Old entry location. + for (; oi != e && !oi->type; ++oi) ; // Note that since we store all the primary packages as // alternative dependencies (which must be all of the same @@ -1477,18 +1698,106 @@ namespace bpkg // `== <version>` constraints (see below), so we can use min // version of such a constraint as the primary package version. // - if (da.buildtime != td.buildtime) + if (oi != e && oi->buildtime != td.buildtime) + { + dependency_alternatives_ex& das (*oi); + assert (!das.empty ()); // Cannot be empty if present. + + const dependency_alternative& da (das[0]); + + // We always add the primary package to the test package as a + // single-dependency alternative (see below). + // + assert (da.size () == 1); + fail << to_string (td.type) << " package " << td.name << " is a " << "build-time dependency for one primary package and a " << "run-time for another" << - info << (da.buildtime ? "build-time for " : "run-time for ") + info << (das.buildtime ? "build-time for " : "run-time for ") << package_string (da[0].name, *da[0].constraint->min_version) << info << (td.buildtime ? "build-time for " : "run-time for ") - << package_string (p->id.name, p->version); + << package_string (n, v); + } + + // Find the (new) location for the special test dependencies entry. + // + // Note that if the entry is already present, it can only be moved + // towards the end of the list. + // + auto ni (e); + + // First, find the last depends clause that explicitly specifies + // this main package but goes after the special entry current + // location, if present. Note that we only consider clauses with + // the matching buildtime flag. + // + for (auto i (oi != e ? oi + 1 : b); i != e; ++i) + { + const dependency_alternatives_ex& das (*i); + if (das.buildtime == td.buildtime) + { + bool specifies (false); + + for (const dependency_alternative& da: das) + { + for (const dependency& d: da) + { + if (d.name == n) + { + specifies = true; + break; + } + } + + if (specifies) + break; + } + + if (specifies) + ni = i; + } + } + + // Now, set ni to refer to the special test dependencies entry, + // moving or creating one, if required. + // + if (oi != e) // The entry already exists? + { + if (ni != e) // Move the entry to the new location? + { + // Move the [oi + 1, ni] range 1 position to the left and + // move the *oi element to the now vacant ni slot. + // + rotate (oi, oi + 1, ni + 1); + } + else + ni = oi; // Leave the entry at the old location. + } + else // The entry doesn't exist. + { + if (ni != e) // Create the entry right after ni? + ++ni; + else + ni = b; // Create the entry at the beginning of the list. + + ni = ds.emplace (ni, td.type, td.buildtime); // Create the entry. + } + + // Finally, add the new dependency alternative to the special + // entry. + // + dependency_alternative da (td.enable, + td.reflect, + nullopt /* prefer */, + nullopt /* accept */, + nullopt /* require */); + + da.push_back (dependency {n, version_constraint (v)}); + + assert (ni != ds.end ()); // Must be deduced by now. - da.push_back (dependency {p->id.name, - version_constraint (p->version)}); + ni->push_back (move (da)); db.update (tp); } |