diff options
Diffstat (limited to 'load')
| -rw-r--r-- | load/buildfile | 9 | ||||
| -rw-r--r-- | load/load.cli | 67 | ||||
| -rw-r--r-- | load/load.cxx | 986 | ||||
| -rw-r--r-- | load/types-parsers.cxx | 1 | ||||
| -rw-r--r-- | load/types-parsers.hxx | 1 |
5 files changed, 809 insertions, 255 deletions
diff --git a/load/buildfile b/load/buildfile index 493d067..4278f20 100644 --- a/load/buildfile +++ b/load/buildfile @@ -1,5 +1,4 @@ # file : load/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file import libs = libodb%lib{odb} @@ -12,6 +11,10 @@ include ../libbrep/ exe{brep-load}: {hxx ixx cxx}{* -load-options} {hxx ixx cxx}{load-options} \ ../libbrep/lib{brep} $libs +# Build options. +# +obj{load}: cxx.poptions += -DBREP_COPYRIGHT=\"$copyright\" + # Generated options parser. # if $cli.configured @@ -20,8 +23,8 @@ if $cli.configured cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix load --guard-prefix LOAD --generate-specifier \ ---cxx-prologue "#include <load/types-parsers.hxx>" --page-usage print_ \ ---ansi-color --long-usage +--generate-modifier --cxx-prologue "#include <load/types-parsers.hxx>" \ +--page-usage print_ --ansi-color --long-usage # Include the generated cli files into the distribution and don't remove # them when cleaning in src (so that clean results in a state identical to diff --git a/load/load.cli b/load/load.cli index 74c91f2..99d76f6 100644 --- a/load/load.cli +++ b/load/load.cli @@ -1,5 +1,4 @@ // file : load/load.cli -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file include <vector>; @@ -54,9 +53,18 @@ class options bool --shallow { "Don't load package information from prerequisite or complement - repositories." + repositories, don't fail if unable to resolve a package dependency, and + don't detect package dependency cycles." }; + bool --ignore-unresolved-tests + { + "Ignore tests, examples, and benchmarks package manifest entries which + cannot be resolved from the main package's complement repositories, + recursively. Note that in contrast to --shallow option, such entries will + be removed from the main package manifests outright." + } + std::string --tenant { "<id>", @@ -64,6 +72,41 @@ class options specified, then the single-tenant mode is assumed." }; + bool --private + { + "Display the tenant packages in the web interface only in the tenant view + mode." + }; + + std::string --interactive + { + "<bkp>", + "Build the tenant packages interactively, stopping builds at the specified + breakpoint. Implies \cb{--private}." + }; + + std::string --service-id + { + "<id>", + "Third party service information to associate with the being created + tenant. Requires the \cb{--tenant} and \cb{--service-type} options to be + specified." + }; + + std::string --service-type + { + "<type>", + "Type of the service to associate with the being created tenant. Requires + the \cb{--service-id} option to be specified." + }; + + std::string --service-data + { + "<data>", + "Service data to associate with the being created tenant. Requires the + \cb{--service-id} option to be specified." + }; + brep::path --overrides-file { "<file>", @@ -124,6 +167,22 @@ class options this option to specify multiple package manager options." } + brep::path openssl = "openssl" + { + "<path>", + "The openssl program to be used for crypto operations. You can also + specify additional options that should be passed to the openssl program + with \cb{openssl-option}. If the openssl program is not explicitly + specified, then \cb{brep-load} will use \cb{openssl} by default." + } + + brep::strings openssl-option + { + "<opt>", + "Additional option to be passed to the openssl program (see \cb{openssl} + for details). Repeat this option to specify multiple openssl options." + } + std::string --pager // String to allow empty value. { "<path>", @@ -162,8 +221,8 @@ Fatal error.| \li|\cb{2} -An instance of \cb{brep-load} or \l{brep-migrate(1)} is already running. Try -again.| +An instance of \cb{brep-load} or some other \cb{brep} utility is already +running. Try again.| \li|\cb{3} diff --git a/load/load.cxx b/load/load.cxx index 83cc9e6..474b443 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -1,13 +1,13 @@ // file : load/load.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file #include <signal.h> // signal() #include <cerrno> -#include <cstring> // strncmp() +#include <chrono> +#include <thread> // this_thread::sleep_for() +#include <cstring> // strncmp() #include <iostream> -#include <algorithm> // find(), find_if() #include <odb/session.hxx> #include <odb/database.hxx> @@ -17,13 +17,14 @@ #include <odb/pgsql/database.hxx> -#include <libbutl/pager.mxx> -#include <libbutl/sha256.mxx> -#include <libbutl/process.mxx> -#include <libbutl/fdstream.mxx> -#include <libbutl/filesystem.mxx> -#include <libbutl/tab-parser.mxx> -#include <libbutl/manifest-parser.mxx> +#include <libbutl/pager.hxx> +#include <libbutl/sha256.hxx> +#include <libbutl/process.hxx> +#include <libbutl/openssl.hxx> +#include <libbutl/fdstream.hxx> +#include <libbutl/filesystem.hxx> +#include <libbutl/tab-parser.hxx> +#include <libbutl/manifest-parser.hxx> #include <libbpkg/manifest.hxx> @@ -37,6 +38,7 @@ using std::cout; using std::cerr; using std::endl; +using namespace std::this_thread; using namespace odb::core; using namespace butl; using namespace bpkg; @@ -54,6 +56,17 @@ static const char* help_info ( static const path packages ("packages.manifest"); static const path repositories ("repositories.manifest"); +// Retry executing bpkg on recoverable errors for about 10 seconds. +// +// Should we just exit with some "bpkg recoverable" code instead and leave it +// to the caller to perform retries? Feels like it's better to handle such +// errors ourselves rather than to complicate every caller. Note that having +// some frequently updated prerequisite repository can make these errors quite +// probable, even if the internal repositories are rarely updated. +// +static const size_t bpkg_retries (10); +static const std::chrono::seconds bpkg_retry_timeout (1); + struct internal_repository { repository_location location; @@ -249,7 +262,7 @@ load_repositories (path p) bad_line ("invalid buildable option value"); } else - bad_line ("invalid option '" + nv + "'"); + bad_line ("invalid option '" + nv + '\''); } // For now cache option is mandatory. @@ -352,23 +365,22 @@ repository_info (const options& lo, const string& rl, const cstrings& options) // the repository. Should be called once per repository. // static void -load_packages (const shared_ptr<repository>& rp, +load_packages (const options& lo, + const shared_ptr<repository>& rp, + const repository_location& cl, database& db, bool ignore_unknown, - const manifest_name_values& overrides) + const manifest_name_values& overrides, + const string& overrides_name) { // packages_timestamp other than timestamp_nonexistent signals the // repository packages are already loaded. // assert (rp->packages_timestamp == timestamp_nonexistent); - // Only locally accessible repositories allowed until package manager API is - // ready. - // - assert (!rp->cache_location.empty ()); - vector<package_manifest> pms; - const repository_location& cl (rp->cache_location); + + assert (!cl.empty ()); path p (cl.path () / packages); @@ -395,8 +407,8 @@ load_packages (const shared_ptr<repository>& rp, mp, move (nv), ignore_unknown, - false /* complete_depends */, - package_manifest_flags::forbid_incomplete_dependencies); + false /* complete_values */, + package_manifest_flags::forbid_incomplete_values); } else pms = pkg_package_manifests (mp, ignore_unknown); @@ -408,11 +420,15 @@ load_packages (const shared_ptr<repository>& rp, } using brep::dependency; + using brep::dependency_alternative; + using brep::dependency_alternatives; + + const string& tenant (rp->tenant); for (package_manifest& pm: pms) { shared_ptr<package> p ( - db.find<package> (package_id (rp->tenant, pm.name, pm.version))); + db.find<package> (package_id (tenant, pm.name, pm.version))); // sha256sum should always be present if the package manifest comes from // the packages.manifest file belonging to the pkg repository. @@ -421,115 +437,296 @@ load_packages (const shared_ptr<repository>& rp, if (p == nullptr) { - if (rp->internal) + // Apply the package manifest overrides. + // + if (!overrides.empty ()) + try { - try - { - pm.override (overrides, "" /* name */); - } - catch (const manifest_parsing&) + pm.override (overrides, overrides_name); + } + catch (const manifest_parsing& e) + { + cerr << "error: unable to override " << pm.name << ' ' << pm.version + << " manifest: " << e << endl; + + throw failed (); + } + + // Convert the package manifest build configurations (contain public + // keys data) into the brep's build package configurations (contain + // public key object lazy pointers). Keep the bot key lists empty if + // the package is not buildable. + // + package_build_configs build_configs; + + if (!pm.build_configs.empty ()) + { + build_configs.reserve (pm.build_configs.size ()); + + for (bpkg::build_package_config& c: pm.build_configs) { - // Overrides are already validated (see below). - // - assert (false); + build_configs.emplace_back (move (c.name), + move (c.arguments), + move (c.comment), + move (c.builds), + move (c.constraints), + move (c.auxiliaries), + package_build_bot_keys (), + move (c.email), + move (c.warning_email), + move (c.error_email)); } + } + if (rp->internal) + { // Create internal package object. // - optional<string> dsc; - optional<text_type> dst; - - if (pm.description) + // Return nullopt if the text is in a file (can happen if the + // repository is of a type other than pkg) or if the type is not + // recognized (can only happen in the "ignore unknown" mode). + // + auto to_typed_text = [&cl, ignore_unknown] (typed_text_file&& v) { + optional<typed_text> r; + // The description value should not be of the file type if the // package manifest comes from the pkg repository. // - assert (!pm.description->file || cl.type () != repository_type::pkg); + assert (!v.file || cl.type () != repository_type::pkg); - if (!pm.description->file) + if (!v.file) { - dst = pm.effective_description_type (ignore_unknown); + // Cannot throw since the manifest parser has already verified the + // effective type in the same "ignore unknown" mode. + // + optional<text_type> t (v.effective_type (ignore_unknown)); // If the description type is unknown (which may be the case for // some "transitional" period and only if --ignore-unknown is // specified) we just silently drop the description. // - assert (dst || ignore_unknown); + assert (t || ignore_unknown); - if (dst) - dsc = move (pm.description->text); + if (t) + r = typed_text {move (v.text), *t}; } - } - string chn; + return r; + }; + + // Convert descriptions. + // + optional<typed_text> ds ( + pm.description + ? to_typed_text (move (*pm.description)) + : optional<typed_text> ()); + + optional<typed_text> pds ( + pm.package_description + ? to_typed_text (move (*pm.package_description)) + : optional<typed_text> ()); + + // Merge changes into a single typed text object. + // + // If the text type is not recognized for any changes entry or some + // entry refers to a file, then assume that no changes are specified. + // + optional<typed_text> chn; + for (auto& c: pm.changes) { - // The changes value should not be of the file type if the package - // manifest comes from the pkg repository. - // - assert (!c.file || cl.type () != repository_type::pkg); + optional<typed_text> tc (to_typed_text (move (c))); - if (!c.file) + if (!tc) { - if (chn.empty ()) - chn = move (c.text); - else - { - if (chn.back () != '\n') - chn += '\n'; // Always have a blank line as a separator. - - chn += "\n" + c.text; - } + chn = nullopt; + break; } - } - dependencies ds; - - for (auto& pda: pm.dependencies) - { - // Ignore special build2 and bpkg dependencies. We may not have - // packages for them and also showing them for every package is - // probably not very helpful. - // - if (pda.buildtime && !pda.empty ()) + if (!chn) { - const package_name& n (pda.front ().name); - if (n == "build2" || n == "bpkg") - continue; + chn = move (*tc); } + else + { + // Should have failed while parsing the manifest otherwise. + // + assert (tc->type == chn->type); - ds.emplace_back (pda.conditional, pda.buildtime, move (pda.comment)); + string& v (chn->text); - for (auto& pd: pda) - // The package member will be assigned during dependency - // resolution procedure. - // - ds.back ().push_back (dependency {move (pd.name), - move (pd.constraint), - nullptr /* package */}); + assert (!v.empty ()); // Changes manifest value cannot be empty. + + if (v.back () != '\n') + v += '\n'; // Always have a blank line as a separator. + + v += '\n'; + v += tc->text; + } } - auto deps = [] (small_vector<bpkg::dependency, 1>&& ds) + dependencies tds; + + for (auto& das: pm.dependencies) { - small_vector<dependency, 1> r; + dependency_alternatives tdas (das.buildtime, move (das.comment)); - if (!ds.empty ()) + for (auto& da: das) { - r.reserve (ds.size ()); + dependency_alternative tda (move (da.enable), + move (da.reflect), + move (da.prefer), + move (da.accept), + move (da.require)); + + for (auto& d: da) + { + package_name& n (d.name); + + // Ignore special build2 and bpkg dependencies. We may not have + // packages for them and also showing them for every package is + // probably not very helpful. + // + if (das.buildtime && (n == "build2" || n == "bpkg")) + continue; + + // The package member will be assigned during dependency + // resolution procedure. + // + tda.push_back (dependency {move (n), + move (d.constraint), + nullptr /* package */}); + } - for (bpkg::dependency& d: ds) - r.push_back (dependency {move (d.name), - move (d.constraint), - nullptr /* package */}); + if (!tda.empty ()) + tdas.push_back (move (tda)); } - return r; - }; + if (!tdas.empty ()) + tds.push_back (move (tdas)); + } + + small_vector<brep::test_dependency, 1> ts; + + if (!pm.tests.empty ()) + { + ts.reserve (pm.tests.size ()); + + for (bpkg::test_dependency& td: pm.tests) + ts.emplace_back (move (td.name), + td.type, + td.buildtime, + move (td.constraint), + move (td.enable), + move (td.reflect)); + } // Cache before the package name is moved. // package_name project (pm.effective_project ()); + // If the package is buildable, then save the package manifest's + // common and build configuration-specific bot keys into the database + // and translate the key data lists into the lists of the public key + // object lazy pointers. + // + package_build_bot_keys bot_keys; + + if (rp->buildable) + { + // Save the specified bot keys into the database as public key + // objects, unless they are already persisted. Translate these keys + // into the public key object lazy pointers. + // + auto keys_to_objects = [&lo, + &pm, + &tenant, + &db] (strings&& keys) + { + package_build_bot_keys r; + + if (keys.empty ()) + return r; + + r.reserve (keys.size ()); + + for (string& key: keys) + { + // Calculate the key fingerprint. + // + string fp; + + try + { + openssl os (path ("-"), path ("-"), 2, + lo.openssl (), + "pkey", + lo.openssl_option (), "-pubin", "-outform", "DER"); + + os.out << key; + os.out.close (); + + fp = sha256 (os.in).string (); + os.in.close (); + + if (!os.wait ()) + { + cerr << "process " << lo.openssl () << ' ' << *os.exit + << endl; + + throw io_error (""); + } + } + catch (const io_error&) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << endl + << " info: key:" << endl + << key << endl; + + throw failed (); + } + catch (const process_error& e) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << ": " + << e << endl; + + throw failed (); + } + + // Try to find the public_key object for the calculated + // fingerprint. If it doesn't exist, then create and persist the + // new object. + // + public_key_id id (tenant, move (fp)); + shared_ptr<public_key> k (db.find<public_key> (id)); + + if (k == nullptr) + { + k = make_shared<public_key> (move (id.tenant), + move (id.fingerprint), + move (key)); + + db.persist (k); + } + + r.push_back (move (k)); + } + + return r; + }; + + bot_keys = keys_to_objects (move (pm.build_bot_keys)); + + assert (build_configs.size () == pm.build_configs.size ()); + + for (size_t i (0); i != build_configs.size (); ++i) + build_configs[i].bot_keys = + keys_to_objects (move (pm.build_configs[i].bot_keys)); + } + p = make_shared<package> ( move (pm.name), move (pm.version), @@ -540,8 +737,8 @@ load_packages (const shared_ptr<repository>& rp, move (pm.license_alternatives), move (pm.topics), move (pm.keywords), - move (dsc), - move (dst), + move (ds), + move (pds), move (chn), move (pm.url), move (pm.doc_url), @@ -552,13 +749,14 @@ load_packages (const shared_ptr<repository>& rp, move (pm.build_email), move (pm.build_warning_email), move (pm.build_error_email), - move (ds), + move (tds), move (pm.requirements), - deps (move (pm.tests)), - deps (move (pm.examples)), - deps (move (pm.benchmarks)), + move (ts), move (pm.builds), move (pm.build_constraints), + move (pm.build_auxiliaries), + move (bot_keys), + move (build_configs), move (pm.location), move (pm.fragment), move (pm.sha256sum), @@ -567,7 +765,13 @@ load_packages (const shared_ptr<repository>& rp, else // Create external package object. // - p = make_shared<package> (move (pm.name), move (pm.version), rp); + p = make_shared<package> (move (pm.name), + move (pm.version), + move (pm.builds), + move (pm.build_constraints), + move (pm.build_auxiliaries), + move (build_configs), + rp); db.persist (p); } @@ -601,9 +805,14 @@ load_packages (const shared_ptr<repository>& rp, // A non-stub package is buildable if belongs to at least one // buildable repository (see libbrep/package.hxx for details). + // Note that if this is an external test package it will be marked as + // unbuildable later (see resolve_dependencies() for details). // - if (!p->stub () && !p->buildable) - p->buildable = rp->buildable; + if (rp->buildable && !p->buildable && !p->stub ()) + { + p->buildable = true; + p->unbuildable_reason = nullopt; + } } p->other_repositories.push_back (rp); @@ -620,7 +829,9 @@ load_packages (const shared_ptr<repository>& rp, // changed members. Should be called once per persisted internal repository. // static void -load_repositories (const shared_ptr<repository>& rp, +load_repositories (const options& lo, + const shared_ptr<repository>& rp, + const repository_location& cl, database& db, bool ignore_unknown, bool shallow) @@ -630,11 +841,6 @@ load_repositories (const shared_ptr<repository>& rp, // assert (rp->repositories_timestamp == timestamp_nonexistent); - // Only locally accessible repositories allowed until package manager API is - // ready. - // - assert (!rp->cache_location.empty ()); - const string& tenant (rp->tenant); // Repository is already persisted by the load_packages() function call. @@ -644,7 +850,9 @@ load_repositories (const shared_ptr<repository>& rp, pkg_repository_manifests rpm; - path p (rp->cache_location.path () / repositories); + assert (!cl.empty ()); + + path p (cl.path () / repositories); try { @@ -653,6 +861,9 @@ load_repositories (const shared_ptr<repository>& rp, manifest_parser mp (ifs, p.string ()); rpm = pkg_repository_manifests (mp, ignore_unknown); + + if (rpm.empty ()) + rpm.emplace_back (repository_manifest ()); // Add the base repository. } catch (const io_error& e) { @@ -779,15 +990,19 @@ load_repositories (const shared_ptr<repository>& rp, pr = make_shared<repository> (tenant, move (rl)); - // If the prerequsite repository location is a relative path, then - // calculate its cache location. + // If the base repository is internal and the prerequsite repository + // location is a relative path, then calculate its cache location. // - if (rm.location.relative ()) + if (rp->internal && rm.location.relative ()) { + // For an internal repository the cache location always comes from the + // loadtab file. + // + assert (cl.path () == rp->cache_location.path ()); + try { - pr->cache_location = - repository_location (rm.location, rp->cache_location); + pr->cache_location = repository_location (rm.location, cl); } catch (const invalid_argument&) { @@ -795,21 +1010,162 @@ load_repositories (const shared_ptr<repository>& rp, << "repository '" << rm.location << "'" << endl << " info: base (internal) repository location is " << rp->location << endl - << " info: base repository cache location is " - << rp->cache_location << endl; + << " info: base repository cache location is " << cl << endl; throw failed (); } } + // If the (external) prerequisite repository cache location is empty, then + // check if the repository is local and, if that's the case, use its + // location as a cache location. Otherwise, fetch the repository + // information creating a temporary cache for it. + // + auto_rmdir cdr; // Remove the temporary cache after the repo load. + repository_location cl; // Repository temporary cache location. + + if (pr->cache_location.empty ()) + { + if (pr->location.local ()) + { + pr->cache_location = pr->location; + } + else + { + dir_path cd; + + try + { + cd = dir_path::temp_path ("brep-load-cache"); + } + catch (const system_error& e) + { + cerr << "unable to obtain temporary directory: " << e; + throw failed (); + } + + // It's highly unlikely but still possible that the temporary cache + // directory already exists. This can only happen due to the unclean + // loader termination. Let's remove it and retry. + // + try + { + if (try_mkdir (cd) == mkdir_status::already_exists) + { + try_rmdir_r (cd); + + if (try_mkdir (cd) == mkdir_status::already_exists) + throw_generic_error (EEXIST); + } + } + catch (const system_error& e) + { + cerr << "unable to create directory '" << cd << "': " << e; + throw failed (); + } + + cdr = auto_rmdir (cd); + + path rf (cd / repositories); + path pf (cd / packages); + + // Note that the fetch timeout can be overridden via --bpkg-option. + // + cstrings args { + "--fetch-timeout", "60", // 1 minute. + "--deep", + "--manifest", + "--repositories", + "--repositories-file", rf.string ().c_str (), + "--packages", + "--packages-file", pf.string ().c_str ()}; + + if (rm.trust) + { + args.push_back ("--trust"); + args.push_back (rm.trust->c_str ()); + } + + // Always add it, so bpkg won't try to prompt for a certificate + // authentication if the fingerprint doesn't match. + // + args.push_back ("--trust-no"); + + // Retry bpkg-rep-info on recoverable errors, for a while. + // + for (size_t i (0);; ++i) + { + if (i != 0) + { + // Let's follow up the bpkg's diagnostics with the number of + // retries left. + // + cerr << bpkg_retries - i + 1 << " retries left" << endl; + sleep_for (bpkg_retry_timeout); + } + + process p (repository_info (lo, pr->location.string (), args)); + + try + { + // Bail out from the retry loop on success. + // + if (p.wait ()) + break; + + // Assume the child issued diagnostics if terminated normally. + // + if (p.exit->normal ()) + { + // Retry the manifests fetch on a recoverable error, unless the + // retries limit is reached. + // + if (p.exit->code () == 2 && i != bpkg_retries) + continue; + } + else + cerr << "process " << lo.bpkg () << " " << *p.exit << endl; + + cerr << "error: unable to fetch manifests for " + << pr->canonical_name << endl + << " info: base repository location is " + << rp->location << endl; + + throw failed (); + } + catch (const process_error& e) + { + cerr << "error: unable to fetch manifests for " + << pr->canonical_name << ": " << e << endl; + + throw failed (); + } + } + + // Note that this is a non-pkg repository cache and so we create the + // dir repository location (see load_repositories(path) for details). + // + cl = repository_location (repository_url (cd.string ()), + repository_type::dir); + } + } + // We don't apply overrides to the external packages. // - load_packages (pr, + load_packages (lo, + pr, + !pr->cache_location.empty () ? pr->cache_location : cl, db, ignore_unknown, - manifest_name_values () /* overrides */); - - load_repositories (pr, db, ignore_unknown, false /* shallow */); + manifest_name_values () /* overrides */, + "" /* overrides_name */); + + load_repositories (lo, + pr, + !pr->cache_location.empty () ? pr->cache_location : cl, + db, + ignore_unknown, + false /* shallow */); } db.update (rp); @@ -850,29 +1206,36 @@ find (const lazy_shared_ptr<repository>& r, return false; } -// Resolve package run-time dependencies, tests, examples, and benchmarks. -// Make sure that the best matching dependency belongs to the package -// repositories, their complements, recursively, or their immediate -// prerequisite repositories (only for run-time dependencies). Should be -// called once per internal package. +// Resolve package regular dependencies and external tests. Make sure that the +// best matching dependency belongs to the package repositories, their +// complements, recursively, or their immediate prerequisite repositories +// (only for regular dependencies). Set the buildable flag to false for the +// resolved external tests packages. Fail if unable to resolve a regular +// dependency, unless ignore_unresolved is true in which case leave this +// dependency NULL. Fail if unable to resolve an external test, unless +// ignore_unresolved or ignore_unresolved_tests is true in which case leave +// this dependency NULL, if ignore_unresolved_tests is false, and remove the +// respective tests manifest entry otherwise. Should be called once per +// internal package. // static void -resolve_dependencies (package& p, database& db) +resolve_dependencies (package& p, + database& db, + bool ignore_unresolved, + bool ignore_unresolved_tests) { using brep::dependency; + using brep::dependency_alternative; using brep::dependency_alternatives; // Resolve dependencies for internal packages only. // assert (p.internal ()); - if (p.dependencies.empty () && - p.tests.empty () && - p.examples.empty () && - p.benchmarks.empty ()) + if (p.dependencies.empty () && p.tests.empty ()) return; - auto resolve = [&p, &db] (dependency& d, bool prereq) + auto resolve = [&p, &db] (dependency& d, bool test) { // Dependency should not be resolved yet. // @@ -934,9 +1297,26 @@ resolve_dependencies (package& p, database& db) for (const auto& pp: db.query<package> (q + order_by_version_desc (vm))) { - if (find (p.internal_repository, pp, prereq)) + if (find (p.internal_repository, pp, !test)) { d.package.reset (db, pp.id); + + // If the resolved dependency is an external test, then mark it as + // such, unless it is a stub. + // + if (test) + { + shared_ptr<package> dp (d.package.load ()); + + if (!dp->stub ()) + { + dp->buildable = false; + dp->unbuildable_reason = unbuildable_reason::test; + + db.update (dp); + } + } + return true; } } @@ -944,49 +1324,50 @@ resolve_dependencies (package& p, database& db) return false; }; - auto bail = [&p] (const dependency& d, const char* what) + auto bail = [&p] (const dependency& d, const string& what) { - cerr << "error: can't resolve " << what << " " << d << " for the package " - << p.name << " " << p.version << endl + cerr << "error: can't resolve " << what << ' ' << d << " for the package " + << p.name << ' ' << p.version << endl << " info: repository " << p.internal_repository.load ()->location << " appears to be broken" << endl; throw failed (); }; - for (dependency_alternatives& da: p.dependencies) + for (dependency_alternatives& das: p.dependencies) { - for (dependency& d: da) + // Practically it is enough to resolve at least one dependency alternative + // to build a package. Meanwhile here we consider an error specifying in + // the manifest file an alternative which can't be resolved, unless + // unresolved dependencies are allowed. + // + for (dependency_alternative& da: das) { - // Practically it is enough to resolve at least one dependency - // alternative to build a package. Meanwhile here we consider an error - // specifying in the manifest file an alternative which can't be - // resolved. - // - if (!resolve (d, true /* prereq */)) - bail (d, "dependency"); + for (dependency& d: da) + { + if (!resolve (d, false /* test */) && !ignore_unresolved) + bail (d, "dependency"); + } } } - // Should we allow tests, examples, and benchmarks packages to be - // unresolvable? Let's forbid that until we see a use case for that. - // - for (dependency& d: p.tests) + for (auto i (p.tests.begin ()); i != p.tests.end (); ) { - if (!resolve (d, false /* prereq */)) - bail (d, "tests"); - } + brep::test_dependency& td (*i); - for (dependency& d: p.examples) - { - if (!resolve (d, false /* prereq */)) - bail (d, "examples"); - } + if (!resolve (td, true /* test */)) + { + if (!ignore_unresolved && !ignore_unresolved_tests) + bail (td, to_string (td.type)); - for (dependency& d: p.benchmarks) - { - if (!resolve (d, false /* prereq */)) - bail (d, "benchmarks"); + if (ignore_unresolved_tests) + { + i = p.tests.erase (i); + continue; + } + } + + ++i; } db.update (p); // Update the package state. @@ -1043,10 +1424,13 @@ detect_dependency_cycle (const package_id& id, chain.push_back (id); shared_ptr<package> p (db.load<package> (id)); - for (const auto& da: p->dependencies) + for (const auto& das: p->dependencies) { - for (const auto& d: da) - detect_dependency_cycle (d.package.object_id (), chain, db); + for (const auto& da: das) + { + for (const auto& d: da) + detect_dependency_cycle (d.package.object_id (), chain, db); + } } chain.pop_back (); @@ -1064,105 +1448,130 @@ certificate_info (const options& lo, const repository_location& rl, const optional<string>& fp) { - try - { - cstrings args { - "--cert-fingerprint", - "--cert-name", - "--cert-organization", - "--cert-email", - "-q"}; // Don't print info messages. + cstrings args { + "--cert-fingerprint", + "--cert-name", + "--cert-organization", + "--cert-email", + "-q"}; // Don't print info messages. - const char* trust ("--trust-no"); + const char* trust ("--trust-no"); - if (fp) + if (fp) + { + if (!fp->empty ()) { - if (!fp->empty ()) - { - args.push_back ("--trust"); - args.push_back (fp->c_str ()); - } - else - trust = "--trust-yes"; + args.push_back ("--trust"); + args.push_back (fp->c_str ()); + } + else + trust = "--trust-yes"; - if (!rl.remote ()) - { - args.push_back ("--auth"); - args.push_back ("all"); - } + if (!rl.remote ()) + { + args.push_back ("--auth"); + args.push_back ("all"); } + } - args.push_back (trust); + args.push_back (trust); - process pr (repository_info (lo, rl.string (), args)); + // Retry bpkg-rep-info on recoverable errors, for a while. + // + for (size_t i (0);; ++i) + { + if (i != 0) + { + // Let's follow up the bpkg's diagnostics with the number of retries + // left. + // + cerr << bpkg_retries - i + 1 << " retries left" << endl; + sleep_for (bpkg_retry_timeout); + } try { - ifdstream is ( - move (pr.in_ofd), - ifdstream::failbit | ifdstream::badbit | ifdstream::eofbit); + process pr (repository_info (lo, rl.string (), args)); - optional<certificate> cert; + try + { + ifdstream is ( + move (pr.in_ofd), + ifdstream::failbit | ifdstream::badbit | ifdstream::eofbit); - string fingerprint; - getline (is, fingerprint); + optional<certificate> cert; - if (!fingerprint.empty ()) - { - cert = certificate (); - cert->fingerprint = move (fingerprint); - getline (is, cert->name); - getline (is, cert->organization); - getline (is, cert->email); + string fingerprint; + getline (is, fingerprint); + + if (!fingerprint.empty ()) + { + cert = certificate (); + cert->fingerprint = move (fingerprint); + getline (is, cert->name); + getline (is, cert->organization); + getline (is, cert->email); + } + else + { + // Read out empty lines. + // + string s; + getline (is, s); // Name. + getline (is, s); // Organization. + getline (is, s); // Email. + } + + // Check that EOF is successfully reached. + // + is.exceptions (ifdstream::failbit | ifdstream::badbit); + if (is.peek () != ifdstream::traits_type::eof ()) + throw io_error (""); + + is.close (); + + if (pr.wait ()) + return cert; + + // Fall through. + // } - else + catch (const io_error&) { - // Read out empty lines. + // Child exit status doesn't matter. Just wait for the process + // completion and fall through. // - string s; - getline (is, s); // Name. - getline (is, s); // Organization. - getline (is, s); // Email. + pr.wait (); } - // Check that EOF is successfully reached. + // Assume the child issued diagnostics if terminated normally. // - is.exceptions (ifdstream::failbit | ifdstream::badbit); - if (is.peek () != ifdstream::traits_type::eof ()) - throw io_error (""); - - is.close (); + if (pr.exit->normal ()) + { + // Retry the certificate fetch on a recoverable error, unless the + // retries limit is reached. + // + if (pr.exit->code () == 2 && i != bpkg_retries) + continue; + } + else + cerr << "process " << lo.bpkg () << " " << *pr.exit << endl; - if (pr.wait ()) - return cert; + cerr << "error: unable to fetch certificate information for " + << rl.canonical_name () << endl; // Fall through. - // } - catch (const io_error&) + catch (const process_error& e) { - // Child exit status doesn't matter. Just wait for the process - // completion and fall through. - // - pr.wait (); - } - - // Assume the child issued diagnostics. - // - cerr << "error: unable to fetch certificate information for " - << rl.canonical_name () << endl; + cerr << "error: unable to fetch certificate information for " + << rl.canonical_name () << ": " << e << endl; - // Fall through. - } - catch (const process_error& e) - { - cerr << "error: unable to fetch certificate information for " - << rl.canonical_name () << ": " << e << endl; + // Fall through. + } - // Fall through. + throw failed (); } - - throw failed (); } int @@ -1193,7 +1602,7 @@ try << "libbbot " << LIBBBOT_VERSION_ID << endl << "libbpkg " << LIBBPKG_VERSION_ID << endl << "libbutl " << LIBBUTL_VERSION_ID << endl - << "Copyright (c) 2014-2019 Code Synthesis Ltd" << endl + << "Copyright (c) " << BREP_COPYRIGHT << "." << endl << "This is free software released under the MIT license." << endl; return 0; @@ -1241,8 +1650,46 @@ try throw failed (); } + // Verify the --service-* options. + // + if (ops.service_id_specified ()) + { + if (!ops.tenant_specified ()) + { + cerr << "error: --service-id requires --tenant" << endl; + throw failed (); + } + + if (ops.service_type ().empty ()) + { + cerr << "error: --service-id requires --service-type" + << endl; + throw failed (); + } + } + else + { + if (ops.service_type_specified ()) + { + cerr << "error: --service-type requires --service-id" + << endl; + throw failed (); + } + + if (ops.service_data_specified ()) + { + cerr << "error: --service-data requires --service-id" + << endl; + throw failed (); + } + } + // Parse and validate overrides, if specified. // + // Note that here we make sure that the overrides manifest is valid. + // Applying overrides to a specific package manifest may still fail (see + // package_manifest::validate_overrides() for details). + // manifest_name_values overrides; if (ops.overrides_file_specified ()) @@ -1277,7 +1724,7 @@ try ops.db_port (), "options='-c default_transaction_isolation=serializable'"); - // Prevent several brep-load/migrate instances from updating DB + // Prevent several brep utility instances from updating the package database // simultaneously. // database_lock l (db); @@ -1294,6 +1741,11 @@ try throw failed (); } + // Note: the interactive tenant implies private. + // + if (ops.interactive_specified ()) + ops.private_ (true); + // Load the description of all the internal repositories from the // configuration file. // @@ -1313,6 +1765,7 @@ try { db.erase_query<package> (); db.erase_query<repository> (); + db.erase_query<public_key> (); db.erase_query<tenant> (); } else // Multi-tenant mode. @@ -1325,13 +1778,39 @@ try db.erase_query<repository> ( query<repository>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<public_key> ( + query<public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<tenant> ( query<tenant>::id.in_range (ts.begin (), ts.end ())); } // Persist the tenant. // - db.persist (tenant (tnt)); + // Note that if the tenant service is specified and some tenant with the + // same service id and type is already persisted, then we will end up with + // the `object already persistent` error and terminate with the exit code + // 1 (fatal error). We could potentially dedicate a special exit code for + // such a case, so that the caller may recognize it and behave accordingly + // (CI request handler can treat it as a client error rather than an + // internal error, etc). However, let's first see if it ever becomes a + // problem. + // + optional<tenant_service> service; + + if (ops.service_id_specified ()) + service = tenant_service (ops.service_id (), + ops.service_type (), + (ops.service_data_specified () + ? ops.service_data () + : optional<string> ())); + + db.persist (tenant (tnt, + ops.private_ (), + (ops.interactive_specified () + ? ops.interactive () + : optional<string> ()), + move (service))); // On the first pass over the internal repositories we load their // certificate information and packages. @@ -1356,7 +1835,13 @@ try ir.buildable, priority++)); - load_packages (r, db, ops.ignore_unknown (), overrides); + load_packages (ops, + r, + r->cache_location, + db, + ops.ignore_unknown (), + overrides, + ops.overrides_file ().string ()); } // On the second pass over the internal repositories we load their @@ -1369,12 +1854,17 @@ try db.load<repository> ( repository_id (tnt, ir.location.canonical_name ()))); - load_repositories (r, db, ops.ignore_unknown (), ops.shallow ()); + load_repositories (ops, + r, + r->cache_location, + db, + ops.ignore_unknown (), + ops.shallow ()); } - // Resolve internal packages dependencies unless this is a shallow load. + // Resolve internal packages dependencies and, unless this is a shallow + // load, make sure there are no package dependency cycles. // - if (!ops.shallow ()) { session s; using query = query<package>; @@ -1383,16 +1873,20 @@ try db.query<package> ( query::id.tenant == tnt && query::internal_repository.canonical_name.is_not_null ())) - resolve_dependencies (p, db); + resolve_dependencies (p, + db, + ops.shallow (), + ops.ignore_unresolved_tests ()); - // Make sure there is no package dependency cycles. - // - package_ids chain; - for (const auto& p: - db.query<package> ( - query::id.tenant == tnt && - query::internal_repository.canonical_name.is_not_null ())) - detect_dependency_cycle (p.id, chain, db); + if (!ops.shallow ()) + { + package_ids chain; + for (const auto& p: + db.query<package> ( + query::id.tenant == tnt && + query::internal_repository.canonical_name.is_not_null ())) + detect_dependency_cycle (p.id, chain, db); + } } } diff --git a/load/types-parsers.cxx b/load/types-parsers.cxx index bc829f3..4c4ea9d 100644 --- a/load/types-parsers.cxx +++ b/load/types-parsers.cxx @@ -1,5 +1,4 @@ // file : load/types-parsers.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file #include <load/types-parsers.hxx> diff --git a/load/types-parsers.hxx b/load/types-parsers.hxx index de7f001..1d2a6c9 100644 --- a/load/types-parsers.hxx +++ b/load/types-parsers.hxx @@ -1,5 +1,4 @@ // file : load/types-parsers.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file // CLI parsers, included into the generated source files. |