diff options
Diffstat (limited to 'load/load.cxx')
| -rw-r--r-- | load/load.cxx | 527 |
1 files changed, 424 insertions, 103 deletions
diff --git a/load/load.cxx b/load/load.cxx index 14b8374..2b2cd56 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -20,6 +20,7 @@ #include <libbutl/pager.hxx> #include <libbutl/sha256.hxx> #include <libbutl/process.hxx> +#include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/filesystem.hxx> #include <libbutl/tab-parser.hxx> @@ -32,6 +33,7 @@ #include <libbrep/database-lock.hxx> #include <load/load-options.hxx> +#include <load/options-types.hxx> using std::cout; using std::cerr; @@ -364,7 +366,8 @@ repository_info (const options& lo, const string& rl, const cstrings& options) // the repository. Should be called once per repository. // static void -load_packages (const shared_ptr<repository>& rp, +load_packages (const options& lo, + const shared_ptr<repository>& rp, const repository_location& cl, database& db, bool ignore_unknown, @@ -421,10 +424,12 @@ load_packages (const shared_ptr<repository>& rp, using brep::dependency_alternative; using brep::dependency_alternatives; + const string& tenant (rp->tenant); + for (package_manifest& pm: pms) { shared_ptr<package> p ( - db.find<package> (package_id (rp->tenant, pm.name, pm.version))); + db.find<package> (package_id (tenant, pm.name, pm.version))); // sha256sum should always be present if the package manifest comes from // the packages.manifest file belonging to the pkg repository. @@ -433,21 +438,49 @@ load_packages (const shared_ptr<repository>& rp, if (p == nullptr) { - if (rp->internal) + // Apply the package manifest overrides. + // + if (!overrides.empty ()) + try { - if (!overrides.empty ()) - try - { - pm.override (overrides, overrides_name); - } - catch (const manifest_parsing& e) - { - cerr << "error: unable to override " << p << " manifest: " << e - << endl; + pm.override (overrides, overrides_name); + } + catch (const manifest_parsing& e) + { + cerr << "error: unable to override " << pm.name << ' ' << pm.version + << " manifest: " << e << endl; - throw failed (); + throw failed (); + } + + // Convert the package manifest build configurations (contain public + // keys data) into the brep's build package configurations (contain + // public key object lazy pointers). Keep the bot key lists empty if + // the package is not buildable. + // + package_build_configs build_configs; + + if (!pm.build_configs.empty ()) + { + build_configs.reserve (pm.build_configs.size ()); + + for (bpkg::build_package_config& c: pm.build_configs) + { + build_configs.emplace_back (move (c.name), + move (c.arguments), + move (c.comment), + move (c.builds), + move (c.constraints), + move (c.auxiliaries), + package_build_bot_keys (), + move (c.email), + move (c.warning_email), + move (c.error_email)); } + } + if (rp->internal) + { // Create internal package object. // // Return nullopt if the text is in a file (can happen if the @@ -594,6 +627,107 @@ load_packages (const shared_ptr<repository>& rp, // package_name project (pm.effective_project ()); + // If the package is buildable, then save the package manifest's + // common and build configuration-specific bot keys into the database + // and translate the key data lists into the lists of the public key + // object lazy pointers. + // + package_build_bot_keys bot_keys; + + if (rp->buildable) + { + // Save the specified bot keys into the database as public key + // objects, unless they are already persisted. Translate these keys + // into the public key object lazy pointers. + // + auto keys_to_objects = [&lo, + &pm, + &tenant, + &db] (strings&& keys) + { + package_build_bot_keys r; + + if (keys.empty ()) + return r; + + r.reserve (keys.size ()); + + for (string& key: keys) + { + // Calculate the key fingerprint. + // + string fp; + + try + { + openssl os (path ("-"), path ("-"), 2, + lo.openssl (), + "pkey", + lo.openssl_option (), "-pubin", "-outform", "DER"); + + os.out << key; + os.out.close (); + + fp = sha256 (os.in).string (); + os.in.close (); + + if (!os.wait ()) + { + cerr << "process " << lo.openssl () << ' ' << *os.exit + << endl; + + throw io_error (""); + } + } + catch (const io_error&) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << endl + << " info: key:" << endl + << key << endl; + + throw failed (); + } + catch (const process_error& e) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << ": " + << e << endl; + + throw failed (); + } + + // Try to find the public_key object for the calculated + // fingerprint. If it doesn't exist, then create and persist the + // new object. + // + public_key_id id (tenant, move (fp)); + shared_ptr<public_key> k (db.find<public_key> (id)); + + if (k == nullptr) + { + k = make_shared<public_key> (move (id.tenant), + move (id.fingerprint), + move (key)); + + db.persist (k); + } + + r.push_back (move (k)); + } + + return r; + }; + + bot_keys = keys_to_objects (move (pm.build_bot_keys)); + + assert (build_configs.size () == pm.build_configs.size ()); + + for (size_t i (0); i != build_configs.size (); ++i) + build_configs[i].bot_keys = + keys_to_objects (move (pm.build_configs[i].bot_keys)); + } + p = make_shared<package> ( move (pm.name), move (pm.version), @@ -622,7 +756,8 @@ load_packages (const shared_ptr<repository>& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (bot_keys), + move (build_configs), move (pm.location), move (pm.fragment), move (pm.sha256sum), @@ -636,7 +771,7 @@ load_packages (const shared_ptr<repository>& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (build_configs), rp); db.persist (p); @@ -670,8 +805,8 @@ load_packages (const shared_ptr<repository>& rp, } // A non-stub package is buildable if belongs to at least one - // buildable repository (see libbrep/package.hxx for details). - // Note that if this is an external test package it will be marked as + // buildable repository (see libbrep/package.hxx for details). Note + // that if this is an external test package it will be marked as // unbuildable later (see resolve_dependencies() for details). // if (rp->buildable && !p->buildable && !p->stub ()) @@ -1018,7 +1153,8 @@ load_repositories (const options& lo, // We don't apply overrides to the external packages. // - load_packages (pr, + load_packages (lo, + pr, !pr->cache_location.empty () ? pr->cache_location : cl, db, ignore_unknown, @@ -1071,35 +1207,25 @@ find (const lazy_shared_ptr<repository>& r, return false; } -// Resolve package regular dependencies and external tests. Make sure that the -// best matching dependency belongs to the package repositories, their -// complements, recursively, or their immediate prerequisite repositories -// (only for regular dependencies). Set the buildable flag to false for the -// resolved external tests packages. Fail if unable to resolve a regular -// dependency, unless ignore_unresolved is true in which case leave this -// dependency NULL. Fail if unable to resolve an external test, unless -// ignore_unresolved or ignore_unresolved_tests is true in which case leave -// this dependency NULL, if ignore_unresolved_tests is false, and remove the -// respective tests manifest entry otherwise. Should be called once per -// internal package. +// Try to resolve package regular dependencies and external tests. Make sure +// that the best matching dependency belongs to the package repositories, +// their complements, recursively, or their immediate prerequisite +// repositories (only for regular dependencies). Set the buildable flag to +// false for the resolved external tests packages. Leave the package member +// NULL for unresolved dependencies. // static void -resolve_dependencies (package& p, - database& db, - bool ignore_unresolved, - bool ignore_unresolved_tests) +resolve_dependencies (package& p, database& db) { using brep::dependency; using brep::dependency_alternative; using brep::dependency_alternatives; + using brep::test_dependency; // Resolve dependencies for internal packages only. // assert (p.internal ()); - if (p.dependencies.empty () && p.tests.empty ()) - return; - auto resolve = [&p, &db] (dependency& d, bool test) { // Dependency should not be resolved yet. @@ -1189,6 +1315,60 @@ resolve_dependencies (package& p, return false; }; + // Update the package state if any dependency is resolved. + // + bool update (false); + + for (dependency_alternatives& das: p.dependencies) + { + for (dependency_alternative& da: das) + { + for (dependency& d: da) + { + if (resolve (d, false /* test */)) + update = true; + } + } + } + + for (test_dependency& td: p.tests) + { + if (resolve (td, true /* test */)) + update = true; + } + + if (update) + db.update (p); +} + +// Verify that the unresolved dependencies can be ignored. +// +// Specifically, fail for an unresolved regular dependency, unless +// ignore_unresolved is true or this is a conditional dependency and either +// ignore_unresolved_cond argument is 'all' or it is 'tests' and the specified +// package is a tests, examples, or benchmarks package. Fail for an unresolved +// external test, unless ignore_unresolved or ignore_unresolved_tests is +// true. If ignore_unresolved_tests is true, then remove the unresolved tests +// entry from the package manifest. Should be called once per internal package +// after resolve_dependencies() is called for all of them. +// +static void +verify_dependencies ( + package& p, + database& db, + bool ignore_unresolved, + bool ignore_unresolved_tests, + optional<ignore_unresolved_conditional_dependencies> ignore_unresolved_cond) +{ + using brep::dependency; + using brep::dependency_alternative; + using brep::dependency_alternatives; + using brep::test_dependency; + + // Verify dependencies for internal packages only. + // + assert (p.internal ()); + auto bail = [&p] (const dependency& d, const string& what) { cerr << "error: can't resolve " << what << ' ' << d << " for the package " @@ -1199,43 +1379,74 @@ resolve_dependencies (package& p, throw failed (); }; - for (dependency_alternatives& das: p.dependencies) + if (!ignore_unresolved) { - // Practically it is enough to resolve at least one dependency alternative - // to build a package. Meanwhile here we consider an error specifying in - // the manifest file an alternative which can't be resolved, unless - // unresolved dependencies are allowed. + // There must always be a reason why a package is not buildable. // - for (dependency_alternative& da: das) + assert (p.buildable || p.unbuildable_reason); + + bool test (!p.buildable && + *p.unbuildable_reason == unbuildable_reason::test); + + for (dependency_alternatives& das: p.dependencies) { - for (dependency& d: da) + for (dependency_alternative& da: das) { - if (!resolve (d, false /* test */) && !ignore_unresolved) - bail (d, "dependency"); + for (dependency& d: da) + { + if (d.package == nullptr) + { + if (da.enable && ignore_unresolved_cond) + { + switch (*ignore_unresolved_cond) + { + case ignore_unresolved_conditional_dependencies::all: continue; + case ignore_unresolved_conditional_dependencies::tests: + { + if (test) + continue; + + break; + } + } + } + + bail (d, "dependency"); + } + } } } } - for (auto i (p.tests.begin ()); i != p.tests.end (); ) + if (!ignore_unresolved || ignore_unresolved_tests) { - brep::test_dependency& td (*i); + // Update the package state if any test dependency is erased. + // + bool update (false); - if (!resolve (td, true /* test */)) + for (auto i (p.tests.begin ()); i != p.tests.end (); ) { - if (!ignore_unresolved && !ignore_unresolved_tests) - bail (td, to_string (td.type)); + test_dependency& td (*i); - if (ignore_unresolved_tests) + if (td.package == nullptr) { - i = p.tests.erase (i); - continue; + if (!ignore_unresolved && !ignore_unresolved_tests) + bail (td, to_string (td.type)); + + if (ignore_unresolved_tests) + { + i = p.tests.erase (i); + update = true; + continue; + } } + + ++i; } - ++i; + if (update) + db.update (p); } - - db.update (p); // Update the package state. } using package_ids = vector<package_id>; @@ -1294,7 +1505,12 @@ detect_dependency_cycle (const package_id& id, for (const auto& da: das) { for (const auto& d: da) - detect_dependency_cycle (d.package.object_id (), chain, db); + { + // Skip unresolved dependencies. + // + if (d.package != nullptr) + detect_dependency_cycle (d.package.object_id (), chain, db); + } } } @@ -1508,11 +1724,23 @@ try // const string& tnt (ops.tenant ()); - if (ops.tenant_specified () && tnt.empty ()) + if (ops.tenant_specified ()) { - cerr << "error: empty tenant" << endl - << help_info << endl; - throw failed (); + if (tnt.empty ()) + { + cerr << "error: empty tenant" << endl + << help_info << endl; + throw failed (); + } + } + else + { + if (ops.existing_tenant ()) + { + cerr << "error: --existing-tenant requires --tenant" << endl + << help_info << endl; + throw failed (); + } } // Verify the --service-* options. @@ -1521,14 +1749,15 @@ try { if (!ops.tenant_specified ()) { - cerr << "error: --service-id requires --tenant" << endl; + cerr << "error: --service-id requires --tenant" << endl + << help_info << endl; throw failed (); } if (ops.service_type ().empty ()) { - cerr << "error: --service-id requires --service-type" - << endl; + cerr << "error: --service-id requires --service-type" << endl + << help_info << endl; throw failed (); } } @@ -1536,15 +1765,15 @@ try { if (ops.service_type_specified ()) { - cerr << "error: --service-type requires --service-id" - << endl; + cerr << "error: --service-type requires --service-id" << endl + << help_info << endl; throw failed (); } if (ops.service_data_specified ()) { - cerr << "error: --service-data requires --service-id" - << endl; + cerr << "error: --service-data requires --service-id" << endl + << help_info << endl; throw failed (); } } @@ -1618,23 +1847,68 @@ try if (ops.force () || changed (tnt, irs, db)) { + shared_ptr<tenant> t; // Not NULL in the --existing-tenant mode. + // Rebuild repositories persistent state from scratch. // // Note that in the single-tenant mode the tenant must be empty. In the - // multi-tenant mode all tenants must be non-empty. So in the - // single-tenant mode we erase all database objects (possibly from - // multiple tenants). Otherwise, cleanup the specified and the empty - // tenants only. + // multi-tenant mode all tenants, excluding the pre-created ones, must be + // non-empty. So in the single-tenant mode we erase all database objects + // (possibly from multiple tenants). Otherwise, cleanup the empty tenant + // and, unless in the --existing-tenant mode, the specified one. // if (tnt.empty ()) // Single-tenant mode. { db.erase_query<package> (); db.erase_query<repository> (); + db.erase_query<public_key> (); db.erase_query<tenant> (); } else // Multi-tenant mode. { - cstrings ts ({tnt.c_str (), ""}); + // NOTE: don't forget to update ci_start::create() if changing anything + // here. + // + cstrings ts ({""}); + + // In the --existing-tenant mode make sure that the specified tenant + // exists, is not archived, not marked as unloaded, and is + // empty. Otherwise (not in the --existing-tenant mode), remove this + // tenant. + // + if (ops.existing_tenant ()) + { + t = db.find<tenant> (tnt); + + if (t == nullptr) + { + cerr << "error: unable to find tenant " << tnt << endl; + throw failed (); + } + + if (t->archived) + { + cerr << "error: tenant " << tnt << " is archived" << endl; + throw failed (); + } + + if (t->unloaded_timestamp) + { + cerr << "error: tenant " << tnt << " is marked as unloaded" << endl; + throw failed (); + } + + size_t n (db.query_value<repository_count> ( + query<repository_count>::id.tenant == tnt)); + + if (n != 0) + { + cerr << "error: tenant " << tnt << " is not empty" << endl; + throw failed (); + } + } + else + ts.push_back (tnt.c_str ()); db.erase_query<package> ( query<package>::id.tenant.in_range (ts.begin (), ts.end ())); @@ -1642,36 +1916,75 @@ try db.erase_query<repository> ( query<repository>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<public_key> ( + query<public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<tenant> ( query<tenant>::id.in_range (ts.begin (), ts.end ())); } - // Persist the tenant. + // Craft the tenant service object from the --service-* options. // - // Note that if the tenant service is specified and some tenant with the - // same service id and type is already persisted, then we will end up with - // the `object already persistent` error and terminate with the exit code - // 1 (fatal error). We could potentially dedicate a special exit code for - // such a case, so that the caller may recognize it and behave accordingly - // (CI request handler can treat it as a client error rather than an - // internal error, etc). However, let's first see if it ever becomes a - // problem. + // In the --existing-tenant mode make sure that the specified service + // matches the service associated with the pre-created tenant and update + // the service data, if specified. // optional<tenant_service> service; if (ops.service_id_specified ()) + { service = tenant_service (ops.service_id (), ops.service_type (), (ops.service_data_specified () ? ops.service_data () : optional<string> ())); - db.persist (tenant (tnt, - ops.private_ (), - (ops.interactive_specified () - ? ops.interactive () - : optional<string> ()), - move (service))); + if (ops.existing_tenant ()) + { + assert (t != nullptr); + + if (!t->service) + { + cerr << "error: no service associated with tenant " << tnt << endl; + throw failed (); + } + + if (t->service->id != service->id || t->service->type != service->type) + { + cerr << "error: associated service mismatch for tenant " << tnt << endl << + " info: specified service: " << service->id << ' ' + << service->type << endl << + " info: associated service: " << t->service->id << ' ' + << t->service->type << endl; + throw failed (); + } + + if (service->data) + { + t->service->data = move (service->data); + db.update (t); + } + } + } + + // Persist the tenant. + // + // Note that if the tenant service is specified and some tenant with the + // same service id and type is already persisted, then we will end up with + // the `object already persistent` error and terminate with the exit code + // 1 (fatal error). We could potentially dedicate a special exit code for + // such a case, so that the caller may recognize it and behave accordingly + // (CI request handler can treat it as a client error rather than an + // internal error, etc). However, let's first see if it ever becomes a + // problem. + // + if (!ops.existing_tenant ()) + db.persist (tenant (tnt, + ops.private_ (), + (ops.interactive_specified () + ? ops.interactive () + : optional<string> ()), + move (service))); // On the first pass over the internal repositories we load their // certificate information and packages. @@ -1696,7 +2009,8 @@ try ir.buildable, priority++)); - load_packages (r, + load_packages (ops, + r, r->cache_location, db, ops.ignore_unknown (), @@ -1722,29 +2036,36 @@ try ops.shallow ()); } - // Resolve internal packages dependencies and, unless this is a shallow - // load, make sure there are no package dependency cycles. + // Try to resolve the internal packages dependencies and verify that the + // unresolved ones can be ignored. Unless this is a shallow load, make + // sure there are no package dependency cycles. // { session s; using query = query<package>; - for (auto& p: - db.query<package> ( - query::id.tenant == tnt && - query::internal_repository.canonical_name.is_not_null ())) - resolve_dependencies (p, - db, - ops.shallow (), - ops.ignore_unresolved_tests ()); + query q (query::id.tenant == tnt && + query::internal_repository.canonical_name.is_not_null ()); + + for (auto& p: db.query<package> (q)) + resolve_dependencies (p, db); + + for (auto& p: db.query<package> (q)) + { + verify_dependencies ( + p, + db, + ops.shallow (), + ops.ignore_unresolv_tests (), + (ops.ignore_unresolv_cond_specified () + ? ops.ignore_unresolv_cond () + : optional<ignore_unresolved_conditional_dependencies> ())); + } if (!ops.shallow ()) { package_ids chain; - for (const auto& p: - db.query<package> ( - query::id.tenant == tnt && - query::internal_repository.canonical_name.is_not_null ())) + for (const auto& p: db.query<package> (q)) detect_dependency_cycle (p.id, chain, db); } } |