From 7c61322166eb0eab77ee5fb10031bae616ecb192 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 15 Apr 2024 21:36:02 +0300 Subject: Add support for custom build bots --- clean/clean.cxx | 7 +- libbrep/build-extra.sql | 45 ++++++++- libbrep/build-package.hxx | 110 +++++++++++++++++---- libbrep/build.hxx | 2 +- libbrep/build.xml | 2 + libbrep/common.cxx | 10 -- libbrep/common.hxx | 191 +++++++++++++++++++++++++++--------- libbrep/odb.sh | 4 +- libbrep/package.cxx | 31 +++++- libbrep/package.hxx | 126 +++++++++++++++++++++--- libbrep/package.xml | 135 +++++++++++++++++++++++++ load/load.cli | 16 +++ load/load.cxx | 150 ++++++++++++++++++++++++++-- mod/build-config-module.hxx | 3 +- mod/build-result-module.cxx | 145 +++++++++++++++++++-------- mod/build-target-config.cxx | 8 +- mod/build-target-config.hxx | 23 ++++- mod/mod-build-task.cxx | 167 +++++++++++++++++++++++-------- mod/mod-package-version-details.cxx | 8 +- monitor/monitor.cxx | 93 ++++++++++++++++-- 20 files changed, 1067 insertions(+), 209 deletions(-) diff --git a/clean/clean.cxx b/clean/clean.cxx index 55fb59b..828ae4b 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -480,8 +480,8 @@ namespace brep auto tenant_ids (pq.execute ()); if ((ne = !tenant_ids.empty ())) { - // Cache tenant ids and erase packages, repositories, and tenants at - // once. + // Cache tenant ids and erase packages, repositories, public keys, and + // tenants at once. // strings tids; tids.reserve (tenant_ids.size ()); @@ -497,6 +497,9 @@ namespace brep db.erase_query ( query::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query ( + query::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query ( query::id.in_range (tids.begin (), tids.end ())); } diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index a931f31..9e51a51 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -6,6 +6,8 @@ -- package-extra.sql file for details. -- +DROP FOREIGN TABLE IF EXISTS build_package_config_bot_keys; + DROP FOREIGN TABLE IF EXISTS build_package_config_auxiliaries; DROP FOREIGN TABLE IF EXISTS build_package_config_constraints; @@ -14,6 +16,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_config_builds; DROP FOREIGN TABLE IF EXISTS build_package_configs; +DROP FOREIGN TABLE IF EXISTS build_package_bot_keys; + DROP FOREIGN TABLE IF EXISTS build_package_auxiliaries; DROP FOREIGN TABLE IF EXISTS build_package_constraints; @@ -30,6 +34,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_requirements; DROP FOREIGN TABLE IF EXISTS build_package; +DROP FOREIGN TABLE IF EXISTS build_public_key; + DROP FOREIGN TABLE IF EXISTS build_repository; DROP FOREIGN TABLE IF EXISTS build_tenant; @@ -64,6 +70,14 @@ CREATE FOREIGN TABLE build_repository ( certificate_fingerprint TEXT NULL) SERVER package_server OPTIONS (table_name 'repository'); +-- The foreign table for build_public_key object. +-- +CREATE FOREIGN TABLE build_public_key ( + tenant TEXT NOT NULL, + fingerprint TEXT NOT NULL, + "data" TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'public_key'); + -- The foreign table for build_package object. -- CREATE FOREIGN TABLE build_package ( @@ -84,7 +98,8 @@ CREATE FOREIGN TABLE build_package ( build_error_email_comment TEXT NULL, internal_repository_tenant TEXT NULL, internal_repository_canonical_name TEXT NULL, - buildable BOOLEAN NOT NULL) + buildable BOOLEAN NOT NULL, + custom_bot BOOLEAN NULL) SERVER package_server OPTIONS (table_name 'package'); -- The foreign tables for the build_package object requirements member (that @@ -214,6 +229,21 @@ CREATE FOREIGN TABLE build_package_auxiliaries ( comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_auxiliaries'); +-- The foreign table for the build_package object bot_keys member (that is +-- of a container type). +-- +CREATE FOREIGN TABLE build_package_bot_keys ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + key_tenant TEXT NOT NULL, + key_fingerprint TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_bot_keys'); + -- The foreign tables for the build_package object configs member (that is a -- container of values containing containers. -- @@ -277,3 +307,16 @@ CREATE FOREIGN TABLE build_package_config_auxiliaries ( config TEXT NOT NULL, comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_config_auxiliaries'); + +CREATE FOREIGN TABLE build_package_config_bot_keys ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + config_index BIGINT NOT NULL, + index BIGINT NOT NULL, + key_tenant TEXT NOT NULL, + key_fingerprint TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_config_bot_keys'); diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 08fb781..b7bea87 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -23,9 +23,11 @@ namespace brep // // The mapping is established in build-extra.sql. We also explicitly mark // non-primary key foreign-mapped members in the source object. - // + // Foreign object that is mapped to a subset of the tenant object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_tenant") pointer(shared_ptr) class build_tenant { @@ -50,6 +52,8 @@ namespace brep // Foreign object that is mapped to a subset of the repository object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_repository") pointer(shared_ptr) readonly class build_repository { @@ -75,6 +79,54 @@ namespace brep build_repository (): canonical_name (id.canonical_name) {} }; + // Foreign object that is mapped to a subset of the public key object. + // + // Note: table created manually thus assign table name explicitly. + // + #pragma db object table("build_public_key") pointer(shared_ptr) readonly + class build_public_key: public string + { + public: + public_key_id id; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(data) virtual(string) access(this) + + private: + friend class odb::access; + build_public_key () = default; + }; + + // build_package_config + // + using build_package_config = + build_package_config_template>; + + using build_package_configs = + build_package_configs_template>; + + #pragma db value(build_package_config) definition + + #pragma db member(build_package_config::builds) transient + #pragma db member(build_package_config::constraints) transient + #pragma db member(build_package_config::auxiliaries) transient + #pragma db member(build_package_config::bot_keys) transient + + // build_package_bot_keys + // + using build_package_bot_keys = vector>; + using build_package_bot_key_key = odb::nested_key; + + using build_package_bot_keys_map = + std::map>; + + #pragma db value(build_package_bot_key_key) + #pragma db member(build_package_bot_key_key::outer) column("config_index") + #pragma db member(build_package_bot_key_key::inner) column("index") + // Forward declarations. // class build_package; @@ -107,6 +159,8 @@ namespace brep // Foreign object that is mapped to a subset of the package object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_package") pointer(shared_ptr) readonly session class build_package { @@ -132,21 +186,26 @@ namespace brep lazy_shared_ptr internal_repository; bool buildable; + optional custom_bot; // Mapped to the package object builds, build_constraints, - // build_auxiliaries, and build_configs members using the PostgreSQL - // foreign table mechanism. + // build_auxiliaries, bot_keys, and build_configs members using the + // PostgreSQL foreign table mechanism. // - build_class_exprs builds; - build_constraints constraints; - build_auxiliaries auxiliaries; - build_package_configs configs; - - // Group the builds and constraints members of this object as well as of - // the nested configs entries for an explicit load. Note that the configs - // top-level members are loaded implicitly. + build_class_exprs builds; + build_constraints constraints; + build_auxiliaries auxiliaries; + build_package_bot_keys bot_keys; + build_package_configs configs; + + // Group the builds/constraints, auxiliaries, and bot_keys members of this + // object together with their respective nested configs entries into the + // separate sections for an explicit load. Note that the configs top-level + // members are loaded implicitly. // odb::section constraints_section; + odb::section auxiliaries_section; + odb::section bot_keys_section; bool internal () const noexcept {return internal_repository != nullptr;} @@ -194,7 +253,7 @@ namespace brep #pragma db member(requirements_tests_section) load(lazy) update(always) - // builds, constraints, and auxiliaries + // builds, constraints, auxiliaries, and bot_keys // #pragma db member(builds) id_column("") value_column("") \ section(constraints_section) @@ -203,13 +262,16 @@ namespace brep section(constraints_section) #pragma db member(auxiliaries) id_column("") value_column("") \ - section(constraints_section) + section(auxiliaries_section) + + #pragma db member(bot_keys) id_column("") value_column("key_") \ + section(bot_keys_section) // configs // - // Note that build_package_config::{builds,constraints,auxiliaries} are - // persisted/loaded via the separate nested containers (see commons.hxx - // for details). + // Note that build_package_config::{builds,constraints,auxiliaries,bot_keys} + // are persisted/loaded via the separate nested containers (see + // commons.hxx for details). // #pragma db member(configs) id_column("") value_column("config_") @@ -244,9 +306,23 @@ namespace brep odb::nested_set (as, std::move (?)); \ move (as).to_configs (this.configs)) \ id_column("") key_column("") value_column("") \ - section(constraints_section) + section(auxiliaries_section) + + #pragma db member(config_bot_keys) \ + virtual(build_package_bot_keys_map) \ + after(config_auxiliaries) \ + get(odb::nested_get ( \ + brep::build_package_config_bot_keys (this.configs))) \ + set(brep::build_package_config_bot_keys< \ + lazy_shared_ptr> bks; \ + odb::nested_set (bks, std::move (?)); \ + move (bks).to_configs (this.configs)) \ + id_column("") key_column("") value_column("key_") \ + section(bot_keys_section) #pragma db member(constraints_section) load(lazy) update(always) + #pragma db member(auxiliaries_section) load(lazy) update(always) + #pragma db member(bot_keys_section) load(lazy) update(always) private: friend class odb::access; diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 4c470cd..af49c03 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -28,7 +28,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 26, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 27, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace diff --git a/libbrep/build.xml b/libbrep/build.xml index 815c915..1eba85a 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ + + diff --git a/libbrep/common.cxx b/libbrep/common.cxx index c97a346..4f729a3 100644 --- a/libbrep/common.cxx +++ b/libbrep/common.cxx @@ -32,14 +32,4 @@ namespace brep else if (r == "unbuildable") return unbuildable_reason::unbuildable; else throw invalid_argument ("invalid unbuildable reason '" + r + '\''); } - - build_package_config* - find (const string& name, build_package_configs& cs) - { - auto i (find_if (cs.begin (), cs.end (), - [&name] (const build_package_config& c) - {return c.name == name;})); - - return i != cs.end () ? &*i : nullptr; - } } diff --git a/libbrep/common.hxx b/libbrep/common.hxx index ea18fd4..1433c8c 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -326,6 +326,19 @@ namespace brep : tenant (move (t)), canonical_name (move (n)) {} }; + // public_key_id + // + #pragma db value + struct public_key_id + { + string tenant; + string fingerprint; + + public_key_id () = default; + public_key_id (string t, string f) + : tenant (move (t)), fingerprint (move (f)) {} + }; + // build_class_expr // using bpkg::build_class_expr; @@ -370,31 +383,38 @@ namespace brep #pragma db value(email) definition #pragma db member(email::value) virtual(string) before access(this) column("") - // build_package_config + // build_package_config_template // - using build_package_config = bpkg::build_package_config; - - #pragma db value(build_package_config) definition + using bpkg::build_package_config_template; // 1 for the default configuration which is always present. // - using build_package_configs = small_vector; + template + using build_package_configs_template = + small_vector, 1>; // Return the address of the configuration object with the specified name, // if present, and NULL otherwise. // - build_package_config* - find (const string& name, build_package_configs&); + template + inline build_package_config_template* + find (const string& name, build_package_configs_template& cs) + { + auto i (find_if (cs.begin (), cs.end (), + [&name] (const build_package_config_template& c) + {return c.name == name;})); + + return i != cs.end () ? &*i : nullptr; + } // Note that ODB doesn't support containers of value types which contain // containers. Thus, we will persist/load - // package_build_config::{builds,constraint,auxiliaries} via the separate - // nested containers using the adapter classes. - // - // build_package_config::builds + // build_package_config_template::{builds,constraint,auxiliaries,bot_keys} + // via the separate nested containers using the adapter classes. // - #pragma db member(build_package_config::builds) transient + // build_package_config_template::builds + // using build_class_expr_key = odb::nested_key; using build_class_exprs_map = std::map; @@ -402,24 +422,27 @@ namespace brep #pragma db member(build_class_expr_key::outer) column("config_index") #pragma db member(build_class_expr_key::inner) column("index") - // Adapter for build_package_config::builds. + // Adapter for build_package_config_template::builds. + // + // Note: 1 as for build_package_configs_template. // - class build_package_config_builds: - public small_vector // 1 as for build_package_configs. + class build_package_config_builds: public small_vector { public: build_package_config_builds () = default; + template explicit - build_package_config_builds (const build_package_configs& cs) + build_package_config_builds (const build_package_configs_template& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template& c: cs) push_back (c.builds); } + template void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -432,10 +455,8 @@ namespace brep } }; - // build_package_config::constraints + // build_package_config_template::constraints // - #pragma db member(build_package_config::constraints) transient - using build_constraint_key = odb::nested_key; using build_constraints_map = std::map; @@ -443,24 +464,29 @@ namespace brep #pragma db member(build_constraint_key::outer) column("config_index") #pragma db member(build_constraint_key::inner) column("index") - // Adapter for build_package_config::constraints. + // Adapter for build_package_config_template::constraints. + // + // Note: 1 as for build_package_configs_template. // class build_package_config_constraints: - public small_vector // 1 as for build_package_configs. + public small_vector { public: build_package_config_constraints () = default; + template explicit - build_package_config_constraints (const build_package_configs& cs) + build_package_config_constraints ( + const build_package_configs_template& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template& c: cs) push_back (c.constraints); } + template void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -473,10 +499,8 @@ namespace brep } }; - // build_package_config::auxiliaries + // build_package_config_template::auxiliaries // - #pragma db member(build_package_config::auxiliaries) transient - using build_auxiliary_key = odb::nested_key; using build_auxiliaries_map = std::map; @@ -484,24 +508,29 @@ namespace brep #pragma db member(build_auxiliary_key::outer) column("config_index") #pragma db member(build_auxiliary_key::inner) column("index") - // Adapter for build_package_config::auxiliaries. + // Adapter for build_package_config_template::auxiliaries. + // + // Note: 1 as for build_package_configs_template. // class build_package_config_auxiliaries: - public small_vector // 1 as for build_package_configs. + public small_vector { public: build_package_config_auxiliaries () = default; + template explicit - build_package_config_auxiliaries (const build_package_configs& cs) + build_package_config_auxiliaries ( + const build_package_configs_template& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template& c: cs) push_back (c.auxiliaries); } + template void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -514,6 +543,40 @@ namespace brep } }; + // build_package_config_template::bot_keys + // + // Adapter for build_package_config_template::bot_keys. + // + // Note: 1 as for build_package_configs_template. + // + template + class build_package_config_bot_keys: public small_vector, 1> + { + public: + build_package_config_bot_keys () = default; + + explicit + build_package_config_bot_keys (const build_package_configs_template& cs) + { + this->reserve (cs.size ()); + for (const build_package_config_template& c: cs) + this->push_back (c.bot_keys); + } + + void + to_configs (build_package_configs_template& cs) && + { + // Note that the empty trailing entries will be missing (see ODB's + // nested-container.hxx for details). + // + assert (this->size () <= cs.size ()); + + auto i (cs.begin ()); + for (vector& bks: *this) + i++->bot_keys = move (bks); + } + }; + // The primary reason why a package is unbuildable by the build bot // controller service. // @@ -611,13 +674,12 @@ namespace brep // Version comparison operators. // - // They allow comparing objects that have epoch, canonical_upstream, - // canonical_release, and revision data members. The idea is that this - // works for both query members of types version and canonical_version. - // Note, though, that the object revisions should be comparable (both - // optional, numeric, etc), so to compare version to query member or - // canonical_version you may need to explicitly convert the version object - // to canonical_version. + // Compare objects that have epoch, canonical_upstream, canonical_release, + // and revision data members. The idea is that this works for both query + // members of types version and canonical_version. Note, though, that the + // object revisions should be comparable (both optional, numeric, etc), so + // to compare version to query member or canonical_version you may need to + // explicitly convert the version object to canonical_version. // template inline auto @@ -769,10 +831,9 @@ namespace brep return compare_version_lt (x.version, y.version, true); } - // They allow comparing objects that have tenant, name, and version data - // members. The idea is that this works for both query members of package id - // types (in particular in join conditions) as well as for values of - // package_id type. + // Compare objects that have tenant, name, and version data members. The + // idea is that this works for both query members of package id types (in + // particular in join conditions) as well as for values of package_id type. // template inline auto @@ -852,10 +913,10 @@ namespace brep return x.canonical_name.compare (y.canonical_name) < 0; } - // They allow comparing objects that have tenant and canonical_name data - // members. The idea is that this works for both query members of repository - // id types (in particular in join conditions) as well as for values of - // repository_id type. + // Compare objects that have tenant and canonical_name data members. The + // idea is that this works for both query members of repository id types (in + // particular in join conditions) as well as for values of repository_id + // type. // template inline auto @@ -872,6 +933,38 @@ namespace brep { return x.tenant != y.tenant || x.canonical_name != y.canonical_name; } + + // Public key id comparison operators. + // + inline bool + operator< (const public_key_id& x, const public_key_id& y) + { + if (int r = x.tenant.compare (y.tenant)) + return r < 0; + + return x.fingerprint.compare (y.fingerprint) < 0; + } + + // Compare objects that have tenant and fingerprint data members. The idea + // is that this works for both query members of public key id types (in + // particular in join conditions) as well as for values of public_key_id + // type. + // + template + inline auto + operator== (const T1& x, const T2& y) + -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint) + { + return x.tenant == y.tenant && x.fingerprint == y.fingerprint; + } + + template + inline auto + operator!= (const T1& x, const T2& y) + -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint) + { + return x.tenant != y.tenant || x.fingerprint != y.fingerprint; + } } #endif // LIBBREP_COMMON_HXX diff --git a/libbrep/odb.sh b/libbrep/odb.sh index 9ee11fa..608ca41 100755 --- a/libbrep/odb.sh +++ b/libbrep/odb.sh @@ -53,7 +53,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query \ --hxx-prologue '#include ' \ -DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \ --include-with-brackets --include-prefix libbrep \ - --guard-prefix LIBBREP \ + --guard-prefix LIBBREP \ common.hxx $odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \ @@ -74,7 +74,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \ --odb-epilogue '#include ' \ --generate-prepared -DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \ --include-with-brackets --include-prefix libbrep \ - --guard-prefix LIBBREP \ + --guard-prefix LIBBREP \ build.hxx $odb "${inc[@]}" -d pgsql --std c++14 --generate-query \ diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 37795f0..4eb6fe8 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -82,7 +82,8 @@ namespace brep build_class_exprs bs, build_constraints_type bc, build_auxiliaries_type ac, - build_package_configs bcs, + package_build_bot_keys bk, + package_build_configs bcs, optional lc, optional fr, optional sh, @@ -116,6 +117,7 @@ namespace brep builds (move (bs)), build_constraints (move (bc)), build_auxiliaries (move (ac)), + build_bot_keys (move (bk)), build_configs (move (bcs)), internal_repository (move (rp)), location (move (lc)), @@ -134,6 +136,31 @@ namespace brep buildable = !unbuildable_reason; + // If the package is buildable deduce the custom_bot flag. + // + if (buildable) + { + for (const package_build_config& bc: build_configs) + { + bool custom (!bc.effective_bot_keys (build_bot_keys).empty ()); + + if (!custom_bot) + { + custom_bot = custom; + } + // + // If both the custom and default bots are used by the package, then + // reset the custom_bot flag to nullopt and bail out from the build + // package configurations loop. + // + else if (*custom_bot != custom) + { + custom_bot = nullopt; + break; + } + } + } + assert (internal_repository->internal); } @@ -143,7 +170,7 @@ namespace brep build_class_exprs bs, build_constraints_type bc, build_auxiliaries_type ac, - build_package_configs bcs, + package_build_configs bcs, shared_ptr rp) : id (rp->tenant, move (nm), vr), tenant (id.tenant), diff --git a/libbrep/package.hxx b/libbrep/package.hxx index b8c3c33..3878530 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 32, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed) namespace brep { @@ -248,8 +248,8 @@ namespace brep string id; - // If true, display the packages in the web interface only in the tenant - // view mode. + // If this flag is true, then display the packages in the web interface + // only in the tenant view mode. // bool private_; // Note: foreign-mapped in build. @@ -456,6 +456,58 @@ namespace brep #pragma db member(text) column("") }; + // Tweak public_key_id mapping to include a constraint (this only affects the + // database schema). + // + #pragma db member(public_key_id::tenant) points_to(tenant) + + #pragma db object pointer(shared_ptr) session + class public_key: public string + { + public: + public_key (string tenant, string fingerprint, string key) + : string (move (key)), id (move (tenant), move (fingerprint)) {} + + public_key_id id; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(data) virtual(string) access(this) + + private: + friend class odb::access; + public_key () = default; + }; + + // package_build_config + // + using package_build_config = + build_package_config_template>; + + using package_build_configs = + build_package_configs_template>; + + #pragma db value(package_build_config) definition + + #pragma db member(package_build_config::builds) transient + #pragma db member(package_build_config::constraints) transient + #pragma db member(package_build_config::auxiliaries) transient + #pragma db member(package_build_config::bot_keys) transient + + // package_build_bot_keys + // + using package_build_bot_keys = vector>; + using package_build_bot_key_key = odb::nested_key; + + using package_build_bot_keys_map = std::map>; + + #pragma db value(package_build_bot_key_key) + #pragma db member(package_build_bot_key_key::outer) column("config_index") + #pragma db member(package_build_bot_key_key::inner) column("index") + // Tweak package_id mapping to include a constraint (this only affects the // database schema). // @@ -507,7 +559,8 @@ namespace brep build_class_exprs, build_constraints_type, build_auxiliaries_type, - build_package_configs, + package_build_bot_keys, + package_build_configs, optional location, optional fragment, optional sha256sum, @@ -535,7 +588,7 @@ namespace brep build_class_exprs, build_constraints_type, build_auxiliaries_type, - build_package_configs, + package_build_configs, shared_ptr); bool @@ -589,15 +642,25 @@ namespace brep requirements_type requirements; // Note: foreign-mapped in build. small_vector tests; // Note: foreign-mapped in build. - // Common build classes, constraints, and auxiliaries that apply to all - // configurations unless overridden. + // Common build classes, constraints, auxiliaries, and bot keys that apply + // to all configurations unless overridden. // build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. build_auxiliaries_type build_auxiliaries; // Note: foreign-mapped in build. + package_build_bot_keys build_bot_keys; // Note: foreign-mapped in build. + package_build_configs build_configs; // Note: foreign-mapped in build. - build_package_configs build_configs; // Note: foreign-mapped in build. - + // Group the build_configs, builds, and build_constraints members of this + // object together with their respective nested configs entries into the + // separate section for an explicit load. + // + // Note that while the build auxiliaries and bot keys are persisted via + // the newly created package objects, they are only used via the + // foreign-mapped build_package objects (see build-package.hxx for + // details). Thus, we add them to the never-loaded unused_section (see + // below). + // odb::section build_section; // Note that it is foreign-mapped in build. @@ -628,6 +691,18 @@ namespace brep bool buildable; // Note: foreign-mapped in build. optional unbuildable_reason; + // If this flag is true, then all the package configurations are buildable + // with the custom build bots. If false, then all configurations are + // buildable with the default bots. If nullopt, then some configurations + // are buildable with the custom and some with the default build bots. + // + // Note: meaningless if buildable is false. + // + optional custom_bot; // Note: foreign-mapped in build. + + private: + odb::section unused_section; + // Database mapping. // #pragma db member(id) id column("") @@ -750,13 +825,19 @@ namespace brep // build_auxiliaries // #pragma db member(build_auxiliaries) id_column("") value_column("") \ - section(build_section) + section(unused_section) + + // build_bot_keys + // + #pragma db member(build_bot_keys) \ + id_column("") value_column("key_") value_not_null \ + section(unused_section) // build_configs // - // Note that build_package_config::{builds,constraints,auxiliaries} are - // persisted/loaded via the separate nested containers (see commons.hxx - // for details). + // Note that package_build_config::{builds,constraints,auxiliaries, + // bot_keys} are persisted/loaded via the separate nested containers (see + // commons.hxx for details). // #pragma db member(build_configs) id_column("") value_column("config_") \ section(build_section) @@ -792,9 +873,22 @@ namespace brep odb::nested_set (as, std::move (?)); \ move (as).to_configs (this.build_configs)) \ id_column("") key_column("") value_column("") \ - section(build_section) - - #pragma db member(build_section) load(lazy) update(always) + section(unused_section) + + #pragma db member(build_config_bot_keys) \ + virtual(package_build_bot_keys_map) \ + after(build_config_auxiliaries) \ + get(odb::nested_get ( \ + brep::build_package_config_bot_keys (this.build_configs))) \ + set(brep::build_package_config_bot_keys< \ + lazy_shared_ptr> bks; \ + odb::nested_set (bks, std::move (?)); \ + move (bks).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("key_") value_not_null \ + section(unused_section) + + #pragma db member(build_section) load(lazy) update(always) + #pragma db member(unused_section) load(lazy) update(manual) // other_repositories // diff --git a/libbrep/package.xml b/libbrep/package.xml index b66a66d..96e93a7 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,139 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/load/load.cli b/load/load.cli index b9aa769..99d76f6 100644 --- a/load/load.cli +++ b/load/load.cli @@ -167,6 +167,22 @@ class options this option to specify multiple package manager options." } + brep::path openssl = "openssl" + { + "", + "The openssl program to be used for crypto operations. You can also + specify additional options that should be passed to the openssl program + with \cb{openssl-option}. If the openssl program is not explicitly + specified, then \cb{brep-load} will use \cb{openssl} by default." + } + + brep::strings openssl-option + { + "", + "Additional option to be passed to the openssl program (see \cb{openssl} + for details). Repeat this option to specify multiple openssl options." + } + std::string --pager // String to allow empty value. { "", diff --git a/load/load.cxx b/load/load.cxx index 14b8374..5b4692c 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -364,7 +365,8 @@ repository_info (const options& lo, const string& rl, const cstrings& options) // the repository. Should be called once per repository. // static void -load_packages (const shared_ptr& rp, +load_packages (const options& lo, + const shared_ptr& rp, const repository_location& cl, database& db, bool ignore_unknown, @@ -421,10 +423,12 @@ load_packages (const shared_ptr& rp, using brep::dependency_alternative; using brep::dependency_alternatives; + const string& tenant (rp->tenant); + for (package_manifest& pm: pms) { shared_ptr p ( - db.find (package_id (rp->tenant, pm.name, pm.version))); + db.find (package_id (tenant, pm.name, pm.version))); // sha256sum should always be present if the package manifest comes from // the packages.manifest file belonging to the pkg repository. @@ -433,6 +437,32 @@ load_packages (const shared_ptr& rp, if (p == nullptr) { + // Convert the package manifest build configurations (contain public + // keys data) into the brep's build package configurations (contain + // public key object lazy pointers). Keep the bot key lists empty if + // the package is not buildable. + // + package_build_configs build_configs; + + if (!pm.build_configs.empty ()) + { + build_configs.reserve (pm.build_configs.size ()); + + for (bpkg::build_package_config& c: pm.build_configs) + { + build_configs.emplace_back (move (c.name), + move (c.arguments), + move (c.comment), + move (c.builds), + move (c.constraints), + move (c.auxiliaries), + package_build_bot_keys (), + move (c.email), + move (c.warning_email), + move (c.error_email)); + } + } + if (rp->internal) { if (!overrides.empty ()) @@ -594,6 +624,107 @@ load_packages (const shared_ptr& rp, // package_name project (pm.effective_project ()); + // If the package is buildable, then save the package manifest's + // common and build configuration-specific bot keys into the database + // and translate the key data lists into the lists of the public key + // object lazy pointers. + // + package_build_bot_keys bot_keys; + + if (rp->buildable) + { + // Save the specified bot keys into the database as public key + // objects, unless they are already persisted. Translate these keys + // into the public key object lazy pointers. + // + auto keys_to_objects = [&lo, + &pm, + &tenant, + &db] (strings&& keys) + { + package_build_bot_keys r; + + if (keys.empty ()) + return r; + + r.reserve (keys.size ()); + + for (string& key: keys) + { + // Calculate the key fingerprint. + // + string fp; + + try + { + openssl os (path ("-"), path ("-"), 2, + lo.openssl (), + "pkey", + lo.openssl_option (), "-pubin", "-outform", "DER"); + + os.out << key; + os.out.close (); + + fp = sha256 (os.in).string (); + os.in.close (); + + if (!os.wait ()) + { + cerr << "process " << lo.openssl () << ' ' << *os.exit + << endl; + + throw io_error (""); + } + } + catch (const io_error&) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << endl + << " info: key:" << endl + << key << endl; + + throw failed (); + } + catch (const process_error& e) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << ": " + << e << endl; + + throw failed (); + } + + // Try to find the public_key object for the calculated + // fingerprint. If it doesn't exist, then create and persist the + // new object. + // + public_key_id id (tenant, move (fp)); + shared_ptr k (db.find (id)); + + if (k == nullptr) + { + k = make_shared (move (id.tenant), + move (id.fingerprint), + move (key)); + + db.persist (k); + } + + r.push_back (move (k)); + } + + return r; + }; + + bot_keys = keys_to_objects (move (pm.build_bot_keys)); + + assert (build_configs.size () == pm.build_configs.size ()); + + for (size_t i (0); i != build_configs.size (); ++i) + build_configs[i].bot_keys = + keys_to_objects (move (pm.build_configs[i].bot_keys)); + } + p = make_shared ( move (pm.name), move (pm.version), @@ -622,7 +753,8 @@ load_packages (const shared_ptr& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (bot_keys), + move (build_configs), move (pm.location), move (pm.fragment), move (pm.sha256sum), @@ -636,7 +768,7 @@ load_packages (const shared_ptr& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (build_configs), rp); db.persist (p); @@ -1018,7 +1150,8 @@ load_repositories (const options& lo, // We don't apply overrides to the external packages. // - load_packages (pr, + load_packages (lo, + pr, !pr->cache_location.empty () ? pr->cache_location : cl, db, ignore_unknown, @@ -1630,6 +1763,7 @@ try { db.erase_query (); db.erase_query (); + db.erase_query (); db.erase_query (); } else // Multi-tenant mode. @@ -1642,6 +1776,9 @@ try db.erase_query ( query::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query ( + query::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query ( query::id.in_range (ts.begin (), ts.end ())); } @@ -1696,7 +1833,8 @@ try ir.buildable, priority++)); - load_packages (r, + load_packages (ops, + r, r->cache_location, db, ops.ignore_unknown (), diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 78661c3..c1630b0 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -36,8 +36,9 @@ namespace brep void init (const options::build&); + template bool - exclude (const build_package_config& pc, + exclude (const build_package_config_template& pc, const build_class_exprs& common_builds, const build_constraints& common_constraints, const build_target_config& tc, diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx index 68fbe4c..9ac1390 100644 --- a/mod/build-result-module.cxx +++ b/mod/build-result-module.cxx @@ -3,11 +3,16 @@ #include +#include + #include #include #include #include +#include +#include + namespace brep { using namespace std; @@ -230,54 +235,112 @@ namespace brep else { assert (b.agent_fingerprint && challenge); - auto i (bot_agent_key_map_->find (*b.agent_fingerprint)); - // The agent's key is recently replaced. + auto auth = [&challenge, + &b, + &o, + &fail, &trace, + &warn_auth, + this] (const path& key) + { + bool r (false); + + try + { + openssl os ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + path ("-"), fdstream_mode::text, 2, + process_env (o.openssl (), o.openssl_envvar ()), + use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", + o.openssl_option (), + use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", + "-pubin", + "-inkey", key); + + for (const auto& c: *challenge) + os.out.put (c); // Sets badbit on failure. + + os.out.close (); + + string s; + getline (os.in, s); + + bool v (os.in.eof ()); + os.in.close (); + + if (os.wait () && v) + { + r = (s == *b.agent_challenge); + + if (!r) + warn_auth ("challenge mismatched"); + } + else // The signature is presumably meaningless. + warn_auth ("unable to verify challenge"); + } + catch (const system_error& e) + { + fail << "unable to verify challenge: " << e; + } + + return r; + }; + + const string& fp (*b.agent_fingerprint); + auto i (bot_agent_key_map_->find (fp)); + + // Note that it is possible that the default vs custom bot + // classification has changed since the task request time. It feels that + // there is nothing wrong with that and we will handle that + // automatically. // - if (i == bot_agent_key_map_->end ()) + if (i != bot_agent_key_map_->end ()) // Default bot? { - warn_auth ("agent's public key not found"); + r = auth (i->second); } - else - try + else // Custom bot. { - openssl os ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - path ("-"), fdstream_mode::text, 2, - process_env (o.openssl (), o.openssl_envvar ()), - use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", - o.openssl_option (), - use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", - "-pubin", - "-inkey", - i->second); - - for (const auto& c: *challenge) - os.out.put (c); // Sets badbit on failure. - - os.out.close (); - - string s; - getline (os.in, s); - - bool v (os.in.eof ()); - os.in.close (); - - if (os.wait () && v) - { - r = (s == *b.agent_challenge); + shared_ptr k ( + build_db_->find (public_key_id (b.tenant, fp))); - if (!r) - warn_auth ("challenge mismatched"); + if (k != nullptr) + { + // Temporarily save the key data to disk (note that it's the + // challenge which is passed via stdin to openssl). Hopefully /tmp + // is using tmpfs. + // + auto_rmfile arm; + + try + { + arm = auto_rmfile (path::temp_path ("brep-custom-bot-key")); + } + catch (const system_error& e) + { + fail << "unable to obtain temporary file: " << e; + } + + try + { + ofdstream os (arm.path); + os << *k; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << arm.path << "': " << e; + } + + r = auth (arm.path); + } + else + { + // The agent's key is recently replaced. + // + warn_auth ("agent's public key not found"); } - else // The signature is presumably meaningless. - warn_auth ("unable to verify challenge"); - } - catch (const system_error& e) - { - fail << "unable to verify challenge: " << e; } } diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx index a30cf07..a30e281 100644 --- a/mod/build-target-config.cxx +++ b/mod/build-target-config.cxx @@ -21,17 +21,13 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const build_package_config& pc, - const build_class_exprs& cbs, - const build_constraints& ccs, + exclude (const build_class_exprs& exprs, + const build_constraints& constrs, const build_target_config& tc, const map& class_inheritance_map, string* reason, bool default_all_ucs) { - const build_class_exprs& exprs (pc.effective_builds (cbs)); - const build_constraints& constrs (pc.effective_constraints (ccs)); - // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx index 180ca80..60d159c 100644 --- a/mod/build-target-config.hxx +++ b/mod/build-target-config.hxx @@ -32,14 +32,31 @@ namespace brep // configuration set needlessly). // bool - exclude (const build_package_config&, - const build_class_exprs& common_builds, - const build_constraints& common_constraints, + exclude (const build_class_exprs& builds, + const build_constraints& constraints, const build_target_config&, const std::map& class_inheritance_map, string* reason = nullptr, bool default_all_ucs = false); + template + inline bool + exclude (const build_package_config_template& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, + const std::map& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false) + { + return exclude (pc.effective_builds (common_builds), + pc.effective_constraints (common_constraints), + tc, + class_inheritance_map, + reason, + default_all_ucs); + } + // Convert dash-separated components (target, build target configuration // name, machine name) or a pattern thereof into a path, replacing dashes // with slashes (directory separators), `**` with `*/**/*`, and appending diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 773d041..e0aad4b 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -132,7 +132,8 @@ init (scanner& s) // template static inline query -package_query (brep::params::build_task& params, +package_query (bool custom_bot, + brep::params::build_task& params, interactive_mode imode, uint64_t queued_expiration_ns) { @@ -141,9 +142,39 @@ package_query (brep::params::build_task& params, query q (!query::build_tenant::archived); + if (custom_bot) + { + // Note that we could potentially only query the packages which refer to + // this custom bot key in one of their build configurations. For that we + // would need to additionally join the current query tables with the bot + // fingerprint-containing build_package_bot_keys and + // build_package_config_bot_keys tables and use the SELECT DISTINCT + // clause. The problem is that we also use the ORDER BY clause and in this + // case PostgreSQL requires all the ORDER BY clause expressions to also be + // present in the SELECT DISTINCT clause and fails with the 'for SELECT + // DISTINCT, ORDER BY expressions must appear in select list' error if + // that's not the case. Also note that in the ODB-generated code the + // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause + // (see the CITEXT type mapping for details in libbrep/common.hxx) would + // not match the 'build_package.name' expression in the ORDER BY clause + // and so we will end up with the mentioned error. One (hackish) way to + // fix that would be to add a dummy member of the string type for the + // build_package.name column. This all sounds quite hairy at the moment + // and it also feels that this can potentially pessimize querying the + // packages built with the default bots only. Thus let's keep it simple + // for now and filter packages by the bot fingerprint at the program + // level. + // + q = q && (query::build_package::custom_bot.is_null () || + query::build_package::custom_bot); + } + else + q = q && (query::build_package::custom_bot.is_null () || + !query::build_package::custom_bot); + // Filter by repositories canonical names (if requested). // - const vector& rp (params.repository ()); + const strings& rp (params.repository ()); if (!rp.empty ()) q = q && @@ -213,20 +244,28 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Obtain the agent's public key fingerprint if requested. If the fingerprint - // is requested but is not present in the request or is unknown, then respond - // with 401 HTTP code (unauthorized). + // Obtain the agent's public key fingerprint if requested. If the + // fingerprint is requested but is not present in the request, then respond + // with 401 HTTP code (unauthorized). If a key with the specified + // fingerprint is not present in the build bot agent keys directory, then + // assume that this is a custom build bot. + // + // Note that if the agent authentication is not configured (the agent keys + // directory is not specified), then the bot can never be custom and its + // fingerprint is ignored, if present. // optional agent_fp; + bool custom_bot (false); if (bot_agent_key_map_ != nullptr) { - if (!tqm.fingerprint || - bot_agent_key_map_->find (*tqm.fingerprint) == - bot_agent_key_map_->end ()) + if (!tqm.fingerprint) throw invalid_request (401, "unauthorized"); agent_fp = move (tqm.fingerprint); + + custom_bot = (bot_agent_key_map_->find (*agent_fp) == + bot_agent_key_map_->end ()); } // The resulting task manifest and the related build, package, and @@ -659,7 +698,8 @@ handle (request& rq, response& rs) using pkg_query = query; using prep_pkg_query = prepared_query; - pkg_query pq (package_query (params, + pkg_query pq (package_query (custom_bot, + params, imode, queued_expiration_ns)); @@ -815,7 +855,8 @@ handle (request& rq, response& rs) { using query = query; - query q (package_query (params, + query q (package_query (custom_bot, + params, imode, queued_expiration_ns)); @@ -1241,7 +1282,8 @@ handle (request& rq, response& rs) // small_vector tests; - build_db_->load (*p, p->requirements_tests_section); + if (!p->requirements_tests_section.loaded ()) + build_db_->load (*p, p->requirements_tests_section); for (const build_test_dependency& td: p->tests) { @@ -1293,6 +1335,8 @@ handle (request& rq, response& rs) true /* default_all_ucs */)) continue; + build_db_->load (*tp, tp->auxiliaries_section); + for (const build_auxiliary& ba: tpc->effective_auxiliaries (tp->auxiliaries)) { @@ -1312,14 +1356,17 @@ handle (request& rq, response& rs) vector tms; vector bms; - tms.reserve (picked_machines.size ()); - bms.reserve (picked_machines.size ()); - - for (pair& pm: picked_machines) + if (size_t n = picked_machines.size ()) { - const machine_header_manifest& m (*pm.first.machine); - tms.push_back (auxiliary_machine {m.name, move (pm.second)}); - bms.push_back (build_machine {m.name, m.summary}); + tms.reserve (n); + bms.reserve (n); + + for (pair& pm: picked_machines) + { + const machine_header_manifest& m (*pm.first.machine); + tms.push_back (auxiliary_machine {m.name, move (pm.second)}); + bms.push_back (build_machine {m.name, m.summary}); + } } return collect_auxiliaries_result { @@ -1603,8 +1650,40 @@ handle (request& rq, response& rs) // bool package_built (false); + build_db_->load (*p, p->bot_keys_section); + for (const build_package_config& pc: p->configs) { + // If this is a custom bot, then skip this configuration if it + // doesn't contain this bot's public key in its custom bot keys + // list. Otherwise (this is a default bot), skip this + // configuration if its custom bot keys list is not empty. + // + { + const build_package_bot_keys& bks ( + pc.effective_bot_keys (p->bot_keys)); + + if (custom_bot) + { + assert (agent_fp); // Wouldn't be here otherwise. + + if (find_if ( + bks.begin (), bks.end (), + [&agent_fp] (const lazy_shared_ptr& k) + { + return k.object_id ().fingerprint == *agent_fp; + }) == bks.end ()) + { + continue; + } + } + else + { + if (!bks.empty ()) + continue; + } + } + pkg_config = pc.name; // Iterate through the built configurations and erase them from the @@ -1647,29 +1726,33 @@ handle (request& rq, response& rs) // the package configuration and for which all the requested // auxiliary machines can be provided. // - auto i (configs.begin ()); - auto e (configs.end ()); + const config_machine* cm (nullptr); + optional aux; build_db_->load (*p, p->constraints_section); - optional aux; - for (; i != e; ++i) + for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i) { - const build_target_config& tc (*i->second.config); + cm = &i->second; + const build_target_config& tc (*cm->config); - if (!exclude (pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, pc, tc))) - break; + if (!exclude (pc, p->builds, p->constraints, tc)) + { + if (!p->auxiliaries_section.loaded ()) + build_db_->load (*p, p->auxiliaries_section); + + if ((aux = collect_auxiliaries (p, pc, tc))) + break; + } } - if (i != e) + if (aux) { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); + machine_header_manifest& mh (*cm->machine); build_id bid (move (id), - cm.config->target, - cm.config->name, + cm->config->target, + cm->config->name, move (pkg_config), move (toolchain_name), toolchain_version); @@ -1704,8 +1787,8 @@ handle (request& rq, response& rs) build_machine { mh.name, move (mh.summary)}, move (aux->build_auxiliary_machines), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); + controller_checksum (*cm->config), + machine_checksum (*cm->machine)); build_db_->persist (b); } @@ -1753,8 +1836,8 @@ handle (request& rq, response& rs) b->auxiliary_machines = move (aux->build_auxiliary_machines); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + string ccs (controller_checksum (*cm->config)); + string mcs (machine_checksum (*cm->machine)); // Issue the hard rebuild if it is forced or the // configuration or machine has changed. @@ -1831,7 +1914,7 @@ handle (request& rq, response& rs) move (aux->tests), move (aux->task_auxiliary_machines), move (bp.interactive), - cm); + *cm); task_build = move (b); task_package = move (p); @@ -1971,13 +2054,17 @@ handle (request& rq, response& rs) (t->interactive.has_value () == (imode == interactive_mode::true_)))) { + const build_target_config& tc (*cm.config); + build_db_->load (*p, p->constraints_section); - const build_target_config& tc (*cm.config); + if (exclude (*pc, p->builds, p->constraints, tc)) + continue; + + build_db_->load (*p, p->auxiliaries_section); - optional aux; - if (!exclude (*pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, *pc, tc))) + if (optional aux = + collect_auxiliaries (p, *pc, tc)) { assert (b->status); diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 51b21c6..91923e5 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -541,7 +541,7 @@ handle (request& rq, response& rs) // builds = false; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { const build_class_exprs& exprs (pc.effective_builds (pkg->builds)); @@ -726,7 +726,7 @@ handle (request& rq, response& rs) s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_package_config& pc, + auto exclude = [&pkg, this] (const package_build_config& pc, const build_target_config& tc, string* rs = nullptr) { @@ -767,7 +767,7 @@ handle (request& rq, response& rs) query sq (false); set unbuilt_configs; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& bc: *target_conf_map_) { @@ -886,7 +886,7 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& tc: *target_conf_) { diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 2f49f81..42d481d 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -95,7 +95,7 @@ namespace brep // not. // void - add_delay (shared_ptr, bool report); + add_delay (shared_ptr, bool custom_bot, bool report); bool empty () const {return reported_delay_count_ == 0;} @@ -108,19 +108,39 @@ namespace brep print (const char* header, bool total, bool full) const; private: - // Maps delays to the report flag. + // Maps delays to the custom bot and report flag. // - map, bool, compare_delay> delays_; + struct delay_info + { + bool custom_bot; + bool report; + }; + + map, delay_info, compare_delay> delays_; size_t reported_delay_count_ = 0; + + // Number of reported/total delayed package configurations which need to + // be built with the custom build bots. + // + size_t custom_total_delay_count_ = 0; + size_t custom_reported_delay_count_ = 0; }; void delay_report:: - add_delay (shared_ptr delay, bool report) + add_delay (shared_ptr delay, bool custom_bot, bool report) { - delays_.emplace (move (delay), report); + delays_.emplace (move (delay), delay_info {custom_bot, report}); + + if (custom_bot) + ++custom_total_delay_count_; if (report) + { ++reported_delay_count_; + + if (custom_bot) + ++custom_reported_delay_count_; + } } void delay_report:: @@ -134,6 +154,17 @@ namespace brep if (total) cerr << '/' << delays_.size (); + if (custom_reported_delay_count_ != 0 || + (total && custom_total_delay_count_ != 0)) + { + cerr << " including " << custom_reported_delay_count_; + + if (total) + cerr << '/' << custom_total_delay_count_; + + cerr << " custom"; + } + cerr << "):" << endl; // Group the printed delays by toolchain and target configuration. @@ -146,10 +177,15 @@ namespace brep size_t config_reported_delay_count (0); size_t config_total_delay_count (0); + size_t config_custom_reported_delay_count (0); + size_t config_custom_total_delay_count (0); + auto brief_config = [&target_config_name, &target, &config_reported_delay_count, &config_total_delay_count, + &config_custom_reported_delay_count, + &config_custom_total_delay_count, total] () { if (target_config_name != nullptr) @@ -166,21 +202,36 @@ namespace brep if (total) cerr << '/' << config_total_delay_count; + if (config_custom_reported_delay_count != 0 || + (total && config_custom_total_delay_count != 0)) + { + cerr << " including " << config_custom_reported_delay_count; + + if (total) + cerr << '/' << config_custom_total_delay_count; + + cerr << " custom"; + } + cerr << ')' << endl; } config_reported_delay_count = 0; config_total_delay_count = 0; + + config_custom_reported_delay_count = 0; + config_custom_total_delay_count = 0; } }; for (const auto& dr: delays_) { - bool report (dr.second); + bool report (dr.second.report); if (full && !report) continue; + bool custom_bot (dr.second.custom_bot); const shared_ptr& d (dr.first); // Print the toolchain, if changed. @@ -239,6 +290,9 @@ namespace brep cerr << " " << d->package_name << '/' << d->package_version << ' ' << d->package_config_name; + if (custom_bot) + cerr << " (custom bot)"; + if (!d->tenant.empty ()) cerr << ' ' << d->tenant; @@ -247,9 +301,17 @@ namespace brep else { if (report) + { ++config_reported_delay_count; + if (custom_bot) + ++config_custom_reported_delay_count; + } + ++config_total_delay_count; + + if (custom_bot) + ++config_custom_total_delay_count; } } @@ -953,6 +1015,21 @@ namespace brep soft_delayed = false; } + // If there is a delay, then deduce if this package + // configuration needs to be built with a custom build bot. + // + // Note: only meaningful if there is a delay. + // + bool custom_bot (false); + + if (hard_delayed || soft_delayed) + { + if (!p->bot_keys_section.loaded ()) + db.load (*p, p->bot_keys_section); + + custom_bot = !pc.effective_bot_keys (p->bot_keys).empty (); + } + // Add hard/soft delays to the respective reports and // collect the delay for update, if it is reported. // @@ -983,7 +1060,7 @@ namespace brep reported = true; } - hard_delays_report.add_delay (d, report); + hard_delays_report.add_delay (d, custom_bot, report); } if (soft_delayed) @@ -997,7 +1074,7 @@ namespace brep reported = true; } - soft_delays_report.add_delay (d, report); + soft_delays_report.add_delay (d, custom_bot, report); } // If we don't consider the report timestamps for reporting -- cgit v1.1