diff options
Diffstat (limited to 'mod')
| -rw-r--r-- | mod/build-config-module.hxx | 3 | ||||
| -rw-r--r-- | mod/build-result-module.cxx | 145 | ||||
| -rw-r--r-- | mod/build-target-config.cxx | 8 | ||||
| -rw-r--r-- | mod/build-target-config.hxx | 23 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 167 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 8 |
6 files changed, 259 insertions, 95 deletions
diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 78661c3..c1630b0 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -36,8 +36,9 @@ namespace brep void init (const options::build&); + template <typename K> bool - exclude (const build_package_config& pc, + exclude (const build_package_config_template<K>& pc, const build_class_exprs& common_builds, const build_constraints& common_constraints, const build_target_config& tc, diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx index 68fbe4c..9ac1390 100644 --- a/mod/build-result-module.cxx +++ b/mod/build-result-module.cxx @@ -3,11 +3,16 @@ #include <mod/build-result-module.hxx> +#include <odb/database.hxx> + #include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/process-io.hxx> #include <libbutl/semantic-version.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + namespace brep { using namespace std; @@ -230,54 +235,112 @@ namespace brep else { assert (b.agent_fingerprint && challenge); - auto i (bot_agent_key_map_->find (*b.agent_fingerprint)); - // The agent's key is recently replaced. + auto auth = [&challenge, + &b, + &o, + &fail, &trace, + &warn_auth, + this] (const path& key) + { + bool r (false); + + try + { + openssl os ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + path ("-"), fdstream_mode::text, 2, + process_env (o.openssl (), o.openssl_envvar ()), + use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", + o.openssl_option (), + use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", + "-pubin", + "-inkey", key); + + for (const auto& c: *challenge) + os.out.put (c); // Sets badbit on failure. + + os.out.close (); + + string s; + getline (os.in, s); + + bool v (os.in.eof ()); + os.in.close (); + + if (os.wait () && v) + { + r = (s == *b.agent_challenge); + + if (!r) + warn_auth ("challenge mismatched"); + } + else // The signature is presumably meaningless. + warn_auth ("unable to verify challenge"); + } + catch (const system_error& e) + { + fail << "unable to verify challenge: " << e; + } + + return r; + }; + + const string& fp (*b.agent_fingerprint); + auto i (bot_agent_key_map_->find (fp)); + + // Note that it is possible that the default vs custom bot + // classification has changed since the task request time. It feels that + // there is nothing wrong with that and we will handle that + // automatically. // - if (i == bot_agent_key_map_->end ()) + if (i != bot_agent_key_map_->end ()) // Default bot? { - warn_auth ("agent's public key not found"); + r = auth (i->second); } - else - try + else // Custom bot. { - openssl os ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - path ("-"), fdstream_mode::text, 2, - process_env (o.openssl (), o.openssl_envvar ()), - use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", - o.openssl_option (), - use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", - "-pubin", - "-inkey", - i->second); - - for (const auto& c: *challenge) - os.out.put (c); // Sets badbit on failure. - - os.out.close (); - - string s; - getline (os.in, s); - - bool v (os.in.eof ()); - os.in.close (); - - if (os.wait () && v) - { - r = (s == *b.agent_challenge); + shared_ptr<build_public_key> k ( + build_db_->find<build_public_key> (public_key_id (b.tenant, fp))); - if (!r) - warn_auth ("challenge mismatched"); + if (k != nullptr) + { + // Temporarily save the key data to disk (note that it's the + // challenge which is passed via stdin to openssl). Hopefully /tmp + // is using tmpfs. + // + auto_rmfile arm; + + try + { + arm = auto_rmfile (path::temp_path ("brep-custom-bot-key")); + } + catch (const system_error& e) + { + fail << "unable to obtain temporary file: " << e; + } + + try + { + ofdstream os (arm.path); + os << *k; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << arm.path << "': " << e; + } + + r = auth (arm.path); + } + else + { + // The agent's key is recently replaced. + // + warn_auth ("agent's public key not found"); } - else // The signature is presumably meaningless. - warn_auth ("unable to verify challenge"); - } - catch (const system_error& e) - { - fail << "unable to verify challenge: " << e; } } diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx index a30cf07..a30e281 100644 --- a/mod/build-target-config.cxx +++ b/mod/build-target-config.cxx @@ -21,17 +21,13 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const build_package_config& pc, - const build_class_exprs& cbs, - const build_constraints& ccs, + exclude (const build_class_exprs& exprs, + const build_constraints& constrs, const build_target_config& tc, const map<string, string>& class_inheritance_map, string* reason, bool default_all_ucs) { - const build_class_exprs& exprs (pc.effective_builds (cbs)); - const build_constraints& constrs (pc.effective_constraints (ccs)); - // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx index 180ca80..60d159c 100644 --- a/mod/build-target-config.hxx +++ b/mod/build-target-config.hxx @@ -32,14 +32,31 @@ namespace brep // configuration set needlessly). // bool - exclude (const build_package_config&, - const build_class_exprs& common_builds, - const build_constraints& common_constraints, + exclude (const build_class_exprs& builds, + const build_constraints& constraints, const build_target_config&, const std::map<string, string>& class_inheritance_map, string* reason = nullptr, bool default_all_ucs = false); + template <typename K> + inline bool + exclude (const build_package_config_template<K>& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, + const std::map<string, string>& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false) + { + return exclude (pc.effective_builds (common_builds), + pc.effective_constraints (common_constraints), + tc, + class_inheritance_map, + reason, + default_all_ucs); + } + // Convert dash-separated components (target, build target configuration // name, machine name) or a pattern thereof into a path, replacing dashes // with slashes (directory separators), `**` with `*/**/*`, and appending diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 773d041..e0aad4b 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -132,7 +132,8 @@ init (scanner& s) // template <typename T> static inline query<T> -package_query (brep::params::build_task& params, +package_query (bool custom_bot, + brep::params::build_task& params, interactive_mode imode, uint64_t queued_expiration_ns) { @@ -141,9 +142,39 @@ package_query (brep::params::build_task& params, query q (!query::build_tenant::archived); + if (custom_bot) + { + // Note that we could potentially only query the packages which refer to + // this custom bot key in one of their build configurations. For that we + // would need to additionally join the current query tables with the bot + // fingerprint-containing build_package_bot_keys and + // build_package_config_bot_keys tables and use the SELECT DISTINCT + // clause. The problem is that we also use the ORDER BY clause and in this + // case PostgreSQL requires all the ORDER BY clause expressions to also be + // present in the SELECT DISTINCT clause and fails with the 'for SELECT + // DISTINCT, ORDER BY expressions must appear in select list' error if + // that's not the case. Also note that in the ODB-generated code the + // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause + // (see the CITEXT type mapping for details in libbrep/common.hxx) would + // not match the 'build_package.name' expression in the ORDER BY clause + // and so we will end up with the mentioned error. One (hackish) way to + // fix that would be to add a dummy member of the string type for the + // build_package.name column. This all sounds quite hairy at the moment + // and it also feels that this can potentially pessimize querying the + // packages built with the default bots only. Thus let's keep it simple + // for now and filter packages by the bot fingerprint at the program + // level. + // + q = q && (query::build_package::custom_bot.is_null () || + query::build_package::custom_bot); + } + else + q = q && (query::build_package::custom_bot.is_null () || + !query::build_package::custom_bot); + // Filter by repositories canonical names (if requested). // - const vector<string>& rp (params.repository ()); + const strings& rp (params.repository ()); if (!rp.empty ()) q = q && @@ -213,20 +244,28 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Obtain the agent's public key fingerprint if requested. If the fingerprint - // is requested but is not present in the request or is unknown, then respond - // with 401 HTTP code (unauthorized). + // Obtain the agent's public key fingerprint if requested. If the + // fingerprint is requested but is not present in the request, then respond + // with 401 HTTP code (unauthorized). If a key with the specified + // fingerprint is not present in the build bot agent keys directory, then + // assume that this is a custom build bot. + // + // Note that if the agent authentication is not configured (the agent keys + // directory is not specified), then the bot can never be custom and its + // fingerprint is ignored, if present. // optional<string> agent_fp; + bool custom_bot (false); if (bot_agent_key_map_ != nullptr) { - if (!tqm.fingerprint || - bot_agent_key_map_->find (*tqm.fingerprint) == - bot_agent_key_map_->end ()) + if (!tqm.fingerprint) throw invalid_request (401, "unauthorized"); agent_fp = move (tqm.fingerprint); + + custom_bot = (bot_agent_key_map_->find (*agent_fp) == + bot_agent_key_map_->end ()); } // The resulting task manifest and the related build, package, and @@ -659,7 +698,8 @@ handle (request& rq, response& rs) using pkg_query = query<buildable_package>; using prep_pkg_query = prepared_query<buildable_package>; - pkg_query pq (package_query<buildable_package> (params, + pkg_query pq (package_query<buildable_package> (custom_bot, + params, imode, queued_expiration_ns)); @@ -815,7 +855,8 @@ handle (request& rq, response& rs) { using query = query<buildable_package_count>; - query q (package_query<buildable_package_count> (params, + query q (package_query<buildable_package_count> (custom_bot, + params, imode, queued_expiration_ns)); @@ -1241,7 +1282,8 @@ handle (request& rq, response& rs) // small_vector<bpkg::test_dependency, 1> tests; - build_db_->load (*p, p->requirements_tests_section); + if (!p->requirements_tests_section.loaded ()) + build_db_->load (*p, p->requirements_tests_section); for (const build_test_dependency& td: p->tests) { @@ -1293,6 +1335,8 @@ handle (request& rq, response& rs) true /* default_all_ucs */)) continue; + build_db_->load (*tp, tp->auxiliaries_section); + for (const build_auxiliary& ba: tpc->effective_auxiliaries (tp->auxiliaries)) { @@ -1312,14 +1356,17 @@ handle (request& rq, response& rs) vector<auxiliary_machine> tms; vector<build_machine> bms; - tms.reserve (picked_machines.size ()); - bms.reserve (picked_machines.size ()); - - for (pair<auxiliary_config_machine, string>& pm: picked_machines) + if (size_t n = picked_machines.size ()) { - const machine_header_manifest& m (*pm.first.machine); - tms.push_back (auxiliary_machine {m.name, move (pm.second)}); - bms.push_back (build_machine {m.name, m.summary}); + tms.reserve (n); + bms.reserve (n); + + for (pair<auxiliary_config_machine, string>& pm: picked_machines) + { + const machine_header_manifest& m (*pm.first.machine); + tms.push_back (auxiliary_machine {m.name, move (pm.second)}); + bms.push_back (build_machine {m.name, m.summary}); + } } return collect_auxiliaries_result { @@ -1603,8 +1650,40 @@ handle (request& rq, response& rs) // bool package_built (false); + build_db_->load (*p, p->bot_keys_section); + for (const build_package_config& pc: p->configs) { + // If this is a custom bot, then skip this configuration if it + // doesn't contain this bot's public key in its custom bot keys + // list. Otherwise (this is a default bot), skip this + // configuration if its custom bot keys list is not empty. + // + { + const build_package_bot_keys& bks ( + pc.effective_bot_keys (p->bot_keys)); + + if (custom_bot) + { + assert (agent_fp); // Wouldn't be here otherwise. + + if (find_if ( + bks.begin (), bks.end (), + [&agent_fp] (const lazy_shared_ptr<build_public_key>& k) + { + return k.object_id ().fingerprint == *agent_fp; + }) == bks.end ()) + { + continue; + } + } + else + { + if (!bks.empty ()) + continue; + } + } + pkg_config = pc.name; // Iterate through the built configurations and erase them from the @@ -1647,29 +1726,33 @@ handle (request& rq, response& rs) // the package configuration and for which all the requested // auxiliary machines can be provided. // - auto i (configs.begin ()); - auto e (configs.end ()); + const config_machine* cm (nullptr); + optional<collect_auxiliaries_result> aux; build_db_->load (*p, p->constraints_section); - optional<collect_auxiliaries_result> aux; - for (; i != e; ++i) + for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i) { - const build_target_config& tc (*i->second.config); + cm = &i->second; + const build_target_config& tc (*cm->config); - if (!exclude (pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, pc, tc))) - break; + if (!exclude (pc, p->builds, p->constraints, tc)) + { + if (!p->auxiliaries_section.loaded ()) + build_db_->load (*p, p->auxiliaries_section); + + if ((aux = collect_auxiliaries (p, pc, tc))) + break; + } } - if (i != e) + if (aux) { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); + machine_header_manifest& mh (*cm->machine); build_id bid (move (id), - cm.config->target, - cm.config->name, + cm->config->target, + cm->config->name, move (pkg_config), move (toolchain_name), toolchain_version); @@ -1704,8 +1787,8 @@ handle (request& rq, response& rs) build_machine { mh.name, move (mh.summary)}, move (aux->build_auxiliary_machines), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); + controller_checksum (*cm->config), + machine_checksum (*cm->machine)); build_db_->persist (b); } @@ -1753,8 +1836,8 @@ handle (request& rq, response& rs) b->auxiliary_machines = move (aux->build_auxiliary_machines); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + string ccs (controller_checksum (*cm->config)); + string mcs (machine_checksum (*cm->machine)); // Issue the hard rebuild if it is forced or the // configuration or machine has changed. @@ -1831,7 +1914,7 @@ handle (request& rq, response& rs) move (aux->tests), move (aux->task_auxiliary_machines), move (bp.interactive), - cm); + *cm); task_build = move (b); task_package = move (p); @@ -1971,13 +2054,17 @@ handle (request& rq, response& rs) (t->interactive.has_value () == (imode == interactive_mode::true_)))) { + const build_target_config& tc (*cm.config); + build_db_->load (*p, p->constraints_section); - const build_target_config& tc (*cm.config); + if (exclude (*pc, p->builds, p->constraints, tc)) + continue; + + build_db_->load (*p, p->auxiliaries_section); - optional<collect_auxiliaries_result> aux; - if (!exclude (*pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, *pc, tc))) + if (optional<collect_auxiliaries_result> aux = + collect_auxiliaries (p, *pc, tc)) { assert (b->status); diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 51b21c6..91923e5 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -541,7 +541,7 @@ handle (request& rq, response& rs) // builds = false; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { const build_class_exprs& exprs (pc.effective_builds (pkg->builds)); @@ -726,7 +726,7 @@ handle (request& rq, response& rs) s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_package_config& pc, + auto exclude = [&pkg, this] (const package_build_config& pc, const build_target_config& tc, string* rs = nullptr) { @@ -767,7 +767,7 @@ handle (request& rq, response& rs) query sq (false); set<config_toolchain> unbuilt_configs; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& bc: *target_conf_map_) { @@ -886,7 +886,7 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& tc: *target_conf_) { |