aboutsummaryrefslogtreecommitdiff
path: root/mod
diff options
context:
space:
mode:
Diffstat (limited to 'mod')
-rw-r--r--mod/build-config-module.hxx3
-rw-r--r--mod/build-result-module.cxx145
-rw-r--r--mod/build-target-config.cxx8
-rw-r--r--mod/build-target-config.hxx23
-rw-r--r--mod/mod-build-task.cxx167
-rw-r--r--mod/mod-package-version-details.cxx8
6 files changed, 259 insertions, 95 deletions
diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx
index 78661c3..c1630b0 100644
--- a/mod/build-config-module.hxx
+++ b/mod/build-config-module.hxx
@@ -36,8 +36,9 @@ namespace brep
void
init (const options::build&);
+ template <typename K>
bool
- exclude (const build_package_config& pc,
+ exclude (const build_package_config_template<K>& pc,
const build_class_exprs& common_builds,
const build_constraints& common_constraints,
const build_target_config& tc,
diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx
index 68fbe4c..9ac1390 100644
--- a/mod/build-result-module.cxx
+++ b/mod/build-result-module.cxx
@@ -3,11 +3,16 @@
#include <mod/build-result-module.hxx>
+#include <odb/database.hxx>
+
#include <libbutl/openssl.hxx>
#include <libbutl/fdstream.hxx>
#include <libbutl/process-io.hxx>
#include <libbutl/semantic-version.hxx>
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
namespace brep
{
using namespace std;
@@ -230,54 +235,112 @@ namespace brep
else
{
assert (b.agent_fingerprint && challenge);
- auto i (bot_agent_key_map_->find (*b.agent_fingerprint));
- // The agent's key is recently replaced.
+ auto auth = [&challenge,
+ &b,
+ &o,
+ &fail, &trace,
+ &warn_auth,
+ this] (const path& key)
+ {
+ bool r (false);
+
+ try
+ {
+ openssl os ([&trace, this] (const char* args[], size_t n)
+ {
+ l2 ([&]{trace << process_args {args, n};});
+ },
+ path ("-"), fdstream_mode::text, 2,
+ process_env (o.openssl (), o.openssl_envvar ()),
+ use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl",
+ o.openssl_option (),
+ use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify",
+ "-pubin",
+ "-inkey", key);
+
+ for (const auto& c: *challenge)
+ os.out.put (c); // Sets badbit on failure.
+
+ os.out.close ();
+
+ string s;
+ getline (os.in, s);
+
+ bool v (os.in.eof ());
+ os.in.close ();
+
+ if (os.wait () && v)
+ {
+ r = (s == *b.agent_challenge);
+
+ if (!r)
+ warn_auth ("challenge mismatched");
+ }
+ else // The signature is presumably meaningless.
+ warn_auth ("unable to verify challenge");
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to verify challenge: " << e;
+ }
+
+ return r;
+ };
+
+ const string& fp (*b.agent_fingerprint);
+ auto i (bot_agent_key_map_->find (fp));
+
+ // Note that it is possible that the default vs custom bot
+ // classification has changed since the task request time. It feels that
+ // there is nothing wrong with that and we will handle that
+ // automatically.
//
- if (i == bot_agent_key_map_->end ())
+ if (i != bot_agent_key_map_->end ()) // Default bot?
{
- warn_auth ("agent's public key not found");
+ r = auth (i->second);
}
- else
- try
+ else // Custom bot.
{
- openssl os ([&trace, this] (const char* args[], size_t n)
- {
- l2 ([&]{trace << process_args {args, n};});
- },
- path ("-"), fdstream_mode::text, 2,
- process_env (o.openssl (), o.openssl_envvar ()),
- use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl",
- o.openssl_option (),
- use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify",
- "-pubin",
- "-inkey",
- i->second);
-
- for (const auto& c: *challenge)
- os.out.put (c); // Sets badbit on failure.
-
- os.out.close ();
-
- string s;
- getline (os.in, s);
-
- bool v (os.in.eof ());
- os.in.close ();
-
- if (os.wait () && v)
- {
- r = (s == *b.agent_challenge);
+ shared_ptr<build_public_key> k (
+ build_db_->find<build_public_key> (public_key_id (b.tenant, fp)));
- if (!r)
- warn_auth ("challenge mismatched");
+ if (k != nullptr)
+ {
+ // Temporarily save the key data to disk (note that it's the
+ // challenge which is passed via stdin to openssl). Hopefully /tmp
+ // is using tmpfs.
+ //
+ auto_rmfile arm;
+
+ try
+ {
+ arm = auto_rmfile (path::temp_path ("brep-custom-bot-key"));
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to obtain temporary file: " << e;
+ }
+
+ try
+ {
+ ofdstream os (arm.path);
+ os << *k;
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to write to '" << arm.path << "': " << e;
+ }
+
+ r = auth (arm.path);
+ }
+ else
+ {
+ // The agent's key is recently replaced.
+ //
+ warn_auth ("agent's public key not found");
}
- else // The signature is presumably meaningless.
- warn_auth ("unable to verify challenge");
- }
- catch (const system_error& e)
- {
- fail << "unable to verify challenge: " << e;
}
}
diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx
index a30cf07..a30e281 100644
--- a/mod/build-target-config.cxx
+++ b/mod/build-target-config.cxx
@@ -21,17 +21,13 @@ namespace brep
{"all"}, '+', "All.");
bool
- exclude (const build_package_config& pc,
- const build_class_exprs& cbs,
- const build_constraints& ccs,
+ exclude (const build_class_exprs& exprs,
+ const build_constraints& constrs,
const build_target_config& tc,
const map<string, string>& class_inheritance_map,
string* reason,
bool default_all_ucs)
{
- const build_class_exprs& exprs (pc.effective_builds (cbs));
- const build_constraints& constrs (pc.effective_constraints (ccs));
-
// Save the first sentence of the reason, lower-case the first letter if
// the beginning looks like a word (all subsequent characters until a
// whitespace are lower-case letters).
diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx
index 180ca80..60d159c 100644
--- a/mod/build-target-config.hxx
+++ b/mod/build-target-config.hxx
@@ -32,14 +32,31 @@ namespace brep
// configuration set needlessly).
//
bool
- exclude (const build_package_config&,
- const build_class_exprs& common_builds,
- const build_constraints& common_constraints,
+ exclude (const build_class_exprs& builds,
+ const build_constraints& constraints,
const build_target_config&,
const std::map<string, string>& class_inheritance_map,
string* reason = nullptr,
bool default_all_ucs = false);
+ template <typename K>
+ inline bool
+ exclude (const build_package_config_template<K>& pc,
+ const build_class_exprs& common_builds,
+ const build_constraints& common_constraints,
+ const build_target_config& tc,
+ const std::map<string, string>& class_inheritance_map,
+ string* reason = nullptr,
+ bool default_all_ucs = false)
+ {
+ return exclude (pc.effective_builds (common_builds),
+ pc.effective_constraints (common_constraints),
+ tc,
+ class_inheritance_map,
+ reason,
+ default_all_ucs);
+ }
+
// Convert dash-separated components (target, build target configuration
// name, machine name) or a pattern thereof into a path, replacing dashes
// with slashes (directory separators), `**` with `*/**/*`, and appending
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 773d041..e0aad4b 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -132,7 +132,8 @@ init (scanner& s)
//
template <typename T>
static inline query<T>
-package_query (brep::params::build_task& params,
+package_query (bool custom_bot,
+ brep::params::build_task& params,
interactive_mode imode,
uint64_t queued_expiration_ns)
{
@@ -141,9 +142,39 @@ package_query (brep::params::build_task& params,
query q (!query::build_tenant::archived);
+ if (custom_bot)
+ {
+ // Note that we could potentially only query the packages which refer to
+ // this custom bot key in one of their build configurations. For that we
+ // would need to additionally join the current query tables with the bot
+ // fingerprint-containing build_package_bot_keys and
+ // build_package_config_bot_keys tables and use the SELECT DISTINCT
+ // clause. The problem is that we also use the ORDER BY clause and in this
+ // case PostgreSQL requires all the ORDER BY clause expressions to also be
+ // present in the SELECT DISTINCT clause and fails with the 'for SELECT
+ // DISTINCT, ORDER BY expressions must appear in select list' error if
+ // that's not the case. Also note that in the ODB-generated code the
+ // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause
+ // (see the CITEXT type mapping for details in libbrep/common.hxx) would
+ // not match the 'build_package.name' expression in the ORDER BY clause
+ // and so we will end up with the mentioned error. One (hackish) way to
+ // fix that would be to add a dummy member of the string type for the
+ // build_package.name column. This all sounds quite hairy at the moment
+ // and it also feels that this can potentially pessimize querying the
+ // packages built with the default bots only. Thus let's keep it simple
+ // for now and filter packages by the bot fingerprint at the program
+ // level.
+ //
+ q = q && (query::build_package::custom_bot.is_null () ||
+ query::build_package::custom_bot);
+ }
+ else
+ q = q && (query::build_package::custom_bot.is_null () ||
+ !query::build_package::custom_bot);
+
// Filter by repositories canonical names (if requested).
//
- const vector<string>& rp (params.repository ());
+ const strings& rp (params.repository ());
if (!rp.empty ())
q = q &&
@@ -213,20 +244,28 @@ handle (request& rq, response& rs)
throw invalid_request (400, e.what ());
}
- // Obtain the agent's public key fingerprint if requested. If the fingerprint
- // is requested but is not present in the request or is unknown, then respond
- // with 401 HTTP code (unauthorized).
+ // Obtain the agent's public key fingerprint if requested. If the
+ // fingerprint is requested but is not present in the request, then respond
+ // with 401 HTTP code (unauthorized). If a key with the specified
+ // fingerprint is not present in the build bot agent keys directory, then
+ // assume that this is a custom build bot.
+ //
+ // Note that if the agent authentication is not configured (the agent keys
+ // directory is not specified), then the bot can never be custom and its
+ // fingerprint is ignored, if present.
//
optional<string> agent_fp;
+ bool custom_bot (false);
if (bot_agent_key_map_ != nullptr)
{
- if (!tqm.fingerprint ||
- bot_agent_key_map_->find (*tqm.fingerprint) ==
- bot_agent_key_map_->end ())
+ if (!tqm.fingerprint)
throw invalid_request (401, "unauthorized");
agent_fp = move (tqm.fingerprint);
+
+ custom_bot = (bot_agent_key_map_->find (*agent_fp) ==
+ bot_agent_key_map_->end ());
}
// The resulting task manifest and the related build, package, and
@@ -659,7 +698,8 @@ handle (request& rq, response& rs)
using pkg_query = query<buildable_package>;
using prep_pkg_query = prepared_query<buildable_package>;
- pkg_query pq (package_query<buildable_package> (params,
+ pkg_query pq (package_query<buildable_package> (custom_bot,
+ params,
imode,
queued_expiration_ns));
@@ -815,7 +855,8 @@ handle (request& rq, response& rs)
{
using query = query<buildable_package_count>;
- query q (package_query<buildable_package_count> (params,
+ query q (package_query<buildable_package_count> (custom_bot,
+ params,
imode,
queued_expiration_ns));
@@ -1241,7 +1282,8 @@ handle (request& rq, response& rs)
//
small_vector<bpkg::test_dependency, 1> tests;
- build_db_->load (*p, p->requirements_tests_section);
+ if (!p->requirements_tests_section.loaded ())
+ build_db_->load (*p, p->requirements_tests_section);
for (const build_test_dependency& td: p->tests)
{
@@ -1293,6 +1335,8 @@ handle (request& rq, response& rs)
true /* default_all_ucs */))
continue;
+ build_db_->load (*tp, tp->auxiliaries_section);
+
for (const build_auxiliary& ba:
tpc->effective_auxiliaries (tp->auxiliaries))
{
@@ -1312,14 +1356,17 @@ handle (request& rq, response& rs)
vector<auxiliary_machine> tms;
vector<build_machine> bms;
- tms.reserve (picked_machines.size ());
- bms.reserve (picked_machines.size ());
-
- for (pair<auxiliary_config_machine, string>& pm: picked_machines)
+ if (size_t n = picked_machines.size ())
{
- const machine_header_manifest& m (*pm.first.machine);
- tms.push_back (auxiliary_machine {m.name, move (pm.second)});
- bms.push_back (build_machine {m.name, m.summary});
+ tms.reserve (n);
+ bms.reserve (n);
+
+ for (pair<auxiliary_config_machine, string>& pm: picked_machines)
+ {
+ const machine_header_manifest& m (*pm.first.machine);
+ tms.push_back (auxiliary_machine {m.name, move (pm.second)});
+ bms.push_back (build_machine {m.name, m.summary});
+ }
}
return collect_auxiliaries_result {
@@ -1603,8 +1650,40 @@ handle (request& rq, response& rs)
//
bool package_built (false);
+ build_db_->load (*p, p->bot_keys_section);
+
for (const build_package_config& pc: p->configs)
{
+ // If this is a custom bot, then skip this configuration if it
+ // doesn't contain this bot's public key in its custom bot keys
+ // list. Otherwise (this is a default bot), skip this
+ // configuration if its custom bot keys list is not empty.
+ //
+ {
+ const build_package_bot_keys& bks (
+ pc.effective_bot_keys (p->bot_keys));
+
+ if (custom_bot)
+ {
+ assert (agent_fp); // Wouldn't be here otherwise.
+
+ if (find_if (
+ bks.begin (), bks.end (),
+ [&agent_fp] (const lazy_shared_ptr<build_public_key>& k)
+ {
+ return k.object_id ().fingerprint == *agent_fp;
+ }) == bks.end ())
+ {
+ continue;
+ }
+ }
+ else
+ {
+ if (!bks.empty ())
+ continue;
+ }
+ }
+
pkg_config = pc.name;
// Iterate through the built configurations and erase them from the
@@ -1647,29 +1726,33 @@ handle (request& rq, response& rs)
// the package configuration and for which all the requested
// auxiliary machines can be provided.
//
- auto i (configs.begin ());
- auto e (configs.end ());
+ const config_machine* cm (nullptr);
+ optional<collect_auxiliaries_result> aux;
build_db_->load (*p, p->constraints_section);
- optional<collect_auxiliaries_result> aux;
- for (; i != e; ++i)
+ for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i)
{
- const build_target_config& tc (*i->second.config);
+ cm = &i->second;
+ const build_target_config& tc (*cm->config);
- if (!exclude (pc, p->builds, p->constraints, tc) &&
- (aux = collect_auxiliaries (p, pc, tc)))
- break;
+ if (!exclude (pc, p->builds, p->constraints, tc))
+ {
+ if (!p->auxiliaries_section.loaded ())
+ build_db_->load (*p, p->auxiliaries_section);
+
+ if ((aux = collect_auxiliaries (p, pc, tc)))
+ break;
+ }
}
- if (i != e)
+ if (aux)
{
- config_machine& cm (i->second);
- machine_header_manifest& mh (*cm.machine);
+ machine_header_manifest& mh (*cm->machine);
build_id bid (move (id),
- cm.config->target,
- cm.config->name,
+ cm->config->target,
+ cm->config->name,
move (pkg_config),
move (toolchain_name),
toolchain_version);
@@ -1704,8 +1787,8 @@ handle (request& rq, response& rs)
build_machine {
mh.name, move (mh.summary)},
move (aux->build_auxiliary_machines),
- controller_checksum (*cm.config),
- machine_checksum (*cm.machine));
+ controller_checksum (*cm->config),
+ machine_checksum (*cm->machine));
build_db_->persist (b);
}
@@ -1753,8 +1836,8 @@ handle (request& rq, response& rs)
b->auxiliary_machines =
move (aux->build_auxiliary_machines);
- string ccs (controller_checksum (*cm.config));
- string mcs (machine_checksum (*cm.machine));
+ string ccs (controller_checksum (*cm->config));
+ string mcs (machine_checksum (*cm->machine));
// Issue the hard rebuild if it is forced or the
// configuration or machine has changed.
@@ -1831,7 +1914,7 @@ handle (request& rq, response& rs)
move (aux->tests),
move (aux->task_auxiliary_machines),
move (bp.interactive),
- cm);
+ *cm);
task_build = move (b);
task_package = move (p);
@@ -1971,13 +2054,17 @@ handle (request& rq, response& rs)
(t->interactive.has_value () ==
(imode == interactive_mode::true_))))
{
+ const build_target_config& tc (*cm.config);
+
build_db_->load (*p, p->constraints_section);
- const build_target_config& tc (*cm.config);
+ if (exclude (*pc, p->builds, p->constraints, tc))
+ continue;
+
+ build_db_->load (*p, p->auxiliaries_section);
- optional<collect_auxiliaries_result> aux;
- if (!exclude (*pc, p->builds, p->constraints, tc) &&
- (aux = collect_auxiliaries (p, *pc, tc)))
+ if (optional<collect_auxiliaries_result> aux =
+ collect_auxiliaries (p, *pc, tc))
{
assert (b->status);
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 51b21c6..91923e5 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -541,7 +541,7 @@ handle (request& rq, response& rs)
//
builds = false;
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
const build_class_exprs& exprs (pc.effective_builds (pkg->builds));
@@ -726,7 +726,7 @@ handle (request& rq, response& rs)
s << H3 << "Builds" << ~H3
<< DIV(ID="builds");
- auto exclude = [&pkg, this] (const build_package_config& pc,
+ auto exclude = [&pkg, this] (const package_build_config& pc,
const build_target_config& tc,
string* rs = nullptr)
{
@@ -767,7 +767,7 @@ handle (request& rq, response& rs)
query sq (false);
set<config_toolchain> unbuilt_configs;
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
for (const auto& bc: *target_conf_map_)
{
@@ -886,7 +886,7 @@ handle (request& rq, response& rs)
//
if (!tn->interactive)
{
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
for (const auto& tc: *target_conf_)
{