aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--clean/clean.cxx7
-rw-r--r--libbrep/build-extra.sql45
-rw-r--r--libbrep/build-package.hxx110
-rw-r--r--libbrep/build.hxx2
-rw-r--r--libbrep/build.xml2
-rw-r--r--libbrep/common.cxx10
-rw-r--r--libbrep/common.hxx191
-rwxr-xr-xlibbrep/odb.sh4
-rw-r--r--libbrep/package.cxx31
-rw-r--r--libbrep/package.hxx126
-rw-r--r--libbrep/package.xml135
-rw-r--r--load/load.cli16
-rw-r--r--load/load.cxx150
-rw-r--r--mod/build-config-module.hxx3
-rw-r--r--mod/build-result-module.cxx145
-rw-r--r--mod/build-target-config.cxx8
-rw-r--r--mod/build-target-config.hxx23
-rw-r--r--mod/mod-build-task.cxx167
-rw-r--r--mod/mod-package-version-details.cxx8
-rw-r--r--monitor/monitor.cxx93
20 files changed, 1067 insertions, 209 deletions
diff --git a/clean/clean.cxx b/clean/clean.cxx
index 55fb59b..828ae4b 100644
--- a/clean/clean.cxx
+++ b/clean/clean.cxx
@@ -480,8 +480,8 @@ namespace brep
auto tenant_ids (pq.execute ());
if ((ne = !tenant_ids.empty ()))
{
- // Cache tenant ids and erase packages, repositories, and tenants at
- // once.
+ // Cache tenant ids and erase packages, repositories, public keys, and
+ // tenants at once.
//
strings tids;
tids.reserve (tenant_ids.size ());
@@ -497,6 +497,9 @@ namespace brep
db.erase_query<repository> (
query<repository>::id.tenant.in_range (tids.begin (), tids.end ()));
+ db.erase_query<public_key> (
+ query<public_key>::id.tenant.in_range (tids.begin (), tids.end ()));
+
db.erase_query<tenant> (
query<tenant>::id.in_range (tids.begin (), tids.end ()));
}
diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql
index a931f31..9e51a51 100644
--- a/libbrep/build-extra.sql
+++ b/libbrep/build-extra.sql
@@ -6,6 +6,8 @@
-- package-extra.sql file for details.
--
+DROP FOREIGN TABLE IF EXISTS build_package_config_bot_keys;
+
DROP FOREIGN TABLE IF EXISTS build_package_config_auxiliaries;
DROP FOREIGN TABLE IF EXISTS build_package_config_constraints;
@@ -14,6 +16,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_config_builds;
DROP FOREIGN TABLE IF EXISTS build_package_configs;
+DROP FOREIGN TABLE IF EXISTS build_package_bot_keys;
+
DROP FOREIGN TABLE IF EXISTS build_package_auxiliaries;
DROP FOREIGN TABLE IF EXISTS build_package_constraints;
@@ -30,6 +34,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_requirements;
DROP FOREIGN TABLE IF EXISTS build_package;
+DROP FOREIGN TABLE IF EXISTS build_public_key;
+
DROP FOREIGN TABLE IF EXISTS build_repository;
DROP FOREIGN TABLE IF EXISTS build_tenant;
@@ -64,6 +70,14 @@ CREATE FOREIGN TABLE build_repository (
certificate_fingerprint TEXT NULL)
SERVER package_server OPTIONS (table_name 'repository');
+-- The foreign table for build_public_key object.
+--
+CREATE FOREIGN TABLE build_public_key (
+ tenant TEXT NOT NULL,
+ fingerprint TEXT NOT NULL,
+ "data" TEXT NOT NULL)
+SERVER package_server OPTIONS (table_name 'public_key');
+
-- The foreign table for build_package object.
--
CREATE FOREIGN TABLE build_package (
@@ -84,7 +98,8 @@ CREATE FOREIGN TABLE build_package (
build_error_email_comment TEXT NULL,
internal_repository_tenant TEXT NULL,
internal_repository_canonical_name TEXT NULL,
- buildable BOOLEAN NOT NULL)
+ buildable BOOLEAN NOT NULL,
+ custom_bot BOOLEAN NULL)
SERVER package_server OPTIONS (table_name 'package');
-- The foreign tables for the build_package object requirements member (that
@@ -214,6 +229,21 @@ CREATE FOREIGN TABLE build_package_auxiliaries (
comment TEXT NOT NULL)
SERVER package_server OPTIONS (table_name 'package_build_auxiliaries');
+-- The foreign table for the build_package object bot_keys member (that is
+-- of a container type).
+--
+CREATE FOREIGN TABLE build_package_bot_keys (
+ tenant TEXT NOT NULL,
+ name CITEXT NOT NULL,
+ version_epoch INTEGER NOT NULL,
+ version_canonical_upstream TEXT NOT NULL,
+ version_canonical_release TEXT NOT NULL COLLATE "C",
+ version_revision INTEGER NOT NULL,
+ index BIGINT NOT NULL,
+ key_tenant TEXT NOT NULL,
+ key_fingerprint TEXT NOT NULL)
+SERVER package_server OPTIONS (table_name 'package_build_bot_keys');
+
-- The foreign tables for the build_package object configs member (that is a
-- container of values containing containers.
--
@@ -277,3 +307,16 @@ CREATE FOREIGN TABLE build_package_config_auxiliaries (
config TEXT NOT NULL,
comment TEXT NOT NULL)
SERVER package_server OPTIONS (table_name 'package_build_config_auxiliaries');
+
+CREATE FOREIGN TABLE build_package_config_bot_keys (
+ tenant TEXT NOT NULL,
+ name CITEXT NOT NULL,
+ version_epoch INTEGER NOT NULL,
+ version_canonical_upstream TEXT NOT NULL,
+ version_canonical_release TEXT NOT NULL COLLATE "C",
+ version_revision INTEGER NOT NULL,
+ config_index BIGINT NOT NULL,
+ index BIGINT NOT NULL,
+ key_tenant TEXT NOT NULL,
+ key_fingerprint TEXT NOT NULL)
+SERVER package_server OPTIONS (table_name 'package_build_config_bot_keys');
diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx
index 08fb781..b7bea87 100644
--- a/libbrep/build-package.hxx
+++ b/libbrep/build-package.hxx
@@ -23,9 +23,11 @@ namespace brep
//
// The mapping is established in build-extra.sql. We also explicitly mark
// non-primary key foreign-mapped members in the source object.
- //
+
// Foreign object that is mapped to a subset of the tenant object.
//
+ // Note: table created manually thus assign table name explicitly.
+ //
#pragma db object table("build_tenant") pointer(shared_ptr)
class build_tenant
{
@@ -50,6 +52,8 @@ namespace brep
// Foreign object that is mapped to a subset of the repository object.
//
+ // Note: table created manually thus assign table name explicitly.
+ //
#pragma db object table("build_repository") pointer(shared_ptr) readonly
class build_repository
{
@@ -75,6 +79,54 @@ namespace brep
build_repository (): canonical_name (id.canonical_name) {}
};
+ // Foreign object that is mapped to a subset of the public key object.
+ //
+ // Note: table created manually thus assign table name explicitly.
+ //
+ #pragma db object table("build_public_key") pointer(shared_ptr) readonly
+ class build_public_key: public string
+ {
+ public:
+ public_key_id id;
+
+ // Database mapping.
+ //
+ #pragma db member(id) id column("")
+
+ #pragma db member(data) virtual(string) access(this)
+
+ private:
+ friend class odb::access;
+ build_public_key () = default;
+ };
+
+ // build_package_config
+ //
+ using build_package_config =
+ build_package_config_template<lazy_shared_ptr<build_public_key>>;
+
+ using build_package_configs =
+ build_package_configs_template<lazy_shared_ptr<build_public_key>>;
+
+ #pragma db value(build_package_config) definition
+
+ #pragma db member(build_package_config::builds) transient
+ #pragma db member(build_package_config::constraints) transient
+ #pragma db member(build_package_config::auxiliaries) transient
+ #pragma db member(build_package_config::bot_keys) transient
+
+ // build_package_bot_keys
+ //
+ using build_package_bot_keys = vector<lazy_shared_ptr<build_public_key>>;
+ using build_package_bot_key_key = odb::nested_key<build_package_bot_keys>;
+
+ using build_package_bot_keys_map =
+ std::map<build_package_bot_key_key, lazy_shared_ptr<build_public_key>>;
+
+ #pragma db value(build_package_bot_key_key)
+ #pragma db member(build_package_bot_key_key::outer) column("config_index")
+ #pragma db member(build_package_bot_key_key::inner) column("index")
+
// Forward declarations.
//
class build_package;
@@ -107,6 +159,8 @@ namespace brep
// Foreign object that is mapped to a subset of the package object.
//
+ // Note: table created manually thus assign table name explicitly.
+ //
#pragma db object table("build_package") pointer(shared_ptr) readonly session
class build_package
{
@@ -132,21 +186,26 @@ namespace brep
lazy_shared_ptr<build_repository> internal_repository;
bool buildable;
+ optional<bool> custom_bot;
// Mapped to the package object builds, build_constraints,
- // build_auxiliaries, and build_configs members using the PostgreSQL
- // foreign table mechanism.
+ // build_auxiliaries, bot_keys, and build_configs members using the
+ // PostgreSQL foreign table mechanism.
//
- build_class_exprs builds;
- build_constraints constraints;
- build_auxiliaries auxiliaries;
- build_package_configs configs;
-
- // Group the builds and constraints members of this object as well as of
- // the nested configs entries for an explicit load. Note that the configs
- // top-level members are loaded implicitly.
+ build_class_exprs builds;
+ build_constraints constraints;
+ build_auxiliaries auxiliaries;
+ build_package_bot_keys bot_keys;
+ build_package_configs configs;
+
+ // Group the builds/constraints, auxiliaries, and bot_keys members of this
+ // object together with their respective nested configs entries into the
+ // separate sections for an explicit load. Note that the configs top-level
+ // members are loaded implicitly.
//
odb::section constraints_section;
+ odb::section auxiliaries_section;
+ odb::section bot_keys_section;
bool
internal () const noexcept {return internal_repository != nullptr;}
@@ -194,7 +253,7 @@ namespace brep
#pragma db member(requirements_tests_section) load(lazy) update(always)
- // builds, constraints, and auxiliaries
+ // builds, constraints, auxiliaries, and bot_keys
//
#pragma db member(builds) id_column("") value_column("") \
section(constraints_section)
@@ -203,13 +262,16 @@ namespace brep
section(constraints_section)
#pragma db member(auxiliaries) id_column("") value_column("") \
- section(constraints_section)
+ section(auxiliaries_section)
+
+ #pragma db member(bot_keys) id_column("") value_column("key_") \
+ section(bot_keys_section)
// configs
//
- // Note that build_package_config::{builds,constraints,auxiliaries} are
- // persisted/loaded via the separate nested containers (see commons.hxx
- // for details).
+ // Note that build_package_config::{builds,constraints,auxiliaries,bot_keys}
+ // are persisted/loaded via the separate nested containers (see
+ // commons.hxx for details).
//
#pragma db member(configs) id_column("") value_column("config_")
@@ -244,9 +306,23 @@ namespace brep
odb::nested_set (as, std::move (?)); \
move (as).to_configs (this.configs)) \
id_column("") key_column("") value_column("") \
- section(constraints_section)
+ section(auxiliaries_section)
+
+ #pragma db member(config_bot_keys) \
+ virtual(build_package_bot_keys_map) \
+ after(config_auxiliaries) \
+ get(odb::nested_get ( \
+ brep::build_package_config_bot_keys (this.configs))) \
+ set(brep::build_package_config_bot_keys< \
+ lazy_shared_ptr<brep::build_public_key>> bks; \
+ odb::nested_set (bks, std::move (?)); \
+ move (bks).to_configs (this.configs)) \
+ id_column("") key_column("") value_column("key_") \
+ section(bot_keys_section)
#pragma db member(constraints_section) load(lazy) update(always)
+ #pragma db member(auxiliaries_section) load(lazy) update(always)
+ #pragma db member(bot_keys_section) load(lazy) update(always)
private:
friend class odb::access;
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index 4c470cd..af49c03 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -28,7 +28,7 @@
//
#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 26, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 27, closed)
// We have to keep these mappings at the global scope instead of inside the
// brep namespace because they need to be also effective in the bbot namespace
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 815c915..1eba85a 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,6 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <changeset version="27"/>
+
<changeset version="26"/>
<changeset version="25">
diff --git a/libbrep/common.cxx b/libbrep/common.cxx
index c97a346..4f729a3 100644
--- a/libbrep/common.cxx
+++ b/libbrep/common.cxx
@@ -32,14 +32,4 @@ namespace brep
else if (r == "unbuildable") return unbuildable_reason::unbuildable;
else throw invalid_argument ("invalid unbuildable reason '" + r + '\'');
}
-
- build_package_config*
- find (const string& name, build_package_configs& cs)
- {
- auto i (find_if (cs.begin (), cs.end (),
- [&name] (const build_package_config& c)
- {return c.name == name;}));
-
- return i != cs.end () ? &*i : nullptr;
- }
}
diff --git a/libbrep/common.hxx b/libbrep/common.hxx
index ea18fd4..1433c8c 100644
--- a/libbrep/common.hxx
+++ b/libbrep/common.hxx
@@ -326,6 +326,19 @@ namespace brep
: tenant (move (t)), canonical_name (move (n)) {}
};
+ // public_key_id
+ //
+ #pragma db value
+ struct public_key_id
+ {
+ string tenant;
+ string fingerprint;
+
+ public_key_id () = default;
+ public_key_id (string t, string f)
+ : tenant (move (t)), fingerprint (move (f)) {}
+ };
+
// build_class_expr
//
using bpkg::build_class_expr;
@@ -370,31 +383,38 @@ namespace brep
#pragma db value(email) definition
#pragma db member(email::value) virtual(string) before access(this) column("")
- // build_package_config
+ // build_package_config_template
//
- using build_package_config = bpkg::build_package_config;
-
- #pragma db value(build_package_config) definition
+ using bpkg::build_package_config_template;
// 1 for the default configuration which is always present.
//
- using build_package_configs = small_vector<build_package_config, 1>;
+ template <typename K>
+ using build_package_configs_template =
+ small_vector<build_package_config_template<K>, 1>;
// Return the address of the configuration object with the specified name,
// if present, and NULL otherwise.
//
- build_package_config*
- find (const string& name, build_package_configs&);
+ template <typename K>
+ inline build_package_config_template<K>*
+ find (const string& name, build_package_configs_template<K>& cs)
+ {
+ auto i (find_if (cs.begin (), cs.end (),
+ [&name] (const build_package_config_template<K>& c)
+ {return c.name == name;}));
+
+ return i != cs.end () ? &*i : nullptr;
+ }
// Note that ODB doesn't support containers of value types which contain
// containers. Thus, we will persist/load
- // package_build_config::{builds,constraint,auxiliaries} via the separate
- // nested containers using the adapter classes.
- //
- // build_package_config::builds
+ // build_package_config_template<K>::{builds,constraint,auxiliaries,bot_keys}
+ // via the separate nested containers using the adapter classes.
//
- #pragma db member(build_package_config::builds) transient
+ // build_package_config_template<K>::builds
+ //
using build_class_expr_key = odb::nested_key<build_class_exprs>;
using build_class_exprs_map = std::map<build_class_expr_key, build_class_expr>;
@@ -402,24 +422,27 @@ namespace brep
#pragma db member(build_class_expr_key::outer) column("config_index")
#pragma db member(build_class_expr_key::inner) column("index")
- // Adapter for build_package_config::builds.
+ // Adapter for build_package_config_template<K>::builds.
+ //
+ // Note: 1 as for build_package_configs_template.
//
- class build_package_config_builds:
- public small_vector<build_class_exprs, 1> // 1 as for build_package_configs.
+ class build_package_config_builds: public small_vector<build_class_exprs, 1>
{
public:
build_package_config_builds () = default;
+ template <typename K>
explicit
- build_package_config_builds (const build_package_configs& cs)
+ build_package_config_builds (const build_package_configs_template<K>& cs)
{
reserve (cs.size ());
- for (const build_package_config& c: cs)
+ for (const build_package_config_template<K>& c: cs)
push_back (c.builds);
}
+ template <typename K>
void
- to_configs (build_package_configs& cs) &&
+ to_configs (build_package_configs_template<K>& cs) &&
{
// Note that the empty trailing entries will be missing (see ODB's
// nested-container.hxx for details).
@@ -432,10 +455,8 @@ namespace brep
}
};
- // build_package_config::constraints
+ // build_package_config_template<K>::constraints
//
- #pragma db member(build_package_config::constraints) transient
-
using build_constraint_key = odb::nested_key<build_constraints>;
using build_constraints_map = std::map<build_constraint_key, build_constraint>;
@@ -443,24 +464,29 @@ namespace brep
#pragma db member(build_constraint_key::outer) column("config_index")
#pragma db member(build_constraint_key::inner) column("index")
- // Adapter for build_package_config::constraints.
+ // Adapter for build_package_config_template<K>::constraints.
+ //
+ // Note: 1 as for build_package_configs_template.
//
class build_package_config_constraints:
- public small_vector<build_constraints, 1> // 1 as for build_package_configs.
+ public small_vector<build_constraints, 1>
{
public:
build_package_config_constraints () = default;
+ template <typename K>
explicit
- build_package_config_constraints (const build_package_configs& cs)
+ build_package_config_constraints (
+ const build_package_configs_template<K>& cs)
{
reserve (cs.size ());
- for (const build_package_config& c: cs)
+ for (const build_package_config_template<K>& c: cs)
push_back (c.constraints);
}
+ template <typename K>
void
- to_configs (build_package_configs& cs) &&
+ to_configs (build_package_configs_template<K>& cs) &&
{
// Note that the empty trailing entries will be missing (see ODB's
// nested-container.hxx for details).
@@ -473,10 +499,8 @@ namespace brep
}
};
- // build_package_config::auxiliaries
+ // build_package_config_template<K>::auxiliaries
//
- #pragma db member(build_package_config::auxiliaries) transient
-
using build_auxiliary_key = odb::nested_key<build_auxiliaries>;
using build_auxiliaries_map = std::map<build_auxiliary_key, build_auxiliary>;
@@ -484,24 +508,29 @@ namespace brep
#pragma db member(build_auxiliary_key::outer) column("config_index")
#pragma db member(build_auxiliary_key::inner) column("index")
- // Adapter for build_package_config::auxiliaries.
+ // Adapter for build_package_config_template<K>::auxiliaries.
+ //
+ // Note: 1 as for build_package_configs_template.
//
class build_package_config_auxiliaries:
- public small_vector<build_auxiliaries, 1> // 1 as for build_package_configs.
+ public small_vector<build_auxiliaries, 1>
{
public:
build_package_config_auxiliaries () = default;
+ template <typename K>
explicit
- build_package_config_auxiliaries (const build_package_configs& cs)
+ build_package_config_auxiliaries (
+ const build_package_configs_template<K>& cs)
{
reserve (cs.size ());
- for (const build_package_config& c: cs)
+ for (const build_package_config_template<K>& c: cs)
push_back (c.auxiliaries);
}
+ template <typename K>
void
- to_configs (build_package_configs& cs) &&
+ to_configs (build_package_configs_template<K>& cs) &&
{
// Note that the empty trailing entries will be missing (see ODB's
// nested-container.hxx for details).
@@ -514,6 +543,40 @@ namespace brep
}
};
+ // build_package_config_template<K>::bot_keys
+ //
+ // Adapter for build_package_config_template<K>::bot_keys.
+ //
+ // Note: 1 as for build_package_configs_template.
+ //
+ template <typename K>
+ class build_package_config_bot_keys: public small_vector<vector<K>, 1>
+ {
+ public:
+ build_package_config_bot_keys () = default;
+
+ explicit
+ build_package_config_bot_keys (const build_package_configs_template<K>& cs)
+ {
+ this->reserve (cs.size ());
+ for (const build_package_config_template<K>& c: cs)
+ this->push_back (c.bot_keys);
+ }
+
+ void
+ to_configs (build_package_configs_template<K>& cs) &&
+ {
+ // Note that the empty trailing entries will be missing (see ODB's
+ // nested-container.hxx for details).
+ //
+ assert (this->size () <= cs.size ());
+
+ auto i (cs.begin ());
+ for (vector<K>& bks: *this)
+ i++->bot_keys = move (bks);
+ }
+ };
+
// The primary reason why a package is unbuildable by the build bot
// controller service.
//
@@ -611,13 +674,12 @@ namespace brep
// Version comparison operators.
//
- // They allow comparing objects that have epoch, canonical_upstream,
- // canonical_release, and revision data members. The idea is that this
- // works for both query members of types version and canonical_version.
- // Note, though, that the object revisions should be comparable (both
- // optional, numeric, etc), so to compare version to query member or
- // canonical_version you may need to explicitly convert the version object
- // to canonical_version.
+ // Compare objects that have epoch, canonical_upstream, canonical_release,
+ // and revision data members. The idea is that this works for both query
+ // members of types version and canonical_version. Note, though, that the
+ // object revisions should be comparable (both optional, numeric, etc), so
+ // to compare version to query member or canonical_version you may need to
+ // explicitly convert the version object to canonical_version.
//
template <typename T1, typename T2>
inline auto
@@ -769,10 +831,9 @@ namespace brep
return compare_version_lt (x.version, y.version, true);
}
- // They allow comparing objects that have tenant, name, and version data
- // members. The idea is that this works for both query members of package id
- // types (in particular in join conditions) as well as for values of
- // package_id type.
+ // Compare objects that have tenant, name, and version data members. The
+ // idea is that this works for both query members of package id types (in
+ // particular in join conditions) as well as for values of package_id type.
//
template <typename T1, typename T2>
inline auto
@@ -852,10 +913,10 @@ namespace brep
return x.canonical_name.compare (y.canonical_name) < 0;
}
- // They allow comparing objects that have tenant and canonical_name data
- // members. The idea is that this works for both query members of repository
- // id types (in particular in join conditions) as well as for values of
- // repository_id type.
+ // Compare objects that have tenant and canonical_name data members. The
+ // idea is that this works for both query members of repository id types (in
+ // particular in join conditions) as well as for values of repository_id
+ // type.
//
template <typename T1, typename T2>
inline auto
@@ -872,6 +933,38 @@ namespace brep
{
return x.tenant != y.tenant || x.canonical_name != y.canonical_name;
}
+
+ // Public key id comparison operators.
+ //
+ inline bool
+ operator< (const public_key_id& x, const public_key_id& y)
+ {
+ if (int r = x.tenant.compare (y.tenant))
+ return r < 0;
+
+ return x.fingerprint.compare (y.fingerprint) < 0;
+ }
+
+ // Compare objects that have tenant and fingerprint data members. The idea
+ // is that this works for both query members of public key id types (in
+ // particular in join conditions) as well as for values of public_key_id
+ // type.
+ //
+ template <typename T1, typename T2>
+ inline auto
+ operator== (const T1& x, const T2& y)
+ -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint)
+ {
+ return x.tenant == y.tenant && x.fingerprint == y.fingerprint;
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ operator!= (const T1& x, const T2& y)
+ -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint)
+ {
+ return x.tenant != y.tenant || x.fingerprint != y.fingerprint;
+ }
}
#endif // LIBBREP_COMMON_HXX
diff --git a/libbrep/odb.sh b/libbrep/odb.sh
index 9ee11fa..608ca41 100755
--- a/libbrep/odb.sh
+++ b/libbrep/odb.sh
@@ -53,7 +53,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query \
--hxx-prologue '#include <libbrep/common-traits.hxx>' \
-DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \
--include-with-brackets --include-prefix libbrep \
- --guard-prefix LIBBREP \
+ --guard-prefix LIBBREP \
common.hxx
$odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \
@@ -74,7 +74,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \
--odb-epilogue '#include <libbrep/wrapper-traits.hxx>' \
--generate-prepared -DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \
--include-with-brackets --include-prefix libbrep \
- --guard-prefix LIBBREP \
+ --guard-prefix LIBBREP \
build.hxx
$odb "${inc[@]}" -d pgsql --std c++14 --generate-query \
diff --git a/libbrep/package.cxx b/libbrep/package.cxx
index 37795f0..4eb6fe8 100644
--- a/libbrep/package.cxx
+++ b/libbrep/package.cxx
@@ -82,7 +82,8 @@ namespace brep
build_class_exprs bs,
build_constraints_type bc,
build_auxiliaries_type ac,
- build_package_configs bcs,
+ package_build_bot_keys bk,
+ package_build_configs bcs,
optional<path> lc,
optional<string> fr,
optional<string> sh,
@@ -116,6 +117,7 @@ namespace brep
builds (move (bs)),
build_constraints (move (bc)),
build_auxiliaries (move (ac)),
+ build_bot_keys (move (bk)),
build_configs (move (bcs)),
internal_repository (move (rp)),
location (move (lc)),
@@ -134,6 +136,31 @@ namespace brep
buildable = !unbuildable_reason;
+ // If the package is buildable deduce the custom_bot flag.
+ //
+ if (buildable)
+ {
+ for (const package_build_config& bc: build_configs)
+ {
+ bool custom (!bc.effective_bot_keys (build_bot_keys).empty ());
+
+ if (!custom_bot)
+ {
+ custom_bot = custom;
+ }
+ //
+ // If both the custom and default bots are used by the package, then
+ // reset the custom_bot flag to nullopt and bail out from the build
+ // package configurations loop.
+ //
+ else if (*custom_bot != custom)
+ {
+ custom_bot = nullopt;
+ break;
+ }
+ }
+ }
+
assert (internal_repository->internal);
}
@@ -143,7 +170,7 @@ namespace brep
build_class_exprs bs,
build_constraints_type bc,
build_auxiliaries_type ac,
- build_package_configs bcs,
+ package_build_configs bcs,
shared_ptr<repository_type> rp)
: id (rp->tenant, move (nm), vr),
tenant (id.tenant),
diff --git a/libbrep/package.hxx b/libbrep/package.hxx
index b8c3c33..3878530 100644
--- a/libbrep/package.hxx
+++ b/libbrep/package.hxx
@@ -20,7 +20,7 @@
//
#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27
-#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 32, closed)
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed)
namespace brep
{
@@ -248,8 +248,8 @@ namespace brep
string id;
- // If true, display the packages in the web interface only in the tenant
- // view mode.
+ // If this flag is true, then display the packages in the web interface
+ // only in the tenant view mode.
//
bool private_; // Note: foreign-mapped in build.
@@ -456,6 +456,58 @@ namespace brep
#pragma db member(text) column("")
};
+ // Tweak public_key_id mapping to include a constraint (this only affects the
+ // database schema).
+ //
+ #pragma db member(public_key_id::tenant) points_to(tenant)
+
+ #pragma db object pointer(shared_ptr) session
+ class public_key: public string
+ {
+ public:
+ public_key (string tenant, string fingerprint, string key)
+ : string (move (key)), id (move (tenant), move (fingerprint)) {}
+
+ public_key_id id;
+
+ // Database mapping.
+ //
+ #pragma db member(id) id column("")
+
+ #pragma db member(data) virtual(string) access(this)
+
+ private:
+ friend class odb::access;
+ public_key () = default;
+ };
+
+ // package_build_config
+ //
+ using package_build_config =
+ build_package_config_template<lazy_shared_ptr<public_key>>;
+
+ using package_build_configs =
+ build_package_configs_template<lazy_shared_ptr<public_key>>;
+
+ #pragma db value(package_build_config) definition
+
+ #pragma db member(package_build_config::builds) transient
+ #pragma db member(package_build_config::constraints) transient
+ #pragma db member(package_build_config::auxiliaries) transient
+ #pragma db member(package_build_config::bot_keys) transient
+
+ // package_build_bot_keys
+ //
+ using package_build_bot_keys = vector<lazy_shared_ptr<public_key>>;
+ using package_build_bot_key_key = odb::nested_key<package_build_bot_keys>;
+
+ using package_build_bot_keys_map = std::map<package_build_bot_key_key,
+ lazy_shared_ptr<public_key>>;
+
+ #pragma db value(package_build_bot_key_key)
+ #pragma db member(package_build_bot_key_key::outer) column("config_index")
+ #pragma db member(package_build_bot_key_key::inner) column("index")
+
// Tweak package_id mapping to include a constraint (this only affects the
// database schema).
//
@@ -507,7 +559,8 @@ namespace brep
build_class_exprs,
build_constraints_type,
build_auxiliaries_type,
- build_package_configs,
+ package_build_bot_keys,
+ package_build_configs,
optional<path> location,
optional<string> fragment,
optional<string> sha256sum,
@@ -535,7 +588,7 @@ namespace brep
build_class_exprs,
build_constraints_type,
build_auxiliaries_type,
- build_package_configs,
+ package_build_configs,
shared_ptr<repository_type>);
bool
@@ -589,15 +642,25 @@ namespace brep
requirements_type requirements; // Note: foreign-mapped in build.
small_vector<test_dependency, 1> tests; // Note: foreign-mapped in build.
- // Common build classes, constraints, and auxiliaries that apply to all
- // configurations unless overridden.
+ // Common build classes, constraints, auxiliaries, and bot keys that apply
+ // to all configurations unless overridden.
//
build_class_exprs builds; // Note: foreign-mapped in build.
build_constraints_type build_constraints; // Note: foreign-mapped in build.
build_auxiliaries_type build_auxiliaries; // Note: foreign-mapped in build.
+ package_build_bot_keys build_bot_keys; // Note: foreign-mapped in build.
+ package_build_configs build_configs; // Note: foreign-mapped in build.
- build_package_configs build_configs; // Note: foreign-mapped in build.
-
+ // Group the build_configs, builds, and build_constraints members of this
+ // object together with their respective nested configs entries into the
+ // separate section for an explicit load.
+ //
+ // Note that while the build auxiliaries and bot keys are persisted via
+ // the newly created package objects, they are only used via the
+ // foreign-mapped build_package objects (see build-package.hxx for
+ // details). Thus, we add them to the never-loaded unused_section (see
+ // below).
+ //
odb::section build_section;
// Note that it is foreign-mapped in build.
@@ -628,6 +691,18 @@ namespace brep
bool buildable; // Note: foreign-mapped in build.
optional<brep::unbuildable_reason> unbuildable_reason;
+ // If this flag is true, then all the package configurations are buildable
+ // with the custom build bots. If false, then all configurations are
+ // buildable with the default bots. If nullopt, then some configurations
+ // are buildable with the custom and some with the default build bots.
+ //
+ // Note: meaningless if buildable is false.
+ //
+ optional<bool> custom_bot; // Note: foreign-mapped in build.
+
+ private:
+ odb::section unused_section;
+
// Database mapping.
//
#pragma db member(id) id column("")
@@ -750,13 +825,19 @@ namespace brep
// build_auxiliaries
//
#pragma db member(build_auxiliaries) id_column("") value_column("") \
- section(build_section)
+ section(unused_section)
+
+ // build_bot_keys
+ //
+ #pragma db member(build_bot_keys) \
+ id_column("") value_column("key_") value_not_null \
+ section(unused_section)
// build_configs
//
- // Note that build_package_config::{builds,constraints,auxiliaries} are
- // persisted/loaded via the separate nested containers (see commons.hxx
- // for details).
+ // Note that package_build_config::{builds,constraints,auxiliaries,
+ // bot_keys} are persisted/loaded via the separate nested containers (see
+ // commons.hxx for details).
//
#pragma db member(build_configs) id_column("") value_column("config_") \
section(build_section)
@@ -792,9 +873,22 @@ namespace brep
odb::nested_set (as, std::move (?)); \
move (as).to_configs (this.build_configs)) \
id_column("") key_column("") value_column("") \
- section(build_section)
-
- #pragma db member(build_section) load(lazy) update(always)
+ section(unused_section)
+
+ #pragma db member(build_config_bot_keys) \
+ virtual(package_build_bot_keys_map) \
+ after(build_config_auxiliaries) \
+ get(odb::nested_get ( \
+ brep::build_package_config_bot_keys (this.build_configs))) \
+ set(brep::build_package_config_bot_keys< \
+ lazy_shared_ptr<brep::public_key>> bks; \
+ odb::nested_set (bks, std::move (?)); \
+ move (bks).to_configs (this.build_configs)) \
+ id_column("") key_column("") value_column("key_") value_not_null \
+ section(unused_section)
+
+ #pragma db member(build_section) load(lazy) update(always)
+ #pragma db member(unused_section) load(lazy) update(manual)
// other_repositories
//
diff --git a/libbrep/package.xml b/libbrep/package.xml
index b66a66d..96e93a7 100644
--- a/libbrep/package.xml
+++ b/libbrep/package.xml
@@ -1,4 +1,139 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
+ <changeset version="33">
+ <add-table name="public_key" kind="object">
+ <column name="tenant" type="TEXT" null="false"/>
+ <column name="fingerprint" type="TEXT" null="false"/>
+ <column name="data" type="TEXT" null="false"/>
+ <primary-key>
+ <column name="tenant"/>
+ <column name="fingerprint"/>
+ </primary-key>
+ <foreign-key name="tenant_fk" deferrable="DEFERRED">
+ <column name="tenant"/>
+ <references table="tenant">
+ <column name="id"/>
+ </references>
+ </foreign-key>
+ </add-table>
+ <alter-table name="package">
+ <add-column name="custom_bot" type="BOOLEAN" null="true"/>
+ </alter-table>
+ <add-table name="package_build_bot_keys" kind="container">
+ <column name="tenant" type="TEXT" null="false"/>
+ <column name="name" type="CITEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="key_tenant" type="TEXT" null="false"/>
+ <column name="key_fingerprint" type="TEXT" null="false"/>
+ <foreign-key name="tenant_fk" deferrable="DEFERRED">
+ <column name="tenant"/>
+ <references table="tenant">
+ <column name="id"/>
+ </references>
+ </foreign-key>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_build_bot_keys_object_id_i">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_build_bot_keys_index_i">
+ <column name="index"/>
+ </index>
+ <foreign-key name="key_tenant_fk" deferrable="DEFERRED">
+ <column name="key_tenant"/>
+ <references table="tenant">
+ <column name="id"/>
+ </references>
+ </foreign-key>
+ <foreign-key name="key_fk" deferrable="DEFERRED">
+ <column name="key_tenant"/>
+ <column name="key_fingerprint"/>
+ <references table="public_key">
+ <column name="tenant"/>
+ <column name="fingerprint"/>
+ </references>
+ </foreign-key>
+ </add-table>
+ <add-table name="package_build_config_bot_keys" kind="container">
+ <column name="tenant" type="TEXT" null="false"/>
+ <column name="name" type="CITEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="config_index" type="BIGINT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="key_tenant" type="TEXT" null="false"/>
+ <column name="key_fingerprint" type="TEXT" null="false"/>
+ <foreign-key name="tenant_fk" deferrable="DEFERRED">
+ <column name="tenant"/>
+ <references table="tenant">
+ <column name="id"/>
+ </references>
+ </foreign-key>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_build_config_bot_keys_object_id_i">
+ <column name="tenant"/>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <foreign-key name="key_tenant_fk" deferrable="DEFERRED">
+ <column name="key_tenant"/>
+ <references table="tenant">
+ <column name="id"/>
+ </references>
+ </foreign-key>
+ <foreign-key name="key_fk" deferrable="DEFERRED">
+ <column name="key_tenant"/>
+ <column name="key_fingerprint"/>
+ <references table="public_key">
+ <column name="tenant"/>
+ <column name="fingerprint"/>
+ </references>
+ </foreign-key>
+ </add-table>
+ </changeset>
+
<changeset version="32">
<alter-table name="tenant">
<add-column name="build_toolchain_name" type="TEXT" null="true"/>
diff --git a/load/load.cli b/load/load.cli
index b9aa769..99d76f6 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -167,6 +167,22 @@ class options
this option to specify multiple package manager options."
}
+ brep::path openssl = "openssl"
+ {
+ "<path>",
+ "The openssl program to be used for crypto operations. You can also
+ specify additional options that should be passed to the openssl program
+ with \cb{openssl-option}. If the openssl program is not explicitly
+ specified, then \cb{brep-load} will use \cb{openssl} by default."
+ }
+
+ brep::strings openssl-option
+ {
+ "<opt>",
+ "Additional option to be passed to the openssl program (see \cb{openssl}
+ for details). Repeat this option to specify multiple openssl options."
+ }
+
std::string --pager // String to allow empty value.
{
"<path>",
diff --git a/load/load.cxx b/load/load.cxx
index 14b8374..5b4692c 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -20,6 +20,7 @@
#include <libbutl/pager.hxx>
#include <libbutl/sha256.hxx>
#include <libbutl/process.hxx>
+#include <libbutl/openssl.hxx>
#include <libbutl/fdstream.hxx>
#include <libbutl/filesystem.hxx>
#include <libbutl/tab-parser.hxx>
@@ -364,7 +365,8 @@ repository_info (const options& lo, const string& rl, const cstrings& options)
// the repository. Should be called once per repository.
//
static void
-load_packages (const shared_ptr<repository>& rp,
+load_packages (const options& lo,
+ const shared_ptr<repository>& rp,
const repository_location& cl,
database& db,
bool ignore_unknown,
@@ -421,10 +423,12 @@ load_packages (const shared_ptr<repository>& rp,
using brep::dependency_alternative;
using brep::dependency_alternatives;
+ const string& tenant (rp->tenant);
+
for (package_manifest& pm: pms)
{
shared_ptr<package> p (
- db.find<package> (package_id (rp->tenant, pm.name, pm.version)));
+ db.find<package> (package_id (tenant, pm.name, pm.version)));
// sha256sum should always be present if the package manifest comes from
// the packages.manifest file belonging to the pkg repository.
@@ -433,6 +437,32 @@ load_packages (const shared_ptr<repository>& rp,
if (p == nullptr)
{
+ // Convert the package manifest build configurations (contain public
+ // keys data) into the brep's build package configurations (contain
+ // public key object lazy pointers). Keep the bot key lists empty if
+ // the package is not buildable.
+ //
+ package_build_configs build_configs;
+
+ if (!pm.build_configs.empty ())
+ {
+ build_configs.reserve (pm.build_configs.size ());
+
+ for (bpkg::build_package_config& c: pm.build_configs)
+ {
+ build_configs.emplace_back (move (c.name),
+ move (c.arguments),
+ move (c.comment),
+ move (c.builds),
+ move (c.constraints),
+ move (c.auxiliaries),
+ package_build_bot_keys (),
+ move (c.email),
+ move (c.warning_email),
+ move (c.error_email));
+ }
+ }
+
if (rp->internal)
{
if (!overrides.empty ())
@@ -594,6 +624,107 @@ load_packages (const shared_ptr<repository>& rp,
//
package_name project (pm.effective_project ());
+ // If the package is buildable, then save the package manifest's
+ // common and build configuration-specific bot keys into the database
+ // and translate the key data lists into the lists of the public key
+ // object lazy pointers.
+ //
+ package_build_bot_keys bot_keys;
+
+ if (rp->buildable)
+ {
+ // Save the specified bot keys into the database as public key
+ // objects, unless they are already persisted. Translate these keys
+ // into the public key object lazy pointers.
+ //
+ auto keys_to_objects = [&lo,
+ &pm,
+ &tenant,
+ &db] (strings&& keys)
+ {
+ package_build_bot_keys r;
+
+ if (keys.empty ())
+ return r;
+
+ r.reserve (keys.size ());
+
+ for (string& key: keys)
+ {
+ // Calculate the key fingerprint.
+ //
+ string fp;
+
+ try
+ {
+ openssl os (path ("-"), path ("-"), 2,
+ lo.openssl (),
+ "pkey",
+ lo.openssl_option (), "-pubin", "-outform", "DER");
+
+ os.out << key;
+ os.out.close ();
+
+ fp = sha256 (os.in).string ();
+ os.in.close ();
+
+ if (!os.wait ())
+ {
+ cerr << "process " << lo.openssl () << ' ' << *os.exit
+ << endl;
+
+ throw io_error ("");
+ }
+ }
+ catch (const io_error&)
+ {
+ cerr << "error: unable to convert custom build bot public key "
+ << "for package " << pm.name << ' ' << pm.version << endl
+ << " info: key:" << endl
+ << key << endl;
+
+ throw failed ();
+ }
+ catch (const process_error& e)
+ {
+ cerr << "error: unable to convert custom build bot public key "
+ << "for package " << pm.name << ' ' << pm.version << ": "
+ << e << endl;
+
+ throw failed ();
+ }
+
+ // Try to find the public_key object for the calculated
+ // fingerprint. If it doesn't exist, then create and persist the
+ // new object.
+ //
+ public_key_id id (tenant, move (fp));
+ shared_ptr<public_key> k (db.find<public_key> (id));
+
+ if (k == nullptr)
+ {
+ k = make_shared<public_key> (move (id.tenant),
+ move (id.fingerprint),
+ move (key));
+
+ db.persist (k);
+ }
+
+ r.push_back (move (k));
+ }
+
+ return r;
+ };
+
+ bot_keys = keys_to_objects (move (pm.build_bot_keys));
+
+ assert (build_configs.size () == pm.build_configs.size ());
+
+ for (size_t i (0); i != build_configs.size (); ++i)
+ build_configs[i].bot_keys =
+ keys_to_objects (move (pm.build_configs[i].bot_keys));
+ }
+
p = make_shared<package> (
move (pm.name),
move (pm.version),
@@ -622,7 +753,8 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.builds),
move (pm.build_constraints),
move (pm.build_auxiliaries),
- move (pm.build_configs),
+ move (bot_keys),
+ move (build_configs),
move (pm.location),
move (pm.fragment),
move (pm.sha256sum),
@@ -636,7 +768,7 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.builds),
move (pm.build_constraints),
move (pm.build_auxiliaries),
- move (pm.build_configs),
+ move (build_configs),
rp);
db.persist (p);
@@ -1018,7 +1150,8 @@ load_repositories (const options& lo,
// We don't apply overrides to the external packages.
//
- load_packages (pr,
+ load_packages (lo,
+ pr,
!pr->cache_location.empty () ? pr->cache_location : cl,
db,
ignore_unknown,
@@ -1630,6 +1763,7 @@ try
{
db.erase_query<package> ();
db.erase_query<repository> ();
+ db.erase_query<public_key> ();
db.erase_query<tenant> ();
}
else // Multi-tenant mode.
@@ -1642,6 +1776,9 @@ try
db.erase_query<repository> (
query<repository>::id.tenant.in_range (ts.begin (), ts.end ()));
+ db.erase_query<public_key> (
+ query<public_key>::id.tenant.in_range (ts.begin (), ts.end ()));
+
db.erase_query<tenant> (
query<tenant>::id.in_range (ts.begin (), ts.end ()));
}
@@ -1696,7 +1833,8 @@ try
ir.buildable,
priority++));
- load_packages (r,
+ load_packages (ops,
+ r,
r->cache_location,
db,
ops.ignore_unknown (),
diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx
index 78661c3..c1630b0 100644
--- a/mod/build-config-module.hxx
+++ b/mod/build-config-module.hxx
@@ -36,8 +36,9 @@ namespace brep
void
init (const options::build&);
+ template <typename K>
bool
- exclude (const build_package_config& pc,
+ exclude (const build_package_config_template<K>& pc,
const build_class_exprs& common_builds,
const build_constraints& common_constraints,
const build_target_config& tc,
diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx
index 68fbe4c..9ac1390 100644
--- a/mod/build-result-module.cxx
+++ b/mod/build-result-module.cxx
@@ -3,11 +3,16 @@
#include <mod/build-result-module.hxx>
+#include <odb/database.hxx>
+
#include <libbutl/openssl.hxx>
#include <libbutl/fdstream.hxx>
#include <libbutl/process-io.hxx>
#include <libbutl/semantic-version.hxx>
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
namespace brep
{
using namespace std;
@@ -230,54 +235,112 @@ namespace brep
else
{
assert (b.agent_fingerprint && challenge);
- auto i (bot_agent_key_map_->find (*b.agent_fingerprint));
- // The agent's key is recently replaced.
+ auto auth = [&challenge,
+ &b,
+ &o,
+ &fail, &trace,
+ &warn_auth,
+ this] (const path& key)
+ {
+ bool r (false);
+
+ try
+ {
+ openssl os ([&trace, this] (const char* args[], size_t n)
+ {
+ l2 ([&]{trace << process_args {args, n};});
+ },
+ path ("-"), fdstream_mode::text, 2,
+ process_env (o.openssl (), o.openssl_envvar ()),
+ use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl",
+ o.openssl_option (),
+ use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify",
+ "-pubin",
+ "-inkey", key);
+
+ for (const auto& c: *challenge)
+ os.out.put (c); // Sets badbit on failure.
+
+ os.out.close ();
+
+ string s;
+ getline (os.in, s);
+
+ bool v (os.in.eof ());
+ os.in.close ();
+
+ if (os.wait () && v)
+ {
+ r = (s == *b.agent_challenge);
+
+ if (!r)
+ warn_auth ("challenge mismatched");
+ }
+ else // The signature is presumably meaningless.
+ warn_auth ("unable to verify challenge");
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to verify challenge: " << e;
+ }
+
+ return r;
+ };
+
+ const string& fp (*b.agent_fingerprint);
+ auto i (bot_agent_key_map_->find (fp));
+
+ // Note that it is possible that the default vs custom bot
+ // classification has changed since the task request time. It feels that
+ // there is nothing wrong with that and we will handle that
+ // automatically.
//
- if (i == bot_agent_key_map_->end ())
+ if (i != bot_agent_key_map_->end ()) // Default bot?
{
- warn_auth ("agent's public key not found");
+ r = auth (i->second);
}
- else
- try
+ else // Custom bot.
{
- openssl os ([&trace, this] (const char* args[], size_t n)
- {
- l2 ([&]{trace << process_args {args, n};});
- },
- path ("-"), fdstream_mode::text, 2,
- process_env (o.openssl (), o.openssl_envvar ()),
- use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl",
- o.openssl_option (),
- use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify",
- "-pubin",
- "-inkey",
- i->second);
-
- for (const auto& c: *challenge)
- os.out.put (c); // Sets badbit on failure.
-
- os.out.close ();
-
- string s;
- getline (os.in, s);
-
- bool v (os.in.eof ());
- os.in.close ();
-
- if (os.wait () && v)
- {
- r = (s == *b.agent_challenge);
+ shared_ptr<build_public_key> k (
+ build_db_->find<build_public_key> (public_key_id (b.tenant, fp)));
- if (!r)
- warn_auth ("challenge mismatched");
+ if (k != nullptr)
+ {
+ // Temporarily save the key data to disk (note that it's the
+ // challenge which is passed via stdin to openssl). Hopefully /tmp
+ // is using tmpfs.
+ //
+ auto_rmfile arm;
+
+ try
+ {
+ arm = auto_rmfile (path::temp_path ("brep-custom-bot-key"));
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to obtain temporary file: " << e;
+ }
+
+ try
+ {
+ ofdstream os (arm.path);
+ os << *k;
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to write to '" << arm.path << "': " << e;
+ }
+
+ r = auth (arm.path);
+ }
+ else
+ {
+ // The agent's key is recently replaced.
+ //
+ warn_auth ("agent's public key not found");
}
- else // The signature is presumably meaningless.
- warn_auth ("unable to verify challenge");
- }
- catch (const system_error& e)
- {
- fail << "unable to verify challenge: " << e;
}
}
diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx
index a30cf07..a30e281 100644
--- a/mod/build-target-config.cxx
+++ b/mod/build-target-config.cxx
@@ -21,17 +21,13 @@ namespace brep
{"all"}, '+', "All.");
bool
- exclude (const build_package_config& pc,
- const build_class_exprs& cbs,
- const build_constraints& ccs,
+ exclude (const build_class_exprs& exprs,
+ const build_constraints& constrs,
const build_target_config& tc,
const map<string, string>& class_inheritance_map,
string* reason,
bool default_all_ucs)
{
- const build_class_exprs& exprs (pc.effective_builds (cbs));
- const build_constraints& constrs (pc.effective_constraints (ccs));
-
// Save the first sentence of the reason, lower-case the first letter if
// the beginning looks like a word (all subsequent characters until a
// whitespace are lower-case letters).
diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx
index 180ca80..60d159c 100644
--- a/mod/build-target-config.hxx
+++ b/mod/build-target-config.hxx
@@ -32,14 +32,31 @@ namespace brep
// configuration set needlessly).
//
bool
- exclude (const build_package_config&,
- const build_class_exprs& common_builds,
- const build_constraints& common_constraints,
+ exclude (const build_class_exprs& builds,
+ const build_constraints& constraints,
const build_target_config&,
const std::map<string, string>& class_inheritance_map,
string* reason = nullptr,
bool default_all_ucs = false);
+ template <typename K>
+ inline bool
+ exclude (const build_package_config_template<K>& pc,
+ const build_class_exprs& common_builds,
+ const build_constraints& common_constraints,
+ const build_target_config& tc,
+ const std::map<string, string>& class_inheritance_map,
+ string* reason = nullptr,
+ bool default_all_ucs = false)
+ {
+ return exclude (pc.effective_builds (common_builds),
+ pc.effective_constraints (common_constraints),
+ tc,
+ class_inheritance_map,
+ reason,
+ default_all_ucs);
+ }
+
// Convert dash-separated components (target, build target configuration
// name, machine name) or a pattern thereof into a path, replacing dashes
// with slashes (directory separators), `**` with `*/**/*`, and appending
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 773d041..e0aad4b 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -132,7 +132,8 @@ init (scanner& s)
//
template <typename T>
static inline query<T>
-package_query (brep::params::build_task& params,
+package_query (bool custom_bot,
+ brep::params::build_task& params,
interactive_mode imode,
uint64_t queued_expiration_ns)
{
@@ -141,9 +142,39 @@ package_query (brep::params::build_task& params,
query q (!query::build_tenant::archived);
+ if (custom_bot)
+ {
+ // Note that we could potentially only query the packages which refer to
+ // this custom bot key in one of their build configurations. For that we
+ // would need to additionally join the current query tables with the bot
+ // fingerprint-containing build_package_bot_keys and
+ // build_package_config_bot_keys tables and use the SELECT DISTINCT
+ // clause. The problem is that we also use the ORDER BY clause and in this
+ // case PostgreSQL requires all the ORDER BY clause expressions to also be
+ // present in the SELECT DISTINCT clause and fails with the 'for SELECT
+ // DISTINCT, ORDER BY expressions must appear in select list' error if
+ // that's not the case. Also note that in the ODB-generated code the
+ // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause
+ // (see the CITEXT type mapping for details in libbrep/common.hxx) would
+ // not match the 'build_package.name' expression in the ORDER BY clause
+ // and so we will end up with the mentioned error. One (hackish) way to
+ // fix that would be to add a dummy member of the string type for the
+ // build_package.name column. This all sounds quite hairy at the moment
+ // and it also feels that this can potentially pessimize querying the
+ // packages built with the default bots only. Thus let's keep it simple
+ // for now and filter packages by the bot fingerprint at the program
+ // level.
+ //
+ q = q && (query::build_package::custom_bot.is_null () ||
+ query::build_package::custom_bot);
+ }
+ else
+ q = q && (query::build_package::custom_bot.is_null () ||
+ !query::build_package::custom_bot);
+
// Filter by repositories canonical names (if requested).
//
- const vector<string>& rp (params.repository ());
+ const strings& rp (params.repository ());
if (!rp.empty ())
q = q &&
@@ -213,20 +244,28 @@ handle (request& rq, response& rs)
throw invalid_request (400, e.what ());
}
- // Obtain the agent's public key fingerprint if requested. If the fingerprint
- // is requested but is not present in the request or is unknown, then respond
- // with 401 HTTP code (unauthorized).
+ // Obtain the agent's public key fingerprint if requested. If the
+ // fingerprint is requested but is not present in the request, then respond
+ // with 401 HTTP code (unauthorized). If a key with the specified
+ // fingerprint is not present in the build bot agent keys directory, then
+ // assume that this is a custom build bot.
+ //
+ // Note that if the agent authentication is not configured (the agent keys
+ // directory is not specified), then the bot can never be custom and its
+ // fingerprint is ignored, if present.
//
optional<string> agent_fp;
+ bool custom_bot (false);
if (bot_agent_key_map_ != nullptr)
{
- if (!tqm.fingerprint ||
- bot_agent_key_map_->find (*tqm.fingerprint) ==
- bot_agent_key_map_->end ())
+ if (!tqm.fingerprint)
throw invalid_request (401, "unauthorized");
agent_fp = move (tqm.fingerprint);
+
+ custom_bot = (bot_agent_key_map_->find (*agent_fp) ==
+ bot_agent_key_map_->end ());
}
// The resulting task manifest and the related build, package, and
@@ -659,7 +698,8 @@ handle (request& rq, response& rs)
using pkg_query = query<buildable_package>;
using prep_pkg_query = prepared_query<buildable_package>;
- pkg_query pq (package_query<buildable_package> (params,
+ pkg_query pq (package_query<buildable_package> (custom_bot,
+ params,
imode,
queued_expiration_ns));
@@ -815,7 +855,8 @@ handle (request& rq, response& rs)
{
using query = query<buildable_package_count>;
- query q (package_query<buildable_package_count> (params,
+ query q (package_query<buildable_package_count> (custom_bot,
+ params,
imode,
queued_expiration_ns));
@@ -1241,7 +1282,8 @@ handle (request& rq, response& rs)
//
small_vector<bpkg::test_dependency, 1> tests;
- build_db_->load (*p, p->requirements_tests_section);
+ if (!p->requirements_tests_section.loaded ())
+ build_db_->load (*p, p->requirements_tests_section);
for (const build_test_dependency& td: p->tests)
{
@@ -1293,6 +1335,8 @@ handle (request& rq, response& rs)
true /* default_all_ucs */))
continue;
+ build_db_->load (*tp, tp->auxiliaries_section);
+
for (const build_auxiliary& ba:
tpc->effective_auxiliaries (tp->auxiliaries))
{
@@ -1312,14 +1356,17 @@ handle (request& rq, response& rs)
vector<auxiliary_machine> tms;
vector<build_machine> bms;
- tms.reserve (picked_machines.size ());
- bms.reserve (picked_machines.size ());
-
- for (pair<auxiliary_config_machine, string>& pm: picked_machines)
+ if (size_t n = picked_machines.size ())
{
- const machine_header_manifest& m (*pm.first.machine);
- tms.push_back (auxiliary_machine {m.name, move (pm.second)});
- bms.push_back (build_machine {m.name, m.summary});
+ tms.reserve (n);
+ bms.reserve (n);
+
+ for (pair<auxiliary_config_machine, string>& pm: picked_machines)
+ {
+ const machine_header_manifest& m (*pm.first.machine);
+ tms.push_back (auxiliary_machine {m.name, move (pm.second)});
+ bms.push_back (build_machine {m.name, m.summary});
+ }
}
return collect_auxiliaries_result {
@@ -1603,8 +1650,40 @@ handle (request& rq, response& rs)
//
bool package_built (false);
+ build_db_->load (*p, p->bot_keys_section);
+
for (const build_package_config& pc: p->configs)
{
+ // If this is a custom bot, then skip this configuration if it
+ // doesn't contain this bot's public key in its custom bot keys
+ // list. Otherwise (this is a default bot), skip this
+ // configuration if its custom bot keys list is not empty.
+ //
+ {
+ const build_package_bot_keys& bks (
+ pc.effective_bot_keys (p->bot_keys));
+
+ if (custom_bot)
+ {
+ assert (agent_fp); // Wouldn't be here otherwise.
+
+ if (find_if (
+ bks.begin (), bks.end (),
+ [&agent_fp] (const lazy_shared_ptr<build_public_key>& k)
+ {
+ return k.object_id ().fingerprint == *agent_fp;
+ }) == bks.end ())
+ {
+ continue;
+ }
+ }
+ else
+ {
+ if (!bks.empty ())
+ continue;
+ }
+ }
+
pkg_config = pc.name;
// Iterate through the built configurations and erase them from the
@@ -1647,29 +1726,33 @@ handle (request& rq, response& rs)
// the package configuration and for which all the requested
// auxiliary machines can be provided.
//
- auto i (configs.begin ());
- auto e (configs.end ());
+ const config_machine* cm (nullptr);
+ optional<collect_auxiliaries_result> aux;
build_db_->load (*p, p->constraints_section);
- optional<collect_auxiliaries_result> aux;
- for (; i != e; ++i)
+ for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i)
{
- const build_target_config& tc (*i->second.config);
+ cm = &i->second;
+ const build_target_config& tc (*cm->config);
- if (!exclude (pc, p->builds, p->constraints, tc) &&
- (aux = collect_auxiliaries (p, pc, tc)))
- break;
+ if (!exclude (pc, p->builds, p->constraints, tc))
+ {
+ if (!p->auxiliaries_section.loaded ())
+ build_db_->load (*p, p->auxiliaries_section);
+
+ if ((aux = collect_auxiliaries (p, pc, tc)))
+ break;
+ }
}
- if (i != e)
+ if (aux)
{
- config_machine& cm (i->second);
- machine_header_manifest& mh (*cm.machine);
+ machine_header_manifest& mh (*cm->machine);
build_id bid (move (id),
- cm.config->target,
- cm.config->name,
+ cm->config->target,
+ cm->config->name,
move (pkg_config),
move (toolchain_name),
toolchain_version);
@@ -1704,8 +1787,8 @@ handle (request& rq, response& rs)
build_machine {
mh.name, move (mh.summary)},
move (aux->build_auxiliary_machines),
- controller_checksum (*cm.config),
- machine_checksum (*cm.machine));
+ controller_checksum (*cm->config),
+ machine_checksum (*cm->machine));
build_db_->persist (b);
}
@@ -1753,8 +1836,8 @@ handle (request& rq, response& rs)
b->auxiliary_machines =
move (aux->build_auxiliary_machines);
- string ccs (controller_checksum (*cm.config));
- string mcs (machine_checksum (*cm.machine));
+ string ccs (controller_checksum (*cm->config));
+ string mcs (machine_checksum (*cm->machine));
// Issue the hard rebuild if it is forced or the
// configuration or machine has changed.
@@ -1831,7 +1914,7 @@ handle (request& rq, response& rs)
move (aux->tests),
move (aux->task_auxiliary_machines),
move (bp.interactive),
- cm);
+ *cm);
task_build = move (b);
task_package = move (p);
@@ -1971,13 +2054,17 @@ handle (request& rq, response& rs)
(t->interactive.has_value () ==
(imode == interactive_mode::true_))))
{
+ const build_target_config& tc (*cm.config);
+
build_db_->load (*p, p->constraints_section);
- const build_target_config& tc (*cm.config);
+ if (exclude (*pc, p->builds, p->constraints, tc))
+ continue;
+
+ build_db_->load (*p, p->auxiliaries_section);
- optional<collect_auxiliaries_result> aux;
- if (!exclude (*pc, p->builds, p->constraints, tc) &&
- (aux = collect_auxiliaries (p, *pc, tc)))
+ if (optional<collect_auxiliaries_result> aux =
+ collect_auxiliaries (p, *pc, tc))
{
assert (b->status);
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 51b21c6..91923e5 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -541,7 +541,7 @@ handle (request& rq, response& rs)
//
builds = false;
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
const build_class_exprs& exprs (pc.effective_builds (pkg->builds));
@@ -726,7 +726,7 @@ handle (request& rq, response& rs)
s << H3 << "Builds" << ~H3
<< DIV(ID="builds");
- auto exclude = [&pkg, this] (const build_package_config& pc,
+ auto exclude = [&pkg, this] (const package_build_config& pc,
const build_target_config& tc,
string* rs = nullptr)
{
@@ -767,7 +767,7 @@ handle (request& rq, response& rs)
query sq (false);
set<config_toolchain> unbuilt_configs;
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
for (const auto& bc: *target_conf_map_)
{
@@ -886,7 +886,7 @@ handle (request& rq, response& rs)
//
if (!tn->interactive)
{
- for (const build_package_config& pc: pkg->build_configs)
+ for (const package_build_config& pc: pkg->build_configs)
{
for (const auto& tc: *target_conf_)
{
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index 2f49f81..42d481d 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -95,7 +95,7 @@ namespace brep
// not.
//
void
- add_delay (shared_ptr<build_delay>, bool report);
+ add_delay (shared_ptr<build_delay>, bool custom_bot, bool report);
bool
empty () const {return reported_delay_count_ == 0;}
@@ -108,19 +108,39 @@ namespace brep
print (const char* header, bool total, bool full) const;
private:
- // Maps delays to the report flag.
+ // Maps delays to the custom bot and report flag.
//
- map<shared_ptr<const build_delay>, bool, compare_delay> delays_;
+ struct delay_info
+ {
+ bool custom_bot;
+ bool report;
+ };
+
+ map<shared_ptr<const build_delay>, delay_info, compare_delay> delays_;
size_t reported_delay_count_ = 0;
+
+ // Number of reported/total delayed package configurations which need to
+ // be built with the custom build bots.
+ //
+ size_t custom_total_delay_count_ = 0;
+ size_t custom_reported_delay_count_ = 0;
};
void delay_report::
- add_delay (shared_ptr<build_delay> delay, bool report)
+ add_delay (shared_ptr<build_delay> delay, bool custom_bot, bool report)
{
- delays_.emplace (move (delay), report);
+ delays_.emplace (move (delay), delay_info {custom_bot, report});
+
+ if (custom_bot)
+ ++custom_total_delay_count_;
if (report)
+ {
++reported_delay_count_;
+
+ if (custom_bot)
+ ++custom_reported_delay_count_;
+ }
}
void delay_report::
@@ -134,6 +154,17 @@ namespace brep
if (total)
cerr << '/' << delays_.size ();
+ if (custom_reported_delay_count_ != 0 ||
+ (total && custom_total_delay_count_ != 0))
+ {
+ cerr << " including " << custom_reported_delay_count_;
+
+ if (total)
+ cerr << '/' << custom_total_delay_count_;
+
+ cerr << " custom";
+ }
+
cerr << "):" << endl;
// Group the printed delays by toolchain and target configuration.
@@ -146,10 +177,15 @@ namespace brep
size_t config_reported_delay_count (0);
size_t config_total_delay_count (0);
+ size_t config_custom_reported_delay_count (0);
+ size_t config_custom_total_delay_count (0);
+
auto brief_config = [&target_config_name,
&target,
&config_reported_delay_count,
&config_total_delay_count,
+ &config_custom_reported_delay_count,
+ &config_custom_total_delay_count,
total] ()
{
if (target_config_name != nullptr)
@@ -166,21 +202,36 @@ namespace brep
if (total)
cerr << '/' << config_total_delay_count;
+ if (config_custom_reported_delay_count != 0 ||
+ (total && config_custom_total_delay_count != 0))
+ {
+ cerr << " including " << config_custom_reported_delay_count;
+
+ if (total)
+ cerr << '/' << config_custom_total_delay_count;
+
+ cerr << " custom";
+ }
+
cerr << ')' << endl;
}
config_reported_delay_count = 0;
config_total_delay_count = 0;
+
+ config_custom_reported_delay_count = 0;
+ config_custom_total_delay_count = 0;
}
};
for (const auto& dr: delays_)
{
- bool report (dr.second);
+ bool report (dr.second.report);
if (full && !report)
continue;
+ bool custom_bot (dr.second.custom_bot);
const shared_ptr<const build_delay>& d (dr.first);
// Print the toolchain, if changed.
@@ -239,6 +290,9 @@ namespace brep
cerr << " " << d->package_name << '/' << d->package_version
<< ' ' << d->package_config_name;
+ if (custom_bot)
+ cerr << " (custom bot)";
+
if (!d->tenant.empty ())
cerr << ' ' << d->tenant;
@@ -247,9 +301,17 @@ namespace brep
else
{
if (report)
+ {
++config_reported_delay_count;
+ if (custom_bot)
+ ++config_custom_reported_delay_count;
+ }
+
++config_total_delay_count;
+
+ if (custom_bot)
+ ++config_custom_total_delay_count;
}
}
@@ -953,6 +1015,21 @@ namespace brep
soft_delayed = false;
}
+ // If there is a delay, then deduce if this package
+ // configuration needs to be built with a custom build bot.
+ //
+ // Note: only meaningful if there is a delay.
+ //
+ bool custom_bot (false);
+
+ if (hard_delayed || soft_delayed)
+ {
+ if (!p->bot_keys_section.loaded ())
+ db.load (*p, p->bot_keys_section);
+
+ custom_bot = !pc.effective_bot_keys (p->bot_keys).empty ();
+ }
+
// Add hard/soft delays to the respective reports and
// collect the delay for update, if it is reported.
//
@@ -983,7 +1060,7 @@ namespace brep
reported = true;
}
- hard_delays_report.add_delay (d, report);
+ hard_delays_report.add_delay (d, custom_bot, report);
}
if (soft_delayed)
@@ -997,7 +1074,7 @@ namespace brep
reported = true;
}
- soft_delays_report.add_delay (d, report);
+ soft_delays_report.add_delay (d, custom_bot, report);
}
// If we don't consider the report timestamps for reporting
generated by cgit v1.1 at 2024-11-11 01:54:30 +0000