From 7fa57f8522ed7741c00ed3cd64cf956716aebd2c Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 11 Mar 2024 21:23:21 +0300 Subject: Add support for build auxiliary machines/configurations --- libbrep/build-extra.sql | 34 ++++++++++++++++++ libbrep/build-package.hxx | 24 ++++++++++--- libbrep/build.cxx | 50 +++++++++++++++++++++++--- libbrep/build.hxx | 45 ++++++++++++++++++++--- libbrep/build.xml | 73 +++++++++++++++++++++++++++++++++++++ libbrep/common.hxx | 63 ++++++++++++++++++++++++++++---- libbrep/package.cxx | 31 ++++++++-------- libbrep/package.hxx | 42 +++++++++++++++++----- libbrep/package.xml | 91 +++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 408 insertions(+), 45 deletions(-) (limited to 'libbrep') diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index c15ddc1..374cf73 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -6,12 +6,16 @@ -- package-extra.sql file for details. -- +DROP FOREIGN TABLE IF EXISTS build_package_config_auxiliaries; + DROP FOREIGN TABLE IF EXISTS build_package_config_constraints; DROP FOREIGN TABLE IF EXISTS build_package_config_builds; DROP FOREIGN TABLE IF EXISTS build_package_configs; +DROP FOREIGN TABLE IF EXISTS build_package_auxiliaries; + DROP FOREIGN TABLE IF EXISTS build_package_constraints; DROP FOREIGN TABLE IF EXISTS build_package_builds; @@ -187,6 +191,22 @@ CREATE FOREIGN TABLE build_package_constraints ( comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_constraints'); +-- The foreign table for the build_package object auxiliaries member (that is +-- of a container type). +-- +CREATE FOREIGN TABLE build_package_auxiliaries ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + environment_name TEXT NOT NULL, + config TEXT NOT NULL, + comment TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_auxiliaries'); + -- The foreign tables for the build_package object configs member (that is a -- container of values containing containers. -- @@ -236,3 +256,17 @@ CREATE FOREIGN TABLE build_package_config_constraints ( target TEXT NULL, comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_config_constraints'); + +CREATE FOREIGN TABLE build_package_config_auxiliaries ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + config_index BIGINT NOT NULL, + index BIGINT NOT NULL, + environment_name TEXT NOT NULL, + config TEXT NOT NULL, + comment TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_config_auxiliaries'); diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index a0e1082..8377158 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -132,11 +132,13 @@ namespace brep lazy_shared_ptr internal_repository; bool buildable; - // Mapped to the package object builds, build_constraints, and - // build_configs members using the PostgreSQL foreign table mechanism. + // Mapped to the package object builds, build_constraints, + // build_auxiliaries, and build_configs members using the PostgreSQL + // foreign table mechanism. // build_class_exprs builds; build_constraints constraints; + build_auxiliaries auxiliaries; build_package_configs configs; // Group the builds and constraints members of this object as well as of @@ -191,7 +193,7 @@ namespace brep #pragma db member(requirements_tests_section) load(lazy) update(always) - // builds and constraints + // builds, constraints, and auxiliaries // #pragma db member(builds) id_column("") value_column("") \ section(constraints_section) @@ -199,9 +201,12 @@ namespace brep #pragma db member(constraints) id_column("") value_column("") \ section(constraints_section) + #pragma db member(auxiliaries) id_column("") value_column("") \ + section(constraints_section) + // configs // - // Note that build_package_config::{builds,constraints} are + // Note that build_package_config::{builds,constraints,auxiliaries} are // persisted/loaded via the separate nested containers (see commons.hxx // for details). // @@ -229,6 +234,17 @@ namespace brep id_column("") key_column("") value_column("") \ section(constraints_section) + #pragma db member(config_auxiliaries) \ + virtual(build_auxiliaries_map) \ + after(config_constraints) \ + get(odb::nested_get ( \ + brep::build_package_config_auxiliaries (this.configs))) \ + set(brep::build_package_config_auxiliaries as; \ + odb::nested_set (as, std::move (?)); \ + move (as).to_configs (this.configs)) \ + id_column("") key_column("") value_column("") \ + section(constraints_section) + #pragma db member(constraints_section) load(lazy) update(always) private: diff --git a/libbrep/build.cxx b/libbrep/build.cxx index 8fadfa3..13f0818 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -65,7 +65,8 @@ namespace brep string tnm, version tvr, optional inr, optional afp, optional ach, - string mnm, string msm, + build_machine mcn, + vector ams, string ccs, string mcs) : id (package_id (move (tnt), move (pnm), pvr), @@ -86,8 +87,8 @@ namespace brep timestamp (timestamp_type::clock::now ()), force (force_state::unforced), agent_fingerprint (move (afp)), agent_challenge (move (ach)), - machine (move (mnm)), - machine_summary (move (msm)), + machine (move (mcn)), + auxiliary_machines (move (ams)), controller_checksum (move (ccs)), machine_checksum (move (mcs)) { @@ -121,6 +122,43 @@ namespace brep } build:: + build (string tnt, + package_name_type pnm, + version pvr, + target_triplet trg, + string tcf, + string pcf, + string tnm, version tvr, + result_status rst, + operation_results ors, + build_machine mcn, + vector ams) + : id (package_id (move (tnt), move (pnm), pvr), + move (trg), + move (tcf), + move (pcf), + move (tnm), tvr), + tenant (id.package.tenant), + package_name (id.package.name), + package_version (move (pvr)), + target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), + toolchain_name (id.toolchain_name), + toolchain_version (move (tvr)), + state (build_state::built), + timestamp (timestamp_type::clock::now ()), + force (force_state::unforced), + status (rst), + soft_timestamp (timestamp), + hard_timestamp (timestamp), + machine (move (mcn)), + auxiliary_machines (move (ams)), + results (move (ors)) + { + } + + build:: build (build&& b) : id (move (b.id)), tenant (id.package.tenant), @@ -141,7 +179,8 @@ namespace brep agent_fingerprint (move (b.agent_fingerprint)), agent_challenge (move (b.agent_challenge)), machine (move (b.machine)), - machine_summary (move (b.machine_summary)), + auxiliary_machines (move (b.auxiliary_machines)), + auxiliary_machines_section (move (b.auxiliary_machines_section)), results (move (b.results)), results_section (move (b.results_section)), controller_checksum (move (b.controller_checksum)), @@ -170,7 +209,8 @@ namespace brep agent_fingerprint = move (b.agent_fingerprint); agent_challenge = move (b.agent_challenge); machine = move (b.machine); - machine_summary = move (b.machine_summary); + auxiliary_machines = move (b.auxiliary_machines); + auxiliary_machines_section = move (b.auxiliary_machines_section); results = move (b.results); results_section = move (b.results_section); controller_checksum = move (b.controller_checksum); diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 1e9a9fc..be7cdf5 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -28,7 +28,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 24, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 25, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace @@ -230,6 +230,13 @@ namespace brep using bbot::operation_results; + #pragma db value + struct build_machine + { + string name; + string summary; + }; + #pragma db object pointer(shared_ptr) session class build { @@ -249,7 +256,8 @@ namespace brep optional interactive, optional agent_fingerprint, optional agent_challenge, - string machine, string machine_summary, + build_machine, + vector auxiliary_machines, string controller_checksum, string machine_checksum); @@ -262,6 +270,21 @@ namespace brep string package_config_name, string toolchain_name, version toolchain_version); + // Create the build object with the built state, the specified status and + // operation results, all the timestamps set to now, and the force state + // set to unforced. + // + build (string tenant, + package_name_type, version, + target_triplet, + string target_config_name, + string package_config_name, + string toolchain_name, version toolchain_version, + result_status, + operation_results, + build_machine, + vector auxiliary_machines = {}); + // Move-only type. // build (build&&); @@ -325,8 +348,9 @@ namespace brep optional agent_fingerprint; optional agent_challenge; - string machine; - string machine_summary; + build_machine machine; + vector auxiliary_machines; + odb::section auxiliary_machines_section; // Note that the logs are stored as std::string/TEXT which is Ok since // they are UTF-8 and our database is UTF-8. @@ -368,6 +392,19 @@ namespace brep // #pragma db member(timestamp) index + #pragma db member(machine) transient + + #pragma db member(machine_name) virtual(std::string) \ + access(machine.name) column("machine") + + #pragma db member(machine_summary) virtual(std::string) \ + access(machine.summary) + + #pragma db member(auxiliary_machines) id_column("") value_column("") \ + section(auxiliary_machines_section) + + #pragma db member(auxiliary_machines_section) load(lazy) update(always) + #pragma db member(results) id_column("") value_column("") \ section(results_section) diff --git a/libbrep/build.xml b/libbrep/build.xml index e757aba..0dc37ee 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/libbrep/common.hxx b/libbrep/common.hxx index 9c398c8..6220149 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -347,6 +347,13 @@ namespace brep #pragma db value(build_constraint) definition + // build_auxiliaries + // + using bpkg::build_auxiliary; + using build_auxiliaries = vector; + + #pragma db value(build_auxiliary) definition + // email // using bpkg::email; @@ -360,10 +367,6 @@ namespace brep #pragma db value(build_package_config) definition - // @@ TMP AUXILIARY - // - #pragma db member(build_package_config::auxiliaries) transient - // 1 for the default configuration which is always present. // using build_package_configs = small_vector; @@ -376,11 +379,12 @@ namespace brep // Note that ODB doesn't support containers of value types which contain // containers. Thus, we will persist/load - // package_build_config::{builds,constraint} via the separate nested - // containers using the adapter classes. + // package_build_config::{builds,constraint,auxiliaries} via the separate + // nested containers using the adapter classes. + // + // build_package_config::builds // #pragma db member(build_package_config::builds) transient - #pragma db member(build_package_config::constraints) transient using build_class_expr_key = odb::nested_key; using build_class_exprs_map = std::map; @@ -419,6 +423,10 @@ namespace brep } }; + // build_package_config::constraints + // + #pragma db member(build_package_config::constraints) transient + using build_constraint_key = odb::nested_key; using build_constraints_map = std::map; @@ -456,6 +464,47 @@ namespace brep } }; + // build_package_config::auxiliaries + // + #pragma db member(build_package_config::auxiliaries) transient + + using build_auxiliary_key = odb::nested_key; + using build_auxiliaries_map = std::map; + + #pragma db value(build_auxiliary_key) + #pragma db member(build_auxiliary_key::outer) column("config_index") + #pragma db member(build_auxiliary_key::inner) column("index") + + // Adapter for build_package_config::auxiliaries. + // + class build_package_config_auxiliaries: + public small_vector // 1 as for build_package_configs. + { + public: + build_package_config_auxiliaries () = default; + + explicit + build_package_config_auxiliaries (const build_package_configs& cs) + { + reserve (cs.size ()); + for (const build_package_config& c: cs) + push_back (c.auxiliaries); + } + + void + to_configs (build_package_configs& cs) && + { + // Note that the empty trailing entries will be missing (see ODB's + // nested-container.hxx for details). + // + assert (size () <= cs.size ()); + + auto i (cs.begin ()); + for (build_auxiliaries& bas: *this) + i++->auxiliaries = move (bas); + } + }; + // The primary reason why a package is unbuildable by the build bot // controller service. // diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 2320547..37795f0 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -81,6 +81,7 @@ namespace brep small_vector ts, build_class_exprs bs, build_constraints_type bc, + build_auxiliaries_type ac, build_package_configs bcs, optional lc, optional fr, @@ -114,28 +115,17 @@ namespace brep tests (move (ts)), builds (move (bs)), build_constraints (move (bc)), + build_auxiliaries (move (ac)), + build_configs (move (bcs)), internal_repository (move (rp)), location (move (lc)), fragment (move (fr)), sha256sum (move (sh)) { - // Add the default build configuration at the beginning, unless it is - // specified explicitly. + // The default configuration is always added by the package manifest + // parser (see libbpkg/manifest.cxx for details). // - if (find_if (bcs.begin (), bcs.end (), - [] (const build_package_config& c) - {return c.name == "default";}) != bcs.end ()) - { - build_configs = move (bcs); - } - else - { - build_configs.reserve (bcs.size () + 1); - build_configs.emplace_back ("default"); - build_configs.insert (build_configs.end (), - make_move_iterator (bcs.begin ()), - make_move_iterator (bcs.end ())); - } + assert (find ("default", build_configs) != nullptr); if (stub ()) unbuildable_reason = brep::unbuildable_reason::stub; @@ -152,6 +142,8 @@ namespace brep version_type vr, build_class_exprs bs, build_constraints_type bc, + build_auxiliaries_type ac, + build_package_configs bcs, shared_ptr rp) : id (rp->tenant, move (nm), vr), tenant (id.tenant), @@ -159,11 +151,18 @@ namespace brep version (move (vr)), builds (move (bs)), build_constraints (move (bc)), + build_auxiliaries (move (ac)), + build_configs (move (bcs)), buildable (false), unbuildable_reason (stub () ? brep::unbuildable_reason::stub : brep::unbuildable_reason::external) { + // The default configuration is always added by the package manifest + // parser (see libbpkg/manifest.cxx for details). + // + assert (find ("default", build_configs) != nullptr); + assert (!rp->internal); other_repositories.emplace_back (move (rp)); } diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 9bb9af9..96e51a3 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 30, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 31, closed) namespace brep { @@ -453,11 +453,11 @@ namespace brep using dependencies_type = brep::dependencies; using requirements_type = brep::requirements; using build_constraints_type = brep::build_constraints; + using build_auxiliaries_type = brep::build_auxiliaries; // Create internal package object. // - // Note: adds the default build package config at the first position if it - // is not present yet. + // Note: the default build package config is expected to always be present. // package (package_name, version_type, @@ -485,6 +485,7 @@ namespace brep small_vector tests, build_class_exprs, build_constraints_type, + build_auxiliaries_type, build_package_configs, optional location, optional fragment, @@ -500,14 +501,20 @@ namespace brep // // External package can also be a separate test for some primary package // (and belong to a complement but yet external repository), and so we may - // need its build class expressions and constraints to decide if to build - // it together with the primary package or not (see test-exclude task - // manifest value for details). + // need its build class expressions, constraints, and configurations to + // decide if to build it together with the primary package or not (see + // test-exclude task manifest value for details). Additionally, when the + // test package is being built the auxiliary machines may also be + // required. + // + // Note: the default build package config is expected to always be present. // package (package_name name, version_type, build_class_exprs, build_constraints_type, + build_auxiliaries_type, + build_package_configs, shared_ptr); bool @@ -561,11 +568,12 @@ namespace brep requirements_type requirements; // Note: foreign-mapped in build. small_vector tests; // Note: foreign-mapped in build. - // Common build classes/constraints that apply to all configurations - // unless overridden. + // Common build classes, constraints, and auxiliaries that apply to all + // configurations unless overridden. // build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. + build_auxiliaries_type build_auxiliaries; // Note: foreign-mapped in build. build_package_configs build_configs; // Note: foreign-mapped in build. @@ -718,9 +726,14 @@ namespace brep #pragma db member(build_constraints) id_column("") value_column("") \ section(build_section) + // build_auxiliaries + // + #pragma db member(build_auxiliaries) id_column("") value_column("") \ + section(build_section) + // build_configs // - // Note that build_package_config::{builds,constraints} are + // Note that build_package_config::{builds,constraints,auxiliaries} are // persisted/loaded via the separate nested containers (see commons.hxx // for details). // @@ -749,6 +762,17 @@ namespace brep id_column("") key_column("") value_column("") \ section(build_section) + #pragma db member(build_config_auxiliaries) \ + virtual(build_auxiliaries_map) \ + after(build_config_constraints) \ + get(odb::nested_get ( \ + brep::build_package_config_auxiliaries (this.build_configs))) \ + set(brep::build_package_config_auxiliaries as; \ + odb::nested_set (as, std::move (?)); \ + move (as).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(build_section) + #pragma db member(build_section) load(lazy) update(always) // other_repositories diff --git a/libbrep/package.xml b/libbrep/package.xml index 6852f75..966861b 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.1