diff options
53 files changed, 1649 insertions, 931 deletions
diff --git a/clean/buildfile b/clean/buildfile index 11fa2a2..b91b1a0 100644 --- a/clean/buildfile +++ b/clean/buildfile @@ -7,9 +7,10 @@ import libs += libbutl%lib{butl} import libs += libbbot%lib{bbot} include ../libbrep/ +include ../mod/ exe{brep-clean}: {hxx ixx cxx}{* -clean-options} {hxx ixx cxx}{clean-options} \ - ../libbrep/lib{brep} $libs + ../mod/libue{mod} ../libbrep/lib{brep} $libs # Build options. # diff --git a/clean/clean.cxx b/clean/clean.cxx index 6096249..cb9fd07 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -14,8 +14,6 @@ #include <libbutl/pager.hxx> -#include <libbbot/build-config.hxx> - #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> #include <libbrep/package.hxx> @@ -24,10 +22,11 @@ #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> +#include <mod/build-target-config.hxx> + #include <clean/clean-options.hxx> using namespace std; -using namespace bbot; using namespace odb::core; namespace brep @@ -205,12 +204,13 @@ namespace brep return 1; } - set<string> configs; + // Load build target configurations. + // + build_target_configs configs; try { - for (auto& c: parse_buildtab (cp)) - configs.emplace (move (c.name)); + configs = bbot::parse_buildtab (cp); } catch (const io_error& e) { @@ -218,6 +218,13 @@ namespace brep return 1; } + // Note: contains shallow references to the configuration targets/names. + // + set<build_target_config_id> configs_set; + + for (const build_target_config& c: configs) + configs_set.insert (build_target_config_id {c.target, c.name}); + // Parse timestamps. // map<string, timestamp> timeouts; // Toolchain timeouts. @@ -259,17 +266,25 @@ namespace brep // // Query package builds in chunks in order not to hold locks for too long. // Sort the result by package version to minimize number of queries to the - // package database. + // package database. Note that we still need to sort by configuration and + // toolchain to make sure that builds are sorted consistently across + // queries and we don't miss any of them. // using bld_query = query<build>; using prep_bld_query = prepared_query<build>; size_t offset (0); bld_query bq ("ORDER BY" + - bld_query::id.package.tenant + "," + - bld_query::id.package.name + + bld_query::id.package.tenant + "," + + bld_query::id.package.name + order_by_version_desc (bld_query::id.package.version, - false) + + false) + + bld_query::id.target + "," + + bld_query::id.target_config_name + "," + + bld_query::id.package_config_name + "," + + bld_query::id.toolchain_name + + order_by_version (bld_query::id.toolchain_version, + false /* first */) + "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); connection_ptr conn (db.connection ()); @@ -332,7 +347,10 @@ namespace brep // Note that we unable to detect configuration changes and rely on // periodic rebuilds to take care of that. // - configs.find (b.configuration) == configs.end ()); + configs_set.find ( + build_target_config_id {b.target, + b.target_config_name}) == + configs_set.end ()); // Check that the build package still exists. // @@ -349,7 +367,7 @@ namespace brep } cleanup = package_versions.find (b.package_version) == - package_versions.end (); + package_versions.end (); } if (cleanup) diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index 2c204e6..7331ab1 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -6,6 +6,12 @@ -- package-extra.sql file for details. -- +DROP FOREIGN TABLE IF EXISTS build_package_config_constraints; + +DROP FOREIGN TABLE IF EXISTS build_package_config_builds; + +DROP FOREIGN TABLE IF EXISTS build_package_configs; + DROP FOREIGN TABLE IF EXISTS build_package_constraints; DROP FOREIGN TABLE IF EXISTS build_package_builds; @@ -60,7 +66,7 @@ CREATE FOREIGN TABLE build_package ( SERVER package_server OPTIONS (table_name 'package'); -- The foreign tables for the build_package object requirements member (that --- is of a 2-dimensional container type). +-- is of a 3-dimensional container type). -- CREATE FOREIGN TABLE build_package_requirements ( tenant TEXT NOT NULL, @@ -168,3 +174,47 @@ CREATE FOREIGN TABLE build_package_constraints ( target TEXT NULL, comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_constraints'); + +-- The foreign tables for the build_package object configs member (that is a +-- container of values containing containers. +-- +CREATE FOREIGN TABLE build_package_configs ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + config_name TEXT NOT NULL, + config_arguments TEXT NULL, + config_comment TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_configs'); + +CREATE FOREIGN TABLE build_package_config_builds ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + config_index BIGINT NOT NULL, + index BIGINT NOT NULL, + expression TEXT NOT NULL, + comment TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_config_builds'); + +CREATE FOREIGN TABLE build_package_config_constraints ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + config_index BIGINT NOT NULL, + index BIGINT NOT NULL, + exclusion BOOLEAN NOT NULL, + config TEXT NOT NULL, + target TEXT NULL, + comment TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_config_constraints'); diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 54edc2b..79fb6a4 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -120,11 +120,12 @@ namespace brep lazy_shared_ptr<build_repository> internal_repository; bool buildable; - // Mapped to the package object builds and build_constraints members using - // the PostgreSQL foreign table mechanism. + // Mapped to the package object builds, build_constraints, and + // build_configs members using the PostgreSQL foreign table mechanism. // - build_class_exprs builds; - build_constraints constraints; + build_class_exprs builds; + build_constraints constraints; + build_package_configs configs; bool internal () const noexcept {return internal_repository != nullptr;} @@ -146,9 +147,6 @@ namespace brep // Container of the requirement_alternative values. // - #pragma db member(requirement_alternative_key::outer) column("requirement_index") - #pragma db member(requirement_alternative_key::inner) column("index") - #pragma db member(requirement_alternatives) \ virtual(requirement_alternatives_map) \ after(requirements) \ @@ -158,10 +156,6 @@ namespace brep // Container of the requirement (string) values. // - #pragma db member(requirement_key::outer) column("requirement_index") - #pragma db member(requirement_key::middle) column("alternative_index") - #pragma db member(requirement_key::inner) column("index") - #pragma db member(requirement_alternative_requirements) \ virtual(requirement_alternative_requirements_map) \ after(requirement_alternatives) \ @@ -175,6 +169,34 @@ namespace brep #pragma db member(builds) id_column("") value_column("") #pragma db member(constraints) id_column("") value_column("") + // configs + // + // Note that build_package_config::{builds,constraints} are + // persisted/loaded via the separate nested containers (see commons.hxx + // for details). + // + #pragma db member(configs) id_column("") value_column("config_") + + #pragma db member(config_builds) \ + virtual(build_class_exprs_map) \ + after(configs) \ + get(odb::nested_get ( \ + brep::build_package_config_builds (this.configs))) \ + set(brep::build_package_config_builds bs; \ + odb::nested_set (bs, std::move (?)); \ + move (bs).to_configs (this.configs)) \ + id_column("") key_column("") value_column("") + + #pragma db member(config_constraints) \ + virtual(build_constraints_map) \ + after(config_builds) \ + get(odb::nested_get ( \ + brep::build_package_config_constraints (this.configs))) \ + set(brep::build_package_config_constraints cs; \ + odb::nested_set (cs, std::move (?)); \ + move (cs).to_configs (this.configs)) \ + id_column("") key_column("") value_column("") + private: friend class odb::access; build_package () = default; diff --git a/libbrep/build.cxx b/libbrep/build.cxx index c095b32..c8a2cd1 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -57,8 +57,9 @@ namespace brep build (string tnt, package_name_type pnm, version pvr, - string cfg, target_triplet trg, + string tcf, + string pcf, string tnm, version tvr, optional<string> inr, optional<string> afp, optional<string> ach, @@ -66,14 +67,16 @@ namespace brep string ccs, string mcs) : id (package_id (move (tnt), move (pnm), pvr), - move (cfg), move (trg), + move (tcf), + move (pcf), move (tnm), tvr), tenant (id.package.tenant), package_name (id.package.name), package_version (move (pvr)), - configuration (id.configuration), target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), toolchain_name (id.toolchain_name), toolchain_version (move (tvr)), state (build_state::building), @@ -93,19 +96,22 @@ namespace brep build_delay:: build_delay (string tnt, package_name_type pnm, version pvr, - string cfg, target_triplet trg, + string tcf, + string pcf, string tnm, version tvr, timestamp ptm) : id (package_id (move (tnt), move (pnm), pvr), - move (cfg), move (trg), + move (tcf), + move (pcf), move (tnm), tvr), tenant (id.package.tenant), package_name (id.package.name), package_version (move (pvr)), - configuration (id.configuration), target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), toolchain_name (id.toolchain_name), toolchain_version (move (tvr)), package_timestamp (ptm) diff --git a/libbrep/build.hxx b/libbrep/build.hxx index ceea81a..dfd0e55 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -26,9 +26,9 @@ // Used by the data migration entries. // -#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 19 +#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 19, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 20, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace @@ -44,20 +44,23 @@ namespace brep struct build_id { package_id package; - string configuration; target_triplet target; + string target_config_name; + string package_config_name; string toolchain_name; canonical_version toolchain_version; build_id () = default; build_id (package_id p, - string c, target_triplet t, + string tc, + string pc, string n, const brep::version& v) : package (move (p)), - configuration (move (c)), target (move (t)), + target_config_name (move (tc)), + package_config_name (move (pc)), toolchain_name (move (n)), toolchain_version (v) {} }; @@ -68,10 +71,13 @@ namespace brep if (x.package != y.package) return x.package < y.package; - if (int r = x.configuration.compare (y.configuration)) + if (int r = x.target.compare (y.target)) return r < 0; - if (int r = x.target.compare (y.target)) + if (int r = x.target_config_name.compare (y.target_config_name)) + return r < 0; + + if (int r = x.package_config_name.compare (y.package_config_name)) return r < 0; if (int r = x.toolchain_name.compare (y.toolchain_name)) @@ -88,32 +94,36 @@ namespace brep template <typename T> inline auto operator== (const T& x, const build_id& y) - -> decltype (x.package == y.package && - x.configuration == y.configuration && - x.target == y.target && - x.toolchain_name == y.toolchain_name && + -> decltype (x.package == y.package && + x.target == y.target && + x.target_config_name == y.target_config_name && + x.package_config_name == y.package_config_name && + x.toolchain_name == y.toolchain_name && x.toolchain_version.epoch == y.toolchain_version.epoch) { - return x.package == y.package && - x.configuration == y.configuration && - x.target == y.target && - x.toolchain_name == y.toolchain_name && + return x.package == y.package && + x.target == y.target && + x.target_config_name == y.target_config_name && + x.package_config_name == y.package_config_name && + x.toolchain_name == y.toolchain_name && compare_version_eq (x.toolchain_version, y.toolchain_version, true); } template <typename T> inline auto operator!= (const T& x, const build_id& y) - -> decltype (x.package == y.package && - x.configuration == y.configuration && - x.target == y.target && - x.toolchain_name == y.toolchain_name && + -> decltype (x.package == y.package && + x.target == y.target && + x.target_config_name == y.target_config_name && + x.package_config_name == y.package_config_name && + x.toolchain_name == y.toolchain_name && x.toolchain_version.epoch == y.toolchain_version.epoch) { - return x.package != y.package || - x.configuration != y.configuration || - x.target == y.target || - x.toolchain_name != y.toolchain_name || + return x.package != y.package || + x.target != y.target || + x.target_config_name != y.target_config_name || + x.package_config_name != y.package_config_name || + x.toolchain_name != y.toolchain_name || compare_version_ne (x.toolchain_version, y.toolchain_version, true); } @@ -190,10 +200,10 @@ namespace brep // the timestamp set to now, and the force state set to unforced. // build (string tenant, - package_name_type, - version, - string configuration, + package_name_type, version, target_triplet, + string target_config_name, + string package_config_name, string toolchain_name, version toolchain_version, optional<string> interactive, optional<string> agent_fingerprint, @@ -207,8 +217,9 @@ namespace brep string& tenant; // Tracks id.package.tenant. package_name_type& package_name; // Tracks id.package.name. upstream_version package_version; // Original of id.package.version. - string& configuration; // Tracks id.configuration. target_triplet& target; // Tracks id.target. + string& target_config_name; // Tracks id.target_config_name. + string& package_config_name; // Tracks id.package_config_name. string& toolchain_name; // Tracks id.toolchain_name. upstream_version toolchain_version; // Original of id.toolchain_version. @@ -288,8 +299,9 @@ namespace brep #pragma db member(package_name) transient #pragma db member(package_version) \ set(this.package_version.init (this.id.package.version, (?))) - #pragma db member(configuration) transient #pragma db member(target) transient + #pragma db member(target_config_name) transient + #pragma db member(package_config_name) transient #pragma db member(toolchain_name) transient #pragma db member(toolchain_version) \ set(this.toolchain_version.init (this.id.toolchain_version, (?))) @@ -312,8 +324,9 @@ namespace brep build () : tenant (id.package.tenant), package_name (id.package.name), - configuration (id.configuration), target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), toolchain_name (id.toolchain_name) {} }; @@ -408,8 +421,9 @@ namespace brep // build_delay (string tenant, package_name_type, version, - string configuration, target_triplet, + string target_config_name, + string package_config_name, string toolchain_name, version toolchain_version, timestamp package_timestamp); @@ -418,8 +432,9 @@ namespace brep string& tenant; // Tracks id.package.tenant. package_name_type& package_name; // Tracks id.package.name. upstream_version package_version; // Original of id.package.version. - string& configuration; // Tracks id.configuration. target_triplet& target; // Tracks id.target. + string& target_config_name; // Tracks id.target_config_name. + string& package_config_name; // Tracks id.package_config_name. string& toolchain_name; // Tracks id.toolchain_name. upstream_version toolchain_version; // Original of id.toolchain_version. @@ -447,8 +462,9 @@ namespace brep #pragma db member(package_name) transient #pragma db member(package_version) \ set(this.package_version.init (this.id.package.version, (?))) - #pragma db member(configuration) transient #pragma db member(target) transient + #pragma db member(target_config_name) transient + #pragma db member(package_config_name) transient #pragma db member(toolchain_name) transient #pragma db member(toolchain_version) \ set(this.toolchain_version.init (this.id.toolchain_version, (?))) @@ -459,8 +475,9 @@ namespace brep build_delay () : tenant (id.package.tenant), package_name (id.package.name), - configuration (id.configuration), target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), toolchain_name (id.toolchain_name) {} }; } diff --git a/libbrep/build.xml b/libbrep/build.xml index 7fe936d..d1969f1 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,5 +1,5 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> - <model version="19"> + <model version="20"> <table name="build" kind="object"> <column name="package_tenant" type="TEXT" null="false"/> <column name="package_name" type="CITEXT" null="false"/> @@ -7,8 +7,9 @@ <column name="package_version_canonical_upstream" type="TEXT" null="false"/> <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> - <column name="configuration" type="TEXT" null="false"/> <column name="target" type="TEXT" null="false"/> + <column name="target_config_name" type="TEXT" null="false"/> + <column name="package_config_name" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -41,8 +42,9 @@ <column name="package_version_canonical_upstream"/> <column name="package_version_canonical_release"/> <column name="package_version_revision"/> - <column name="configuration"/> <column name="target"/> + <column name="target_config_name"/> + <column name="package_config_name"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -60,8 +62,9 @@ <column name="package_version_canonical_upstream" type="TEXT" null="false"/> <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> - <column name="configuration" type="TEXT" null="false"/> <column name="target" type="TEXT" null="false"/> + <column name="target_config_name" type="TEXT" null="false"/> + <column name="package_config_name" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -78,8 +81,9 @@ <column name="package_version_canonical_upstream"/> <column name="package_version_canonical_release"/> <column name="package_version_revision"/> - <column name="configuration"/> <column name="target"/> + <column name="target_config_name"/> + <column name="package_config_name"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -92,8 +96,9 @@ <column name="package_version_canonical_upstream"/> <column name="package_version_canonical_release"/> <column name="package_version_revision"/> - <column name="configuration"/> <column name="target"/> + <column name="target_config_name"/> + <column name="package_config_name"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -108,8 +113,9 @@ <column name="package_version_canonical_upstream"/> <column name="package_version_canonical_release"/> <column name="package_version_revision"/> - <column name="configuration"/> <column name="target"/> + <column name="target_config_name"/> + <column name="package_config_name"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -127,8 +133,9 @@ <column name="package_version_canonical_upstream" type="TEXT" null="false"/> <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> - <column name="configuration" type="TEXT" null="false"/> <column name="target" type="TEXT" null="false"/> + <column name="target_config_name" type="TEXT" null="false"/> + <column name="package_config_name" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -148,8 +155,9 @@ <column name="package_version_canonical_upstream"/> <column name="package_version_canonical_release"/> <column name="package_version_revision"/> - <column name="configuration"/> <column name="target"/> + <column name="target_config_name"/> + <column name="package_config_name"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> diff --git a/libbrep/common.cxx b/libbrep/common.cxx index 4f729a3..c97a346 100644 --- a/libbrep/common.cxx +++ b/libbrep/common.cxx @@ -32,4 +32,14 @@ namespace brep else if (r == "unbuildable") return unbuildable_reason::unbuildable; else throw invalid_argument ("invalid unbuildable reason '" + r + '\''); } + + build_package_config* + find (const string& name, build_package_configs& cs) + { + auto i (find_if (cs.begin (), cs.end (), + [&name] (const build_package_config& c) + {return c.name == name;})); + + return i != cs.end () ? &*i : nullptr; + } } diff --git a/libbrep/common.hxx b/libbrep/common.hxx index af34f6c..158690e 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -4,6 +4,7 @@ #ifndef LIBBREP_COMMON_HXX #define LIBBREP_COMMON_HXX +#include <map> #include <ratio> #include <chrono> #include <type_traits> // static_assert @@ -332,6 +333,104 @@ namespace brep #pragma db value(build_constraint) definition + // build_package_config + // + using build_package_config = bpkg::build_package_config; + + #pragma db value(build_package_config) definition + + // 1 for the default configuration which is always present. + // + using build_package_configs = small_vector<build_package_config, 1>; + + // Return the address of the configuration object with the specified name, + // if present, and NULL otherwise. + // + build_package_config* + find (const string& name, build_package_configs&); + + // Note that ODB doesn't support containers of value types which contain + // containers. Thus, we will persist/load + // package_build_config::{builds,constraint} via the separate nested + // containers using the adapter classes. + // + #pragma db member(build_package_config::builds) transient + #pragma db member(build_package_config::constraints) transient + + using build_class_expr_key = odb::nested_key<build_class_exprs>; + using build_class_exprs_map = std::map<build_class_expr_key, build_class_expr>; + + #pragma db value(build_class_expr_key) + #pragma db member(build_class_expr_key::outer) column("config_index") + #pragma db member(build_class_expr_key::inner) column("index") + + // Adapter for build_package_config::builds. + // + class build_package_config_builds: + public small_vector<build_class_exprs, 1> // 1 as for build_package_configs. + { + public: + build_package_config_builds () = default; + + explicit + build_package_config_builds (const build_package_configs& cs) + { + reserve (cs.size ()); + for (const build_package_config& c: cs) + push_back (c.builds); + } + + void + to_configs (build_package_configs& cs) && + { + // Note that the empty trailing entries will be missing (see ODB's + // nested-container.hxx for details). + // + assert (size () <= cs.size ()); + + auto i (cs.begin ()); + for (build_class_exprs& ces: *this) + i++->builds = move (ces); + } + }; + + using build_constraint_key = odb::nested_key<build_constraints>; + using build_constraints_map = std::map<build_constraint_key, build_constraint>; + + #pragma db value(build_constraint_key) + #pragma db member(build_constraint_key::outer) column("config_index") + #pragma db member(build_constraint_key::inner) column("index") + + // Adapter for build_package_config::constraints. + // + class build_package_config_constraints: + public small_vector<build_constraints, 1> // 1 as for build_package_configs. + { + public: + build_package_config_constraints () = default; + + explicit + build_package_config_constraints (const build_package_configs& cs) + { + reserve (cs.size ()); + for (const build_package_config& c: cs) + push_back (c.constraints); + } + + void + to_configs (build_package_configs& cs) && + { + // Note that the empty trailing entries will be missing (see ODB's + // nested-container.hxx for details). + // + assert (size () <= cs.size ()); + + auto i (cs.begin ()); + for (build_constraints& bcs: *this) + i++->constraints = move (bcs); + } + }; + // The primary reason why a package is unbuildable by the build bot // controller service. // @@ -398,6 +497,8 @@ namespace brep std::map<requirement_alternative_key, requirement_alternative>; #pragma db value(requirement_alternative_key) + #pragma db member(requirement_alternative_key::outer) column("requirement_index") + #pragma db member(requirement_alternative_key::inner) column("index") using requirement_key = odb::nested2_key<requirement_alternatives>; @@ -405,6 +506,9 @@ namespace brep std::map<requirement_key, string>; #pragma db value(requirement_key) + #pragma db member(requirement_key::outer) column("requirement_index") + #pragma db member(requirement_key::middle) column("alternative_index") + #pragma db member(requirement_key::inner) column("index") // Version comparison operators. // diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 5f99fbb..152d1d8 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -77,6 +77,7 @@ namespace brep small_vector<test_dependency, 1> ts, build_class_exprs bs, build_constraints_type bc, + build_package_configs bcs, optional<path> lc, optional<string> fr, optional<string> sh, @@ -114,6 +115,24 @@ namespace brep fragment (move (fr)), sha256sum (move (sh)) { + // Add the default build configuration at the beginning, unless it is + // specified explicitly. + // + if (find_if (bcs.begin (), bcs.end (), + [] (const build_package_config& c) + {return c.name == "default";}) != bcs.end ()) + { + build_configs = move (bcs); + } + else + { + build_configs.reserve (bcs.size () + 1); + build_configs.emplace_back ("default"); + build_configs.insert (build_configs.end (), + make_move_iterator (bcs.begin ()), + make_move_iterator (bcs.end ())); + } + if (stub ()) unbuildable_reason = brep::unbuildable_reason::stub; else if (!internal_repository->buildable) diff --git a/libbrep/package.hxx b/libbrep/package.hxx index c586d74..4a68a07 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 25 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 25, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 26, closed) namespace brep { @@ -417,6 +417,9 @@ namespace brep // Create internal package object. // + // Note: adds the default build package config at the first position if it + // is not present yet. + // package (package_name, version_type, optional<string> upstream_version, @@ -443,6 +446,7 @@ namespace brep small_vector<test_dependency, 1> tests, build_class_exprs, build_constraints_type, + build_package_configs, optional<path> location, optional<string> fragment, optional<string> sha256sum, @@ -513,8 +517,14 @@ namespace brep requirements_type requirements; // Note: foreign-mapped in build. small_vector<test_dependency, 1> tests; // Note: foreign-mapped in build. + // Common build classes/constraints that apply to all configurations + // unless overridden. + // build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. + + build_package_configs build_configs; // Note: foreign-mapped in build. + odb::section build_section; // Note that it is foreign-mapped in build. @@ -634,9 +644,6 @@ namespace brep // Container of the requirement_alternative values. // - #pragma db member(requirement_alternative_key::outer) column("requirement_index") - #pragma db member(requirement_alternative_key::inner) column("index") - #pragma db member(requirement_alternatives) \ virtual(requirement_alternatives_map) \ after(requirements) \ @@ -646,10 +653,6 @@ namespace brep // Container of the requirement (string) values. // - #pragma db member(requirement_key::outer) column("requirement_index") - #pragma db member(requirement_key::middle) column("alternative_index") - #pragma db member(requirement_key::inner) column("index") - #pragma db member(requirement_alternative_requirements) \ virtual(requirement_alternative_requirements_map) \ after(requirement_alternatives) \ @@ -671,6 +674,37 @@ namespace brep #pragma db member(build_constraints) id_column("") value_column("") \ section(build_section) + // build_configs + // + // Note that build_package_config::{builds,constraints} are + // persisted/loaded via the separate nested containers (see commons.hxx + // for details). + // + #pragma db member(build_configs) id_column("") value_column("config_") \ + section(build_section) + + #pragma db member(build_config_builds) \ + virtual(build_class_exprs_map) \ + after(build_configs) \ + get(odb::nested_get ( \ + brep::build_package_config_builds (this.build_configs))) \ + set(brep::build_package_config_builds bs; \ + odb::nested_set (bs, std::move (?)); \ + move (bs).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(build_section) + + #pragma db member(build_config_constraints) \ + virtual(build_constraints_map) \ + after(build_config_builds) \ + get(odb::nested_get ( \ + brep::build_package_config_constraints (this.build_configs))) \ + set(brep::build_package_config_constraints cs; \ + odb::nested_set (cs, std::move (?)); \ + move (cs).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("") \ + section(build_section) + #pragma db member(build_section) load(lazy) update(always) // other_repositories diff --git a/libbrep/package.xml b/libbrep/package.xml index 2e7a613..4e5544a 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,138 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1"> + <changeset version="26"> + <add-table name="package_build_configs" kind="container"> + <column name="tenant" type="TEXT" null="false"/> + <column name="name" type="CITEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="config_name" type="TEXT" null="false"/> + <column name="config_arguments" type="TEXT" null="false"/> + <column name="config_comment" type="TEXT" null="false"/> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_build_configs_object_id_i"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_build_configs_index_i"> + <column name="index"/> + </index> + </add-table> + <add-table name="package_build_config_builds" kind="container"> + <column name="tenant" type="TEXT" null="false"/> + <column name="name" type="CITEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="config_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="expression" type="TEXT" null="false"/> + <column name="comment" type="TEXT" null="false"/> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_build_config_builds_object_id_i"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + </add-table> + <add-table name="package_build_config_constraints" kind="container"> + <column name="tenant" type="TEXT" null="false"/> + <column name="name" type="CITEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="config_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="exclusion" type="BOOLEAN" null="false"/> + <column name="config" type="TEXT" null="false"/> + <column name="target" type="TEXT" null="true"/> + <column name="comment" type="TEXT" null="false"/> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_build_config_constraints_object_id_i"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + </add-table> + </changeset> + <model version="25"> <table name="tenant" kind="object"> <column name="id" type="TEXT" null="false"/> diff --git a/libbrep/utility.hxx b/libbrep/utility.hxx index 7058c94..fce8fb5 100644 --- a/libbrep/utility.hxx +++ b/libbrep/utility.hxx @@ -4,11 +4,12 @@ #ifndef LIBBREP_UTILITY_HXX #define LIBBREP_UTILITY_HXX -#include <memory> // make_shared() -#include <string> // to_string() -#include <utility> // move(), forward(), declval(), make_pair() -#include <cassert> // assert() -#include <iterator> // make_move_iterator() +#include <memory> // make_shared() +#include <string> // to_string() +#include <utility> // move(), forward(), declval(), make_pair() +#include <cassert> // assert() +#include <iterator> // make_move_iterator() +#include <algorithm> // * #include <libbutl/utility.hxx> // icasecmp(), reverse_iterate(), // operator<<(ostream, exception) diff --git a/load/load.cxx b/load/load.cxx index 760ce28..000743c 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -5,10 +5,9 @@ #include <cerrno> #include <chrono> -#include <thread> // this_thread::sleep_for() -#include <cstring> // strncmp() +#include <thread> // this_thread::sleep_for() +#include <cstring> // strncmp() #include <iostream> -#include <algorithm> // find(), find_if() #include <odb/session.hxx> #include <odb/database.hxx> @@ -581,6 +580,8 @@ load_packages (const shared_ptr<repository>& rp, move (ts), move (pm.builds), move (pm.build_constraints), + build_package_configs (make_move_iterator (pm.build_configs.begin ()), + make_move_iterator (pm.build_configs.end ())), move (pm.location), move (pm.fragment), move (pm.sha256sum), diff --git a/mod/build-config-module.cxx b/mod/build-config-module.cxx index bf21fbb..eba753e 100644 --- a/mod/build-config-module.cxx +++ b/mod/build-config-module.cxx @@ -18,26 +18,25 @@ namespace brep using namespace std; using namespace butl; using namespace bpkg; - using namespace bbot; - // Return pointer to the shared build configurations instance, creating one - // on the first call. Throw tab_parsing on parsing error, io_error on the - // underlying OS error. Note: not thread-safe. + // Return pointer to the shared build target configurations instance, + // creating one on the first call. Throw tab_parsing on parsing error, + // io_error on the underlying OS error. Note: not thread-safe. // - static shared_ptr<const build_configs> + static shared_ptr<const build_target_configs> shared_build_config (const path& p) { - static map<path, weak_ptr<build_configs>> configs; + static map<path, weak_ptr<build_target_configs>> configs; auto i (configs.find (p)); if (i != configs.end ()) { - if (shared_ptr<build_configs> c = i->second.lock ()) + if (shared_ptr<build_target_configs> c = i->second.lock ()) return c; } - shared_ptr<build_configs> c ( - make_shared<build_configs> (parse_buildtab (p))); + shared_ptr<build_target_configs> c ( + make_shared<build_target_configs> (bbot::parse_buildtab (p))); configs[p] = c; return c; @@ -122,7 +121,7 @@ namespace brep { try { - build_conf_ = shared_build_config (bo.build_config ()); + target_conf_ = shared_build_config (bo.build_config ()); } catch (const io_error& e) { @@ -137,19 +136,21 @@ namespace brep bot_agent_key_map_ = shared_bot_agent_keys (bo, bo.build_bot_agent_keys ()); - using conf_map_type = map<build_config_id, const build_config*>; + using conf_map_type = map<build_target_config_id, + const build_target_config*>; + conf_map_type conf_map; - for (const auto& c: *build_conf_) - conf_map[build_config_id {c.name, c.target}] = &c; + for (const auto& c: *target_conf_) + conf_map[build_target_config_id {c.target, c.name}] = &c; - build_conf_map_ = make_shared<conf_map_type> (move (conf_map)); + target_conf_map_ = make_shared<conf_map_type> (move (conf_map)); } bool build_config_module:: - belongs (const bbot::build_config& cfg, const char* cls) const + belongs (const build_target_config& cfg, const char* cls) const { - const map<string, string>& im (build_conf_->class_inheritance_map); + const map<string, string>& im (target_conf_->class_inheritance_map); for (const string& c: cfg.classes) { diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index b276d6c..78661c3 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -10,13 +10,11 @@ #include <libbpkg/manifest.hxx> -#include <libbbot/build-config.hxx> - #include <libbrep/types.hxx> #include <libbrep/utility.hxx> -#include <mod/build-config.hxx> #include <mod/module-options.hxx> +#include <mod/build-target-config.hxx> // Base class for modules that utilize the build controller configuration. // @@ -39,16 +37,18 @@ namespace brep init (const options::build&); bool - exclude (const small_vector<bpkg::build_class_expr, 1>& exprs, - const vector<bpkg::build_constraint>& constrs, - const bbot::build_config& cfg, + exclude (const build_package_config& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, string* reason = nullptr, bool default_all_ucs = false) const { - return brep::exclude (exprs, - constrs, - cfg, - build_conf_->class_inheritance_map, + return brep::exclude (pc, + common_builds, + common_constraints, + tc, + target_conf_->class_inheritance_map, reason, default_all_ucs); } @@ -56,27 +56,30 @@ namespace brep // Check if the configuration belongs to the specified class. // bool - belongs (const bbot::build_config&, const char*) const; + belongs (const build_target_config&, const char*) const; bool - belongs (const bbot::build_config& cfg, const string& cls) const + belongs (const build_target_config& cfg, const string& cls) const { return belongs (cfg, cls.c_str ()); } - // Configuration/target/toolchain combination that, in particular, can be + // Target/configuration/toolchain combination that, in particular, can be // used as a set value. // - // Note: contains shallow references to the configuration, target, - // toolchain name, and version. + // Note: all members are the shallow references. // struct config_toolchain { - const string& configuration; const butl::target_triplet& target; + const string& target_config; + const string& package_config; const string& toolchain_name; const bpkg::version& toolchain_version; + // Note: the comparison reflects the order of unbuilt configurations on + // the Builds page. + // bool operator< (const config_toolchain& ct) const { @@ -86,20 +89,24 @@ namespace brep if (toolchain_version != ct.toolchain_version) return toolchain_version > ct.toolchain_version; - if (int r = configuration.compare (ct.configuration)) + if (int r = target.compare (ct.target)) + return r < 0; + + if (int r = target_config.compare (ct.target_config)) return r < 0; - return target.compare (ct.target) < 0; + return package_config.compare (ct.package_config) < 0; } }; protected: // Build configurations. // - shared_ptr<const bbot::build_configs> build_conf_; + shared_ptr<const build_target_configs> target_conf_; - shared_ptr<const std::map<build_config_id, const bbot::build_config*>> - build_conf_map_; + shared_ptr<const std::map<build_target_config_id, + const build_target_config*>> + target_conf_map_; // Map of build bot agent public keys fingerprints to the key file paths. // diff --git a/mod/build-config.hxx b/mod/build-config.hxx deleted file mode 100644 index 4ef01f6..0000000 --- a/mod/build-config.hxx +++ /dev/null @@ -1,72 +0,0 @@ -// file : mod/build-config.hxx -*- C++ -*- -// license : MIT; see accompanying LICENSE file - -#ifndef MOD_BUILD_CONFIG_HXX -#define MOD_BUILD_CONFIG_HXX - -#include <map> - -#include <libbutl/target-triplet.hxx> - -#include <libbpkg/manifest.hxx> - -#include <libbbot/build-config.hxx> - -#include <libbrep/types.hxx> -#include <libbrep/utility.hxx> - -namespace brep -{ - // Return true if the specified build configuration is excluded by a package - // based on its underlying build class set, build class expressions, and - // build constraints, potentially extending the underlying set with the - // special classes. Set the exclusion reason if requested. Optionally use - // the `all` class as a default underlying build class set rather than the - // `default` class (which is, for example, the case for the external test - // packages not to reduce their build configuration set needlessly). - // - bool - exclude (const small_vector<bpkg::build_class_expr, 1>&, - const vector<bpkg::build_constraint>&, - const bbot::build_config&, - const std::map<string, string>& class_inheritance_map, - string* reason = nullptr, - bool default_all_ucs = false); - - // Convert dash-separated components (target, build configuration name, - // machine name) or a pattern thereof into a path, replacing dashes with - // slashes (directory separators), `**` with `*/**/*`, and appending the - // trailing slash for a subsequent match using the path_match() - // functionality (the idea here is for `linux**` to match `linux-gcc` which - // is quite natural to expect). Throw invalid_path if the resulting path is - // invalid. - // - // Note that the match_absent path match flag must be used for the above - // `**` transformation to work. - // - path - dash_components_to_path (const string&); - - // Build configuration name/target combination that, in particular, - // identifies configurations in the buildtab and thus can be used as a - // set/map key. - // - // Note: contains shallow references to the configuration name and target. - // - struct build_config_id - { - reference_wrapper<const string> name; - reference_wrapper<const butl::target_triplet> target; - - bool - operator< (const build_config_id& x) const - { - if (int r = name.get ().compare (x.name.get ())) - return r < 0; - - return target.get ().compare (x.target.get ()) < 0; - } - }; -} - -#endif // MOD_BUILD_CONFIG diff --git a/mod/build-config.cxx b/mod/build-target-config.cxx index 8fbbf99..a30cf07 100644 --- a/mod/build-config.cxx +++ b/mod/build-target-config.cxx @@ -1,7 +1,7 @@ -// file : mod/build-config-module.cxx -*- C++ -*- +// file : mod/target-build-config.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file -#include <mod/build-config.hxx> +#include <mod/build-target-config.hxx> #include <libbutl/utility.hxx> // alpha(), etc. #include <libbutl/path-pattern.hxx> @@ -11,7 +11,6 @@ namespace brep using namespace std; using namespace butl; using namespace bpkg; - using namespace bbot; // The default underlying class set expressions (see below). // @@ -22,13 +21,17 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const small_vector<build_class_expr, 1>& exprs, - const vector<build_constraint>& constrs, - const build_config& cfg, + exclude (const build_package_config& pc, + const build_class_exprs& cbs, + const build_constraints& ccs, + const build_target_config& tc, const map<string, string>& class_inheritance_map, string* reason, bool default_all_ucs) { + const build_class_exprs& exprs (pc.effective_builds (cbs)); + const build_constraints& constrs (pc.effective_constraints (ccs)); + // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). @@ -74,11 +77,11 @@ namespace brep // (changing the result from true to false) or non-including one (leaving // the false result) as an exclusion reason. // - auto match = [&cfg, &m, reason, &sanitize, &class_inheritance_map] + auto match = [&tc, &m, reason, &sanitize, &class_inheritance_map] (const build_class_expr& e) { bool pm (m); - e.match (cfg.classes, class_inheritance_map, m); + e.match (tc.classes, class_inheritance_map, m); if (reason != nullptr) { @@ -168,8 +171,8 @@ namespace brep if (!constrs.empty ()) try { - path cn (dash_components_to_path (cfg.name)); - path tg (dash_components_to_path (cfg.target.string ())); + path cn (dash_components_to_path (tc.name)); + path tg (dash_components_to_path (tc.target.string ())); for (const build_constraint& c: constrs) { diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx new file mode 100644 index 0000000..180ca80 --- /dev/null +++ b/mod/build-target-config.hxx @@ -0,0 +1,79 @@ +// file : mod/build-target-config.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_BUILD_TARGET_CONFIG_HXX +#define MOD_BUILD_TARGET_CONFIG_HXX + +#include <map> + +#include <libbutl/target-triplet.hxx> + +#include <libbpkg/manifest.hxx> + +#include <libbbot/build-target-config.hxx> + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <libbrep/common.hxx> + +namespace brep +{ + using build_target_config = bbot::build_target_config; + using build_target_configs = bbot::build_target_configs; + + // Return true if the specified build target configuration is excluded by a + // package configuration based on its underlying build class set, build + // class expressions, and build constraints, potentially extending the + // underlying set with the special classes. Set the exclusion reason if + // requested. Optionally use the `all` class as a default underlying build + // class set rather than the `default` class (which is, for example, the + // case for the external test packages not to reduce their build target + // configuration set needlessly). + // + bool + exclude (const build_package_config&, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config&, + const std::map<string, string>& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false); + + // Convert dash-separated components (target, build target configuration + // name, machine name) or a pattern thereof into a path, replacing dashes + // with slashes (directory separators), `**` with `*/**/*`, and appending + // the trailing slash for a subsequent match using the path_match() + // functionality (the idea here is for `linux**` to match `linux-gcc` which + // is quite natural to expect). Throw invalid_path if the resulting path is + // invalid. + // + // Note that the match_absent path match flag must be used for the above + // `**` transformation to work. + // + path + dash_components_to_path (const string&); + + // Build target/target configuration name combination that, in particular, + // identifies configurations in the buildtab and thus can be used as a + // set/map key. + // + // Note: contains shallow references to the target and configuration name. + // + struct build_target_config_id + { + reference_wrapper<const butl::target_triplet> target; + reference_wrapper<const string> config; + + bool + operator< (const build_target_config_id& x) const + { + if (int r = target.get ().compare (x.target.get ())) + return r < 0; + + return config.get ().compare (x.config.get ()) < 0; + } + }; +} + +#endif // MOD_BUILD_TARGET_CONFIG diff --git a/mod/build.cxx b/mod/build.cxx index 3b82aed..4abd416 100644 --- a/mod/build.cxx +++ b/mod/build.cxx @@ -20,13 +20,15 @@ namespace brep // needs to be url-encoded, and only in the query part of the URL. We embed // the package version into the URL path part and so don't encode it. // - string url (host + tenant_dir (root, b.tenant).representation () + - mime_url_encode (b.package_name.string (), false) + '/' + - b.package_version.string () + "/log/" + - mime_url_encode (b.configuration, false /* query */) + '/' + - mime_url_encode (b.target.string (), false /* query */) + '/' + - mime_url_encode (b.toolchain_name, false /* query */) + '/' + - b.toolchain_version.string ()); + string url ( + host + tenant_dir (root, b.tenant).representation () + + mime_url_encode (b.package_name.string (), false) + '/' + + b.package_version.string () + "/log/" + + mime_url_encode (b.target.string (), false /* query */) + '/' + + mime_url_encode (b.target_config_name, false /* query */) + '/' + + mime_url_encode (b.package_config_name, false /* query */) + '/' + + mime_url_encode (b.toolchain_name, false /* query */) + '/' + + b.toolchain_version.string ()); if (op != nullptr) { @@ -45,13 +47,14 @@ namespace brep // we embed the package version into the URL query part, where it is not // encoded by design. // - return host + tenant_dir (root, b.tenant).string () + + return host + tenant_dir (root, b.tenant).string () + "?build-force&pn=" + mime_url_encode (b.package_name.string ()) + - "&pv=" + b.package_version.string () + - "&cf=" + mime_url_encode (b.configuration) + - "&tg=" + mime_url_encode (b.target.string ()) + - "&tn=" + mime_url_encode (b.toolchain_name) + - "&tv=" + b.toolchain_version.string () + + "&pv=" + b.package_version.string () + + "&tg=" + mime_url_encode (b.target.string ()) + + "&tc=" + mime_url_encode (b.target_config_name) + + "&pc=" + mime_url_encode (b.package_config_name) + + "&tn=" + mime_url_encode (b.toolchain_name) + + "&tv=" + b.toolchain_version.string () + "&reason="; } } diff --git a/mod/buildfile b/mod/buildfile index 58a3caf..6693a35 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -25,7 +25,7 @@ include ../web/server/ ./: mod{brep} {libue libus}{mod} -libu_src = options-types types-parsers build-config +libu_src = options-types types-parsers build-target-config mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ libus{mod} ../libbrep/lib{brep} ../web/server/libus{web-server} \ diff --git a/mod/mod-build-configs.cxx b/mod/mod-build-configs.cxx index 126c1d8..79c47f7 100644 --- a/mod/mod-build-configs.cxx +++ b/mod/mod-build-configs.cxx @@ -3,8 +3,6 @@ #include <mod/mod-build-configs.hxx> -#include <algorithm> // replace() - #include <libstudxml/serializer.hxx> #include <web/server/module.hxx> @@ -15,7 +13,6 @@ #include <mod/module-options.hxx> using namespace std; -using namespace bbot; using namespace brep::cli; // While currently the user-defined copy constructor is not required (we don't @@ -49,7 +46,7 @@ handle (request& rq, response& rs) HANDLER_DIAG; - if (build_conf_ == nullptr) + if (target_conf_ == nullptr) throw invalid_request (501, "not implemented"); const size_t page_configs (options_->build_config_page_entries ()); @@ -120,8 +117,8 @@ handle (request& rq, response& rs) // if (params.page () == 0) { - const strings& cls (build_conf_->classes); - const map<string, string>& im (build_conf_->class_inheritance_map); + const strings& cls (target_conf_->classes); + const map<string, string>& im (target_conf_->class_inheritance_map); s << DIV(ID="filter-heading") << "Build Configuration Classes" << ~DIV << P(ID="filter"); @@ -155,12 +152,12 @@ handle (request& rq, response& rs) // before printing the configurations. // size_t count (0); - vector<const build_config*> configs; + vector<const build_target_config*> configs; configs.reserve (page_configs); size_t skip (page * page_configs); size_t print (page_configs); - for (const build_config& c: *build_conf_) + for (const build_target_config& c: *target_conf_) { if (belongs (c, params.class_name ())) { @@ -185,7 +182,7 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - for (const build_config* c: configs) + for (const build_target_config* c: configs) { s << TABLE(CLASS="proplist config") << TBODY diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index 281c76c..af4b8e9 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -3,8 +3,6 @@ #include <mod/mod-build-force.hxx> -#include <algorithm> // replace() - #include <odb/database.hxx> #include <odb/transaction.hxx> @@ -16,7 +14,6 @@ #include <mod/module-options.hxx> using namespace std; -using namespace bbot; using namespace brep::cli; using namespace odb::core; @@ -115,11 +112,6 @@ handle (request& rq, response& rs) version package_version (parse_version (params.version (), "package version")); - string& config (params.configuration ()); - - if (config.empty ()) - throw invalid_argument ("no configuration name"); - target_triplet target; try @@ -131,6 +123,16 @@ handle (request& rq, response& rs) throw invalid_argument (string ("invalid target: ") + e.what ()); } + string& target_config (params.target_config ()); + + if (target_config.empty ()) + throw invalid_argument ("no target configuration name"); + + string& package_config (params.package_config ()); + + if (package_config.empty ()) + throw invalid_argument ("no package configuration name"); + string& toolchain_name (params.toolchain_name ()); if (toolchain_name.empty ()) @@ -140,8 +142,9 @@ handle (request& rq, response& rs) "toolchain version")); id = build_id (package_id (move (tenant), move (p), package_version), - move (config), move (target), + move (target_config), + move (package_config), move (toolchain_name), toolchain_version); } @@ -161,9 +164,11 @@ handle (request& rq, response& rs) // Make sure the build configuration still exists. // - if (build_conf_map_->find (build_config_id {id.configuration, id.target}) == - build_conf_map_->end ()) - config_expired ("no configuration"); + if (target_conf_map_->find ( + build_target_config_id {id.target, + id.target_config_name}) == + target_conf_map_->end ()) + config_expired ("no target configuration"); // Load the package build configuration (if present), set the force flag and // update the object's persistent state. @@ -189,7 +194,8 @@ handle (request& rq, response& rs) l1 ([&]{trace << "force rebuild for " << b->tenant << ' ' << b->package_name << '/' << b->package_version << ' ' - << b->configuration << '/' << b->target << ' ' + << b->target_config_name << '/' << b->target << ' ' + << b->package_config_name << ' ' << b->toolchain_name << '-' << b->toolchain_version << ": " << reason;}); } diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index 328d178..3841fad 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -3,8 +3,6 @@ #include <mod/mod-build-log.hxx> -#include <algorithm> // find_if() - #include <odb/database.hxx> #include <odb/transaction.hxx> @@ -18,7 +16,6 @@ #include <mod/module-options.hxx> using namespace std; -using namespace bbot; using namespace brep::cli; using namespace odb::core; @@ -124,14 +121,6 @@ handle (request& rq, response& rs) assert (i != lpath.end () && *i == "log"); if (++i == lpath.end ()) - throw invalid_argument ("no configuration name"); - - string config (*i++); - - if (config.empty ()) - throw invalid_argument ("empty configuration name"); - - if (i == lpath.end ()) throw invalid_argument ("no target"); target_triplet target; @@ -145,6 +134,22 @@ handle (request& rq, response& rs) } if (i == lpath.end ()) + throw invalid_argument ("no target configuration name"); + + string target_config (*i++); + + if (target_config.empty ()) + throw invalid_argument ("empty target configuration name"); + + if (i == lpath.end ()) + throw invalid_argument ("no package configuration name"); + + string package_config (*i++); + + if (package_config.empty ()) + throw invalid_argument ("empty package configuration name"); + + if (i == lpath.end ()) throw invalid_argument ("no toolchain name"); string toolchain_name (*i++); @@ -158,8 +163,9 @@ handle (request& rq, response& rs) version toolchain_version (parse_version (*i++, "toolchain version")); id = build_id (package_id (tenant, move (name), package_version), - move (config), move (target), + move (target_config), + move (package_config), move (toolchain_name), toolchain_version); @@ -204,9 +210,11 @@ handle (request& rq, response& rs) // Make sure the build configuration still exists. // - if (build_conf_map_->find (build_config_id {id.configuration, id.target}) == - build_conf_map_->end ()) - config_expired ("no configuration"); + if (target_conf_map_->find ( + build_target_config_id {id.target, + id.target_config_name}) == + target_conf_map_->end ()) + config_expired ("no target configuration"); // Load the package build configuration (if present). // @@ -242,15 +250,14 @@ handle (request& rq, response& rs) if (!b->tenant.empty ()) os << options_->tenant_name () << ": " << b->tenant << endl << endl; - os << "package: " << b->package_name << endl - << "version: " << b->package_version << endl - << "toolchain: " << b->toolchain_name << '-' << b->toolchain_version - << endl - << "config: " << b->configuration << endl - << "target: " << b->target << endl - << "machine: " << b->machine << " (" << b->machine_summary << ")" - << endl - << "timestamp: "; + os << "package: " << b->package_name << endl + << "version: " << b->package_version << endl + << "toolchain: " << b->toolchain_name << '-' << b->toolchain_version << endl + << "target: " << b->target << endl + << "tgt config: " << b->target_config_name << endl + << "pkg config: " << b->package_config_name << endl + << "machine: " << b->machine << " (" << b->machine_summary << ")" << endl + << "timestamp: "; butl::to_stream (os, b->timestamp, diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 7eefe95..6f40a0b 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -189,22 +189,11 @@ handle (request& rq, response&) if (package_version != rqm.result.version) throw invalid_argument ("package version mismatch"); - b = p + 1; // Start of configuration name. - p = s.find ('/', b); // End of configuration name. - - if (p == string::npos) - throw invalid_argument ("no target"); - - string config (s, b, p - b); - - if (config.empty ()) - throw invalid_argument ("empty configuration name"); - b = p + 1; // Start of target. p = s.find ('/', b); // End of target. if (p == string::npos) - throw invalid_argument ("no toolchain name"); + throw invalid_argument ("no target configuration name"); target_triplet target; try @@ -216,6 +205,28 @@ handle (request& rq, response&) throw invalid_argument (string ("invalid target: ") + e.what ()); } + b = p + 1; // Start of target configuration name. + p = s.find ('/', b); // End of target configuration name. + + if (p == string::npos) + throw invalid_argument ("no package configuration name"); + + string target_config (s, b, p - b); + + if (target_config.empty ()) + throw invalid_argument ("empty target configuration name"); + + b = p + 1; // Start of package configuration name. + p = s.find ('/', b); // End of package configuration name. + + if (p == string::npos) + throw invalid_argument ("no toolchain name"); + + string package_config (s, b, p - b); + + if (package_config.empty ()) + throw invalid_argument ("empty package configuration name"); + b = p + 1; // Start of toolchain name. p = s.find ('/', b); // End of toolchain name. @@ -236,8 +247,9 @@ handle (request& rq, response&) version toolchain_version (parse_version ("toolchain version")); id = build_id (package_id (move (tenant), move (name), package_version), - move (config), move (target), + move (target_config), + move (package_config), move (toolchain_name), toolchain_version); @@ -278,18 +290,18 @@ handle (request& rq, response&) // Make sure the build configuration still exists. // - const bbot::build_config* cfg; + const build_target_config* tc; { - auto i (build_conf_map_->find (build_config_id {id.configuration, - id.target})); + auto i (target_conf_map_->find ( + build_target_config_id {id.target, id.target_config_name})); - if (i == build_conf_map_->end ()) + if (i == target_conf_map_->end ()) { warn_expired ("no build configuration"); return true; } - cfg = i->second; + tc = i->second; } // Load the built package (if present). @@ -526,13 +538,23 @@ handle (request& rq, response&) // `skip`, the configuration is hidden, or is now excluded by the // package. // - if (rqm.result.status != result_status::skip && belongs (*cfg, "all")) + if (rqm.result.status != result_status::skip && belongs (*tc, "all")) { shared_ptr<build_package> p ( build_db_->load<build_package> (b->id.package)); - if (!exclude (p->builds, p->constraints, *cfg)) - bld = move (b); + // The package configuration should be present (see mod-builds.cxx + // for details) but if it is not, let's log the warning. + // + if (const build_package_config* pc = find (b->package_config_name, + p->configs)) + { + if (!exclude (*pc, p->builds, p->constraints, *tc)) + bld = move (b); + } + else + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << p->id.name << '/' << p->version; } } } @@ -554,12 +576,13 @@ handle (request& rq, response&) return true; } - string subj ((unforced ? "build " : "rebuild ") + - to_string (*bld->status) + ": " + - bld->package_name.string () + '/' + - bld->package_version.string () + '/' + - bld->configuration + '/' + - bld->target.string () + '/' + + string subj ((unforced ? "build " : "rebuild ") + + to_string (*bld->status) + ": " + + bld->package_name.string () + '/' + + bld->package_version.string () + ' ' + + bld->target_config_name + '/' + + bld->target.string () + ' ' + + bld->package_config_name + ' ' + bld->toolchain_name + '-' + bld->toolchain_version.string ()); // Send notification emails to the interested parties. diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 5ba6d06..a15ff90 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -21,7 +21,6 @@ #include <libbutl/manifest-serializer.hxx> #include <libbbot/manifest.hxx> -#include <libbbot/build-config.hxx> #include <web/server/module.hxx> @@ -30,6 +29,8 @@ #include <libbrep/build-package.hxx> #include <libbrep/build-package-odb.hxx> +#include <mod/build-target-config.hxx> + #include <mod/module-options.hxx> using namespace std; @@ -155,20 +156,20 @@ handle (request& rq, response& rs) task_response_manifest tsm; - // Map build configurations to machines that are capable of building them. - // The first matching machine is selected for each configuration. + // Map build target configurations to machines that are capable of building + // them. The first matching machine is selected for each configuration. // struct config_machine { - const build_config* config; + const build_target_config* config; machine_header_manifest* machine; }; - using config_machines = map<build_config_id, config_machine>; + using config_machines = map<build_target_config_id, config_machine>; config_machines conf_machines; - for (const auto& c: *build_conf_) + for (const auto& c: *target_conf_) { for (auto& m: tqm.machines) { @@ -180,24 +181,25 @@ handle (request& rq, response& rs) dash_components_to_path (c.machine_pattern), dir_path () /* start */, path_match_flags::match_absent)) - conf_machines.emplace (build_config_id {c.name, c.target}, + conf_machines.emplace (build_target_config_id {c.target, c.name}, config_machine {&c, &m}); } catch (const invalid_path&) {} } } - // Go through packages until we find one that has no build configuration - // present in the database, or is in the building state but expired - // (collectively called unbuilt). If such a package configuration is found - // then put it into the building state, set the current timestamp and respond - // with the task for building this package configuration. + // Go through package build configurations until we find one that has no + // build target configuration present in the database, or is in the building + // state but expired (collectively called unbuilt). If such a target + // configuration is found then put it into the building state, set the + // current timestamp and respond with the task for building this package + // configuration. // // While trying to find a non-built package configuration we will also - // collect the list of the built package configurations which it's time to - // rebuild. So if no unbuilt package is found, we will pickup one to - // rebuild. The rebuild preference is given in the following order: the - // greater force state, the greater overall status, the lower timestamp. + // collect the list of the built configurations which it's time to + // rebuild. So if no unbuilt package configuration is found, we will pickup + // one to rebuild. The rebuild preference is given in the following order: + // the greater force state, the greater overall status, the lower timestamp. // if (!conf_machines.empty ()) { @@ -208,6 +210,7 @@ handle (request& rq, response& rs) // auto task = [this] (shared_ptr<build>&& b, shared_ptr<build_package>&& p, + build_package_config&& pc, shared_ptr<build_tenant>&& t, const config_machine& cm) -> task_response_manifest { @@ -215,12 +218,13 @@ handle (request& rq, response& rs) chrono::duration_cast<std::chrono::nanoseconds> ( b->timestamp.time_since_epoch ()).count ()); - string session (b->tenant + '/' + - b->package_name.string () + '/' + - b->package_version.string () + '/' + - b->configuration + '/' + - b->target.string () + '/' + - b->toolchain_name + '/' + + string session (b->tenant + '/' + + b->package_name.string () + '/' + + b->package_version.string () + '/' + + b->target.string () + '/' + + b->target_config_name + '/' + + b->package_config_name + '/' + + b->toolchain_name + '/' + b->toolchain_version.string () + '/' + to_string (ts)); @@ -255,13 +259,24 @@ handle (request& rq, response& rs) { shared_ptr<build_package> p (td.package.load ()); + // Use the default test package configuration. + // + // Note that potentially the test package default configuration may + // contain some (bpkg) arguments associated, but we currently don't + // provide build bot worker with such information. This, however, is + // probably too far fetched so let's keep it simple for now. + // + const build_package_config* pc (find ("default", p->configs)); + assert (pc != nullptr); // Must always be present. + // Use the `all` class as a least restrictive default underlying // build class set. Note that we should only apply the explicit // build restrictions to the external test packages (think about // the `builds: all` and `builds: -windows` manifest values for // the primary and external test packages, respectively). // - if (exclude (p->builds, + if (exclude (*pc, + p->builds, p->constraints, *cm.config, nullptr /* reason */, @@ -290,6 +305,7 @@ handle (request& rq, response& rs) cm.config->target, cm.config->environment, cm.config->args, + move (pc.arguments), belongs (*cm.config, module_pkg ? "build2" : "host"), cm.config->warning_regexes, move (t->interactive), @@ -583,23 +599,26 @@ handle (request& rq, response& rs) // Prepare the build prepared query. // // Note that we can not query the database for configurations that a - // package was not built with, as the database contains only those package + // package was not built with, as the database contains only those build // configurations that have already been acted upon (initially empty). // - // This is why we query the database for package configurations that - // should not be built (in the built state, or in the building state and - // not expired). Having such a list we will select the first build + // This is why we query the database for configurations that should not be + // built (in the built state, or in the building state and not + // expired). Having such a list we will select the first build // configuration that is not in the list (if available) for the response. // using bld_query = query<build>; using prep_bld_query = prepared_query<build>; package_id id; + string pkg_config_name; bld_query sq (false); for (const auto& cm: conf_machines) - sq = sq || (bld_query::id.configuration == cm.first.name && - bld_query::id.target == cm.first.target); + sq = sq || (bld_query::id.target == cm.first.target && + bld_query::id.target_config_name == cm.first.config && + bld_query::id.package_config_name == + bld_query::_ref (pkg_config_name)); bld_query bq ( equal<build> (bld_query::id.package, id) && @@ -658,7 +677,7 @@ handle (request& rq, response& rs) // target, environment, arguments, and warning-detecting regular // expressions. // - auto controller_checksum = [] (const build_config& c) + auto controller_checksum = [] (const build_target_config& c) { sha256 cs ("1"); // Hash the logic version. @@ -705,149 +724,157 @@ handle (request& rq, response& rs) { id = move (bp.id); - // Iterate through the package configurations and erase those that - // don't need building from the build configuration map. All those - // configurations that remained can be built. We will take the first - // one, if present. - // - // Also save the built package configurations for which it's time to - // be rebuilt. - // - config_machines configs (conf_machines); // Make a copy for this pkg. - auto pkg_builds (bld_prep_query.execute ()); + shared_ptr<build_package> p (build_db_->load<build_package> (id)); - for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) + for (build_package_config& pc: p->configs) { - auto j (configs.find (build_config_id {i->id.configuration, - i->id.target})); + pkg_config_name = pc.name; - // Outdated configurations are already excluded with the database - // query. + // Iterate through the built configurations and erase them from the + // build configuration map. All those configurations that remained + // can be built. We will take the first one, if present. // - assert (j != configs.end ()); - configs.erase (j); - - if (i->state == build_state::built) - { - assert (i->force != force_state::forcing); - - if (needs_rebuild (*i)) - rebuilds.emplace_back (i.load ()); - } - } - - if (!configs.empty ()) - { - // Find the first build configuration that is not excluded by the - // package. + // Also save the built configurations for which it's time to be + // rebuilt. // - shared_ptr<build_package> p (build_db_->load<build_package> (id)); + config_machines configs (conf_machines); // Make a copy for this pkg. + auto pkg_builds (bld_prep_query.execute ()); - auto i (configs.begin ()); - auto e (configs.end ()); - - for (; - i != e && - exclude (p->builds, p->constraints, *i->second.config); - ++i) ; - - if (i != e) + for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); - - build_id bid (move (id), - cm.config->name, - cm.config->target, - move (tqm.toolchain_name), - toolchain_version); - - shared_ptr<build> b (build_db_->find<build> (bid)); - optional<string> cl (challenge ()); - - shared_ptr<build_tenant> t ( - build_db_->load<build_tenant> (bid.package.tenant)); + auto j ( + configs.find (build_target_config_id {i->id.target, + i->id.target_config_name})); - // Move the interactive build login information into the build - // object, if the package to be built interactively. + // Outdated configurations are already excluded with the database + // query. // - optional<string> login (t->interactive - ? move (tqm.interactive_login) - : nullopt); + assert (j != configs.end ()); + configs.erase (j); - // If build configuration doesn't exist then create the new one - // and persist. Otherwise put it into the building state, refresh - // the timestamp and update. - // - if (b == nullptr) + if (i->state == build_state::built) { - b = make_shared<build> (move (bid.package.tenant), - move (bid.package.name), - move (bp.version), - move (bid.configuration), - move (bid.target), - move (bid.toolchain_name), - move (toolchain_version), - move (login), - move (agent_fp), - move (cl), - mh.name, - move (mh.summary), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); - - build_db_->persist (b); + assert (i->force != force_state::forcing); + + if (needs_rebuild (*i)) + rebuilds.emplace_back (i.load ()); } - else - { - // The package configuration is in the building state. - // - // Note that in both cases we keep the status intact to be able - // to compare it with the final one in the result request - // handling in order to decide if to send the notification - // email. The same is true for the forced flag (in the sense - // that we don't set the force state to unforced). - // - assert (b->state == build_state::building); + } - b->state = build_state::building; - b->interactive = move (login); + if (!configs.empty ()) + { + // Find the first build configuration that is not excluded by the + // package configuration. + // + auto i (configs.begin ()); + auto e (configs.end ()); - // Switch the force state not to reissue the task after the - // forced rebuild timeout. Note that the result handler will - // still recognize that the rebuild was forced. - // - if (b->force == force_state::forcing) - b->force = force_state::forced; + for (; + i != e && + exclude (pc, p->builds, p->constraints, *i->second.config); + ++i) ; - b->agent_fingerprint = move (agent_fp); - b->agent_challenge = move (cl); - b->machine = mh.name; - b->machine_summary = move (mh.summary); + if (i != e) + { + config_machine& cm (i->second); + machine_header_manifest& mh (*cm.machine); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + build_id bid (move (id), + cm.config->target, + cm.config->name, + move (pkg_config_name), + move (tqm.toolchain_name), + toolchain_version); - // Issue the hard rebuild if it is forced or the configuration - // or machine has changed. - // - if (b->hard_timestamp <= hard_rebuild_expiration || - b->force == force_state::forced || - b->controller_checksum != ccs || - b->machine_checksum != mcs) - convert_to_hard (b); + shared_ptr<build> b (build_db_->find<build> (bid)); + optional<string> cl (challenge ()); - b->controller_checksum = move (ccs); - b->machine_checksum = move (mcs); + shared_ptr<build_tenant> t ( + build_db_->load<build_tenant> (bid.package.tenant)); - b->timestamp = system_clock::now (); + // Move the interactive build login information into the build + // object, if the package to be built interactively. + // + optional<string> login (t->interactive + ? move (tqm.interactive_login) + : nullopt); - build_db_->update (b); + // If build configuration doesn't exist then create the new one + // and persist. Otherwise put it into the building state, refresh + // the timestamp and update. + // + if (b == nullptr) + { + b = make_shared<build> (move (bid.package.tenant), + move (bid.package.name), + move (bp.version), + move (bid.target), + move (bid.target_config_name), + move (bid.package_config_name), + move (bid.toolchain_name), + move (toolchain_version), + move (login), + move (agent_fp), + move (cl), + mh.name, + move (mh.summary), + controller_checksum (*cm.config), + machine_checksum (*cm.machine)); + + build_db_->persist (b); + } + else + { + // The build configuration is in the building state. + // + // Note that in both cases we keep the status intact to be + // able to compare it with the final one in the result request + // handling in order to decide if to send the notification + // email. The same is true for the forced flag (in the sense + // that we don't set the force state to unforced). + // + assert (b->state == build_state::building); + + b->state = build_state::building; + b->interactive = move (login); + + // Switch the force state not to reissue the task after the + // forced rebuild timeout. Note that the result handler will + // still recognize that the rebuild was forced. + // + if (b->force == force_state::forcing) + b->force = force_state::forced; + + b->agent_fingerprint = move (agent_fp); + b->agent_challenge = move (cl); + b->machine = mh.name; + b->machine_summary = move (mh.summary); + + string ccs (controller_checksum (*cm.config)); + string mcs (machine_checksum (*cm.machine)); + + // Issue the hard rebuild if it is forced or the configuration + // or machine has changed. + // + if (b->hard_timestamp <= hard_rebuild_expiration || + b->force == force_state::forced || + b->controller_checksum != ccs || + b->machine_checksum != mcs) + convert_to_hard (b); + + b->controller_checksum = move (ccs); + b->machine_checksum = move (mcs); + + b->timestamp = system_clock::now (); + + build_db_->update (b); + } + + // Finally, prepare the task response manifest. + // + tsm = task (move (b), move (p), move (pc), move (t), cm); + break; // Bail out from the package configurations loop. } - - // Finally, prepare the task response manifest. - // - tsm = task (move (b), move (p), move (t), cm); } } @@ -861,13 +888,12 @@ handle (request& rq, response& rs) t.commit (); } - // If we don't have an unbuilt package, then let's see if we have a - // package to rebuild. + // If we don't have an unbuilt package, then let's see if we have a build + // configuration to rebuild. // if (tsm.session.empty () && !rebuilds.empty ()) { - // Sort the package configuration rebuild list with the following sort - // priority: + // Sort the configuration rebuild list with the following sort priority: // // 1: force state // 2: overall status @@ -890,7 +916,7 @@ handle (request& rq, response& rs) optional<string> cl (challenge ()); - // Pick the first package configuration from the ordered list. + // Pick the first build configuration from the ordered list. // // Note that the configurations and packages may not match the required // criteria anymore (as we have committed the database transactions that @@ -911,8 +937,9 @@ handle (request& rq, response& rs) b->state == build_state::built && needs_rebuild (*b)) { - auto i (conf_machines.find (build_config_id {b->configuration, - b->target})); + auto i (conf_machines.find ( + build_target_config_id {b->target, + b->target_config_name})); // Only actual package configurations are loaded (see above). // @@ -934,12 +961,17 @@ handle (request& rq, response& rs) ? build_db_->load<build_tenant> (p->id.tenant) : nullptr); - if (p != nullptr && + build_package_config* pc (p != nullptr + ? find (b->package_config_name, + p->configs) + : nullptr); + + if (pc != nullptr && p->buildable && (imode == interactive_mode::both || (t->interactive.has_value () == (imode == interactive_mode::true_))) && - !exclude (p->builds, p->constraints, *cm.config)) + !exclude (*pc, p->builds, p->constraints, *cm.config)) { assert (b->status); @@ -985,7 +1017,7 @@ handle (request& rq, response& rs) build_db_->update (b); - tsm = task (move (b), move (p), move (t), cm); + tsm = task (move (b), move (p), move (*pc), move (t), cm); } } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index b9d841d..095dee3 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -4,7 +4,6 @@ #include <mod/mod-builds.hxx> #include <set> -#include <algorithm> // find_if() #include <libstudxml/serializer.hxx> @@ -32,7 +31,6 @@ using namespace std; using namespace butl; -using namespace bbot; using namespace web; using namespace odb::core; using namespace brep::cli; @@ -138,7 +136,7 @@ match (const C qc, const string& pattern) // template <typename T> static inline query<T> -build_query (const brep::vector<brep::build_config_id>* config_ids, +build_query (const brep::vector<brep::build_target_config_id>* config_ids, const brep::params::builds& params, const brep::optional<brep::string>& tenant, const brep::optional<bool>& archived) @@ -159,8 +157,8 @@ build_query (const brep::vector<brep::build_config_id>* config_ids, { query sq (false); for (const auto& id: *config_ids) - sq = sq || (qb::id.configuration == id.name && - qb::id.target == id.target); + sq = sq || (qb::id.target == id.target && + qb::id.target_config_name == id.config); q = q && sq; } @@ -191,11 +189,11 @@ build_query (const brep::vector<brep::build_config_id>* config_ids, // Build toolchain name/version. // - const string& tc (params.toolchain ()); + const string& th (params.toolchain ()); - if (tc != "*") + if (th != "*") { - size_t p (tc.find ('-')); + size_t p (th.find ('-')); if (p == string::npos) // Invalid format. throw invalid_argument (""); @@ -203,8 +201,8 @@ build_query (const brep::vector<brep::build_config_id>* config_ids, // the exact version revision, so an absent and zero revisions have the // same semantics and the zero revision is folded. // - string tn (tc, 0, p); - version tv (string (tc, p + 1)); // May throw invalid_argument. + string tn (th, 0, p); + version tv (string (th, p + 1)); // May throw invalid_argument. q = q && qb::id.toolchain_name == tn && @@ -213,20 +211,20 @@ build_query (const brep::vector<brep::build_config_id>* config_ids, true /* revision */); } - // Build configuration name. + // Build target. // - if (!params.configuration ().empty ()) - q = q && match<T> (qb::id.configuration, params.configuration ()); + if (!params.target ().empty ()) + q = q && match<T> (qb::id.target, params.target ()); - // Build machine name. + // Build target configuration name. // - if (!params.machine ().empty ()) - q = q && match<T> (qb::machine, params.machine ()); + if (!params.target_config ().empty ()) + q = q && match<T> (qb::id.target_config_name, params.target_config ()); - // Build target. + // Build package configuration name. // - if (!params.target ().empty ()) - q = q && match<T> (qb::id.target, params.target ()); + if (!params.package_config ().empty ()) + q = q && match<T> (qb::id.package_config_name, params.package_config ()); // Build result. // @@ -244,7 +242,7 @@ build_query (const brep::vector<brep::build_config_id>* config_ids, // May throw invalid_argument. // - result_status st (to_result_status (rs)); + result_status st (bbot::to_result_status (rs)); if (st != result_status::success) { @@ -364,11 +362,6 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Override the name parameter for the old URL (see options.cli for details). - // - if (params.name_legacy_specified ()) - params.name (params.name_legacy ()); - const char* title ("Builds"); xml::serializer s (rs.content (), title); @@ -430,16 +423,16 @@ handle (request& rq, response& rs) // the selected toolchain is still present in the database. Otherwise // fallback to the * wildcard selection. // - string ctc ("*"); + string cth ("*"); vector<pair<string, string>> toolchain_opts ({{"*", "*"}}); { for (const auto& t: toolchains) { - string tc (t.first + '-' + t.second.string ()); - toolchain_opts.emplace_back (tc, tc); + string th (t.first + '-' + t.second.string ()); + toolchain_opts.emplace_back (th, th); - if (tc == params.toolchain ()) - ctc = move (tc); + if (th == params.toolchain ()) + cth = move (th); } } @@ -455,34 +448,42 @@ handle (request& rq, response& rs) << TBODY << TR_INPUT ("name", "builds", params.name (), "*", true) << TR_INPUT ("version", "pv", params.version (), "*") - << TR_SELECT ("toolchain", "tc", ctc, toolchain_opts) + << TR_SELECT ("toolchain", "th", cth, toolchain_opts) + << TR_INPUT ("target", "tg", params.target (), "*") - << TR(CLASS="config") - << TH << "config" << ~TH + << TR(CLASS="tgt-config") + << TH << "tgt config" << ~TH << TD << *INPUT(TYPE="text", - NAME="cf", - VALUE=params.configuration (), + NAME="tc", + VALUE=params.target_config (), PLACEHOLDER="*", - LIST="configs") - << DATALIST(ID="configs") + LIST="target-configs") + << DATALIST(ID="target-configs") << *OPTION(VALUE="*"); - // Print unique config names from the config map. + // Print unique config names from the target config map. // set<const char*, butl::compare_c_string> conf_names; - for (const auto& c: *build_conf_map_) + for (const auto& c: *target_conf_map_) { - if (conf_names.insert (c.first.name.get ().c_str ()).second) - s << *OPTION(VALUE=c.first.name.get ()); + if (conf_names.insert (c.first.config.get ().c_str ()).second) + s << *OPTION(VALUE=c.first.config.get ()); } s << ~DATALIST << ~TD << ~TR - << TR_INPUT ("target", "tg", params.target (), "*") - << TR_INPUT ("machine", "mn", params.machine (), "*") + << TR(CLASS="pkg-config") + << TH << "pkg config" << ~TH + << TD + << *INPUT(TYPE="text", + NAME="pc", + VALUE=params.package_config (), + PLACEHOLDER="*") + << ~TD + << ~TR << TR_SELECT ("result", "rs", params.result (), build_results) << ~TBODY << ~TABLE @@ -504,16 +505,19 @@ handle (request& rq, response& rs) s << DIV_COUNTER (build_count, "Build", "Builds"); }; + const string& tgt (params.target ()); + const string& tgt_cfg (params.target_config ()); + const string& pkg_cfg (params.package_config ()); + // We will not display hidden configurations, unless the configuration is // specified explicitly. // - bool exclude_hidden (params.configuration ().empty () || - path_pattern (params.configuration ())); + bool exclude_hidden (tgt_cfg.empty () || path_pattern (tgt_cfg)); - vector<build_config_id> conf_ids; - conf_ids.reserve (build_conf_map_->size ()); + vector<build_target_config_id> conf_ids; + conf_ids.reserve (target_conf_map_->size ()); - for (const auto& c: *build_conf_map_) + for (const auto& c: *target_conf_map_) { if (!exclude_hidden || belongs (*c.second, "all")) conf_ids.push_back (c.first); @@ -600,17 +604,36 @@ handle (request& rq, response& rs) { shared_ptr<build>& b (pb.build); - auto i (build_conf_map_->find (build_config_id {b->configuration, - b->target})); - assert (i != build_conf_map_->end ()); + auto i (target_conf_map_->find ( + build_target_config_id {b->target, + b->target_config_name})); - // Match the configuration against the package build - // expressions/constraints. + assert (i != target_conf_map_->end ()); + + // Match the target configuration against the package build + // configuration expressions/constraints. // shared_ptr<build_package> p ( build_db_->load<build_package> (b->id.package)); - if (!exclude (p->builds, p->constraints, *i->second)) + const build_package_config* pc (find (b->package_config_name, + p->configs)); + + // The package configuration should be present since the + // configurations set cannot change if the package version doesn't + // change. If that's not the case, then the database has probably + // been manually amended. In this case let's just skip such a build + // as if it excluded and log the warning. + // + if (pc == nullptr) + { + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << p->id.name << '/' << p->version; + + continue; + } + + if (!exclude (*pc, p->builds, p->constraints, *i->second)) { if (skip != 0) --skip; @@ -646,7 +669,7 @@ handle (request& rq, response& rs) } } } - + // // Print the filter form after the build count is calculated. Note: // query_toolchains() must be called inside the build db transaction. // @@ -680,9 +703,9 @@ handle (request& rq, response& rs) << TR_VALUE ("toolchain", b.toolchain_name + '-' + b.toolchain_version.string ()) - << TR_VALUE ("config", b.configuration) << TR_VALUE ("target", b.target.string ()) - << TR_VALUE ("machine", b.machine) + << TR_VALUE ("tgt config", b.target_config_name) + << TR_VALUE ("pkg config", b.package_config_name) << TR_VALUE ("timestamp", ts); if (b.interactive) // Note: can only be present for the building state. @@ -704,47 +727,55 @@ handle (request& rq, response& rs) else // Print unbuilt package configurations. { // Parameters to use for package build configurations queries. Note that - // we cleanup the machine and the result filter arguments, as they are - // irrelevant for unbuilt configurations. + // we cleanup the result filter argument, as it is irrelevant for unbuilt + // configurations. // params::builds bld_params (params); - bld_params.machine ().clear (); bld_params.result () = "*"; - // Query toolchains, filter build configurations and toolchains, and - // create the set of configuration/toolchain combinations, that we will - // print for packages. Also calculate the number of unbuilt package - // configurations. + // Query toolchains, filter build target configurations and toolchains, + // and create the set of target configuration/toolchain combinations, that + // we will print for package configurations. Also calculate the number of + // unbuilt package configurations. // toolchains toolchains; - // Note that config_toolchains contains shallow references to the - // toolchain names and versions. + // Target configuration/toolchain combination. + // + // Note: all members are the shallow references. // - set<config_toolchain> config_toolchains; + struct target_config_toolchain + { + const butl::target_triplet& target; + const string& target_config; + const string& toolchain_name; + const bpkg::version& toolchain_version; + }; + + vector<target_config_toolchain> config_toolchains; { transaction t (build_db_->begin ()); toolchains = query_toolchains (); - string tc_name; - version tc_version; - const string& tc (params.toolchain ()); + string th_name; + version th_version; + const string& th (params.toolchain ()); - if (tc != "*") + if (th != "*") try { - size_t p (tc.find ('-')); + size_t p (th.find ('-')); if (p == string::npos) // Invalid format. throw invalid_argument (""); - tc_name.assign (tc, 0, p); + th_name.assign (th, 0, p); // May throw invalid_argument. // // Note that an absent and zero revisions have the same semantics, // so the zero revision is folded (see above for details). // - tc_version = version (string (tc, p + 1)); + th_version = version (string (th, p + 1)); } catch (const invalid_argument&) { @@ -754,63 +785,64 @@ handle (request& rq, response& rs) throw invalid_request (400, "invalid toolchain"); } - const string& pc (params.configuration ()); - const string& tg (params.target ()); - vector<const build_config*> configs; + vector<const build_target_config*> target_configs; - for (const auto& c: *build_conf_) + for (const auto& c: *target_conf_) { - if ((pc.empty () || path_match (c.name, pc)) && // Filter by name. + // Filter by name. + // + if ((tgt_cfg.empty () || path_match (c.name, tgt_cfg)) && // Filter by target. // - (tg.empty () || path_match (c.target.string (), tg)) && + (tgt.empty () || path_match (c.target.string (), tgt)) && (!exclude_hidden || belongs (c, "all"))) // Filter hidden. { - configs.push_back (&c); + target_configs.push_back (&c); for (const auto& t: toolchains) { // Filter by toolchain. // - if (tc == "*" || (t.first == tc_name && t.second == tc_version)) - config_toolchains.insert ( - config_toolchain {c.name, c.target, t.first, t.second}); + if (th == "*" || (t.first == th_name && t.second == th_version)) + config_toolchains.push_back ( + target_config_toolchain {c.target, c.name, t.first, t.second}); } } } - // Calculate the number of unbuilt package configurations as a - // difference between the maximum possible number of unbuilt - // configurations and the number of existing package builds. - // - // Note that we also need to deduct the package-excluded configurations - // count from the maximum possible number of unbuilt configurations. The - // only way to achieve this is to traverse through the packages and - // match their build expressions/constraints against our configurations. - // - // Also note that some existing builds can now be excluded by packages - // due to the build configuration target or class set change. We should - // deduct such builds count from the number of existing package builds. - // - size_t nmax (config_toolchains.size () * - build_db_->query_value<buildable_package_count> ( - package_query<buildable_package_count> ( - params, tn, false /* archived */))); + if (!config_toolchains.empty ()) + { + // Calculate the number of unbuilt package configurations as a + // difference between the possible number of unbuilt configurations + // and the number of existing package builds. + // + // Note that some existing builds can now be excluded by package + // configurations due to the build target configuration class set + // change. We should deduct such builds count from the number of + // existing package configurations builds. + // + // The only way to calculate both numbers is to traverse through the + // package configurations and match their build + // expressions/constraints against our target configurations. + // + size_t npos (0); - size_t ncur = build_db_->query_value<package_build_count> ( - build_query<package_build_count> ( - &conf_ids, bld_params, tn, false /* archived */)); + size_t ncur = build_db_->query_value<package_build_count> ( + build_query<package_build_count> ( + &conf_ids, bld_params, tn, false /* archived */)); - // From now we will be using specific package name and version for each - // build database query. - // - bld_params.name ().clear (); - bld_params.version ().clear (); + // From now we will be using specific values for the below filters for + // each build database query. Note that the toolchain is the only + // filter left in bld_params. + // + bld_params.name ().clear (); + bld_params.version ().clear (); + bld_params.target ().clear (); + bld_params.target_config ().clear (); + bld_params.package_config ().clear (); - if (!config_toolchains.empty ()) - { // Prepare the build count prepared query. // // For each package-excluded configuration we will query the number of @@ -820,15 +852,17 @@ handle (request& rq, response& rs) using prep_bld_query = prepared_query<package_build_count>; package_id id; - string config; target_triplet target; + string target_config_name; + string package_config_name; const auto& bid (bld_query::build::id); bld_query bq ( - equal<package_build_count> (bid.package, id) && - bid.configuration == bld_query::_ref (config) && - bid.target == bld_query::_ref (target) && + equal<package_build_count> (bid.package, id) && + bid.target == bld_query::_ref (target) && + bid.target_config_name == bld_query::_ref (target_config_name) && + bid.package_config_name == bld_query::_ref (package_config_name) && // Note that the query already constrains configurations via the // configuration name and target. @@ -846,15 +880,16 @@ handle (request& rq, response& rs) build_db_->prepare_query<package_build_count> ( "mod-builds-build-count-query", bq)); - size_t nt (tc == "*" ? toolchains.size () : 1); + // Number of possible builds per package configuration. + // + size_t nt (th == "*" ? toolchains.size () : 1); // The number of packages can potentially be large, and we may // implement some caching in the future. However, the caching will not // be easy as the cached values depend on the filter form parameters. // query<buildable_package> q ( - package_query<buildable_package> ( - params, tn, false /* archived */)); + package_query<buildable_package> (params, tn, false /* archived */)); for (auto& bp: build_db_->query<buildable_package> (q)) { @@ -862,22 +897,33 @@ handle (request& rq, response& rs) shared_ptr<build_package> p (build_db_->load<build_package> (id)); - for (const auto& c: configs) + for (const build_package_config& c: p->configs) { - if (exclude (p->builds, p->constraints, *c)) + // Filter by package config name. + // + if (pkg_cfg.empty () || path_match (c.name, pkg_cfg)) { - nmax -= nt; - - config = c->name; - target = c->target; - ncur -= bld_prep_query.execute_value (); + for (const auto& tc: target_configs) + { + if (exclude (c, p->builds, p->constraints, *tc)) + { + target = tc->target; + target_config_name = tc->name; + package_config_name = c.name; + ncur -= bld_prep_query.execute_value (); + } + else + npos += nt; + } } } } - } - assert (nmax >= ncur); - count = nmax - ncur; + assert (npos >= ncur); + count = npos - ncur; + } + else + count = 0; t.commit (); } @@ -893,8 +939,9 @@ handle (request& rq, response& rs) // 3: package tenant // 4: toolchain name // 5: toolchain version (descending) - // 6: configuration name - // 7: configuration target + // 6: target + // 7: target configuration name + // 8: package configuration name // // Prepare the build package prepared query. // @@ -946,15 +993,14 @@ handle (request& rq, response& rs) package_id id; - bld_query bq ( - equal<package_build> (bld_query::build::id.package, id) && + bld_query bq (equal<package_build> (bld_query::build::id.package, id) && - // Note that while the query already constrains the tenant via the build - // package id, we still need to pass the tenant not to erroneously - // filter out the private tenants. - // - build_query<package_build> ( - &conf_ids, bld_params, tn, false /* archived */)); + // Note that while the query already constrains the tenant + // via the build package id, we still need to pass the + // tenant not to erroneously filter out the private tenants. + // + build_query<package_build> ( + &conf_ids, bld_params, tn, false /* archived */)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); @@ -986,22 +1032,35 @@ handle (request& rq, response& rs) { id = move (p.id); + shared_ptr<build_package> bp (build_db_->load<build_package> (id)); + // Copy configuration/toolchain combinations for this package, // skipping excluded configurations. // set<config_toolchain> unbuilt_configs; - { - shared_ptr<build_package> p (build_db_->load<build_package> (id)); - for (const auto& ct: config_toolchains) + for (const build_package_config& pc: bp->configs) + { + // Filter by package config name. + // + if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) { - auto i (build_conf_map_->find (build_config_id {ct.configuration, - ct.target})); - - assert (i != build_conf_map_->end ()); - - if (!exclude (p->builds, p->constraints, *i->second)) - unbuilt_configs.insert (ct); + for (const target_config_toolchain& ct: config_toolchains) + { + auto i ( + target_conf_map_->find ( + build_target_config_id {ct.target, ct.target_config})); + + assert (i != target_conf_map_->end ()); + + if (!exclude (pc, bp->builds, bp->constraints, *i->second)) + unbuilt_configs.insert ( + config_toolchain {ct.target, + ct.target_config, + pc.name, + ct.toolchain_name, + ct.toolchain_version}); + } } } @@ -1012,8 +1071,9 @@ handle (request& rq, response& rs) { const build& b (*pb.build); - unbuilt_configs.erase (config_toolchain {b.id.configuration, - b.id.target, + unbuilt_configs.erase (config_toolchain {b.target, + b.target_config_name, + b.package_config_name, b.toolchain_name, b.toolchain_version}); } @@ -1035,8 +1095,9 @@ handle (request& rq, response& rs) << TR_VALUE ("toolchain", string (ct.toolchain_name) + '-' + ct.toolchain_version.string ()) - << TR_VALUE ("config", ct.configuration) - << TR_VALUE ("target", ct.target.string ()); + << TR_VALUE ("target", ct.target.string ()) + << TR_VALUE ("tgt config", ct.target_config) + << TR_VALUE ("pkg config", ct.package_config); // In the global view mode add the tenant builds link. Note that // the global view (and the link) makes sense only in the @@ -1084,10 +1145,10 @@ handle (request& rq, response& rs) }; add_filter ("pv", params.version ()); - add_filter ("tc", params.toolchain (), "*"); - add_filter ("cf", params.configuration ()); - add_filter ("mn", params.machine ()); - add_filter ("tg", params.target ()); + add_filter ("th", params.toolchain (), "*"); + add_filter ("tg", tgt); + add_filter ("tc", tgt_cfg); + add_filter ("pc", pkg_cfg); add_filter ("rs", params.result (), "*"); s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u) diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 37eb3c6..b158228 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -528,15 +528,14 @@ handle (request& rq, response& rs) if (builds) { - using bbot::build_config; - s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_config& cfg, - string* reason = nullptr) + auto exclude = [&pkg, this] (const build_package_config& pc, + const build_target_config& tc, + string* rs = nullptr) { - return this->exclude (pkg->builds, pkg->build_constraints, cfg, reason); + return this->exclude (pc, pkg->builds, pkg->build_constraints, tc, rs); }; timestamp now (system_clock::now ()); @@ -569,34 +568,46 @@ handle (request& rq, response& rs) } // Compose the configuration filtering sub-query and collect unbuilt - // configurations, skipping those that are hidden or excluded by the - // package. + // target configurations, skipping those that are hidden or excluded by + // the package configurations. // using query = query<build>; query sq (false); set<config_toolchain> unbuilt_configs; - for (const auto& c: *build_conf_map_) + for (const build_package_config& pc: pkg->build_configs) { - const build_config& cfg (*c.second); - - if (belongs (cfg, "all") && !exclude (cfg)) + for (const auto& bc: *target_conf_map_) { - const build_config_id& id (c.first); + const build_target_config& tc (*bc.second); - sq = sq || (query::id.configuration == id.name && - query::id.target == id.target); + if (belongs (tc, "all") && !exclude (pc, tc)) + { + const build_target_config_id& id (bc.first); - // Note: we will erase built configurations from the unbuilt - // configurations set later (see below). - // - for (const auto& t: toolchains) - unbuilt_configs.insert ( - config_toolchain {cfg.name, cfg.target, t.first, t.second}); + sq = sq || (query::id.target == id.target && + query::id.target_config_name == id.config && + query::id.package_config_name == pc.name); + + // Note: we will erase built configurations from the unbuilt + // configurations set later (see below). + // + for (const auto& t: toolchains) + unbuilt_configs.insert (config_toolchain {tc.target, + tc.name, + pc.name, + t.first, + t.second}); + } } } + // Let's not print the package configuration row if the default + // configuration is the only one. + // + bool ppc (pkg->build_configs.size () != 1); // Note: can't be empty. + // Print the package built configurations in the time-descending order. // for (auto& b: build_db_->query<build> ( @@ -617,9 +628,13 @@ handle (request& rq, response& rs) << TR_VALUE ("toolchain", b.toolchain_name + '-' + b.toolchain_version.string ()) - << TR_VALUE ("config", - b.configuration + " / " + b.target.string ()) - << TR_VALUE ("timestamp", ts); + << TR_VALUE ("target", b.target.string ()) + << TR_VALUE ("tgt config", b.target_config_name); + + if (ppc) + s << TR_VALUE ("pkg config", b.package_config_name); + + s << TR_VALUE ("timestamp", ts); if (b.interactive) // Note: can only be present for the building state. s << TR_VALUE ("login", *b.interactive); @@ -631,8 +646,9 @@ handle (request& rq, response& rs) // While at it, erase the built configuration from the unbuilt // configurations set. // - unbuilt_configs.erase (config_toolchain {b.configuration, - b.target, + unbuilt_configs.erase (config_toolchain {b.target, + b.target_config_name, + b.package_config_name, b.toolchain_name, b.toolchain_version}); } @@ -642,7 +658,9 @@ handle (request& rq, response& rs) // // 1: toolchain name // 2: toolchain version (descending) - // 3: configuration name + // 3: target + // 4: target configuration name + // 5: package configuration name // for (const auto& ct: unbuilt_configs) { @@ -651,9 +669,13 @@ handle (request& rq, response& rs) << TR_VALUE ("toolchain", ct.toolchain_name + '-' + ct.toolchain_version.string ()) - << TR_VALUE ("config", - ct.configuration + " / " + ct.target.string ()) - << TR_VALUE ("result", "unbuilt") + << TR_VALUE ("target", ct.target.string ()) + << TR_VALUE ("tgt config", ct.target_config); + + if (ppc) + s << TR_VALUE ("pkg config", ct.package_config); + + s << TR_VALUE ("result", "unbuilt") << ~TBODY << ~TABLE; } @@ -664,20 +686,28 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const auto& c: *build_conf_) + for (const build_package_config& pc: pkg->build_configs) { - string reason; - if (belongs (c, "default") && exclude (c, &reason)) + for (const auto& tc: *target_conf_) { - s << TABLE(CLASS="proplist build") - << TBODY - << TR_VALUE ("config", c.name + " / " + c.target.string ()) - << TR_VALUE ("result", - !reason.empty () - ? "excluded (" + reason + ')' - : "excluded") - << ~TBODY - << ~TABLE; + string reason; + if (belongs (tc, "default") && exclude (pc, tc, &reason)) + { + s << TABLE(CLASS="proplist build") + << TBODY + << TR_VALUE ("target", tc.target.string ()) + << TR_VALUE ("tgt config", tc.name); + + if (ppc) + s << TR_VALUE ("pkg config", pc.name); + + s << TR_VALUE ("result", + !reason.empty () + ? "excluded (" + reason + ')' + : "excluded") + << ~TBODY + << ~TABLE; + } } } } diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx index 1cbb5cb..082903b 100644 --- a/mod/mod-repository-details.cxx +++ b/mod/mod-repository-details.cxx @@ -3,8 +3,6 @@ #include <mod/mod-repository-details.hxx> -#include <algorithm> // max() - #include <libstudxml/serializer.hxx> #include <odb/database.hxx> diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 02d6c93..f00e80e 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -8,7 +8,6 @@ #include <cmark-gfm-core-extensions.h> #include <sstream> -#include <algorithm> // find() #include <web/server/module.hxx> diff --git a/mod/module.cli b/mod/module.cli index 5814f37..7c6e0b4 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -765,14 +765,18 @@ namespace brep // string version | pv; - // Package build configuration. - // - string configuration | cf; - // Package build target. // string target | tg; + // Target build configuration. + // + string target_config | tc; + + // Package build configuration. + // + string package_config | pc; + // Toolchain name. // string toolchain_name | tn; @@ -805,13 +809,10 @@ namespace brep // // https://cppget.org/?builds=bbot // - // To support the already distributed URLs the name_legacy (pn) parameter - // overrides the name (builds) parameter, if present. Note that the - // builds parameter is renamed to '_' by the root handler (see the - // request_proxy class for details). + // Note that the builds parameter is renamed to '_' by the root handler + // (see the request_proxy class for details). // string name | _; - string name_legacy | pn; // Package version. If empty or *, then no version constraint is applied. // Otherwise the build package version must match the value exactly. @@ -822,22 +823,22 @@ namespace brep // toolchain constraint is applied. Otherwise the build toolchain name // and version must match the value exactly. // - string toolchain | tc = "*"; + string toolchain | th = "*"; - // Package build configuration name wildcard. An empty value is treated - // the same way as *. + // Package build target wildcard. An empty value is treated the same way + // as *. // - string configuration | cf; + string target | tg; - // Package build machine name wildcard. An empty value is treated the - // same way as *. + // Package build target configuration name wildcard. An empty value is + // treated the same way as *. // - string machine | mn; + string target_config | tc; - // Package build target wildcard. An empty value is treated the same way - // as *. + // Package build package configuration name wildcard. An empty value is + // treated the same way as *. // - string target | tg; + string package_config | pc; // Package build result. If *, then no build result constraint is // applied. Otherwise the value is supposed to be the one of the diff --git a/mod/page.cxx b/mod/page.cxx index a73e336..e2a8b84 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -7,10 +7,10 @@ #include <cmark-gfm-extension_api.h> #include <set> -#include <ios> // hex, uppercase, right +#include <ios> // hex, uppercase, right #include <sstream> -#include <iomanip> // setw(), setfill() -#include <algorithm> // min(), find() +#include <iomanip> // setw(), setfill() +#include <iterator> // back_inserter() #include <libstudxml/serializer.hxx> @@ -36,6 +36,20 @@ using namespace web::xhtml; // namespace brep { + static inline string + label_to_class (const string& label) + { + if (label.find (' ') == string::npos) + return label; + + string r; + transform (label.begin (), label.end (), + back_inserter (r), + [] (char c) {return c != ' ' ? c : '-';}); + + return r; + } + // CSS_LINKS // static const dir_path css_path ("@"); @@ -134,7 +148,8 @@ namespace brep void TR_VALUE:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << SPAN(CLASS="value") << value_ << ~SPAN << ~TD << ~TR; @@ -145,7 +160,8 @@ namespace brep void TR_INPUT:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << INPUT(TYPE="text", NAME=name_); @@ -169,7 +185,8 @@ namespace brep void TR_SELECT:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << SELECT(NAME=name_); @@ -604,7 +621,8 @@ namespace brep void TR_URL:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << SPAN(CLASS="value"); @@ -634,7 +652,8 @@ namespace brep void TR_EMAIL:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << SPAN(CLASS="value") @@ -698,7 +717,8 @@ namespace brep void TR_LINK:: operator() (serializer& s) const { - s << TR(CLASS=label_) + string c (label_to_class (label_)); + s << TR(CLASS=c) << TH << label_ << ~TH << TD << SPAN(CLASS="value") << A(HREF=url_) << text_ << ~A << ~SPAN diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 0d5545e..7dab2a0 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -5,7 +5,6 @@ #include <set> #include <chrono> #include <iostream> -#include <algorithm> // find_if() #include <odb/database.hxx> #include <odb/transaction.hxx> @@ -15,8 +14,6 @@ #include <libbutl/pager.hxx> -#include <libbbot/build-config.hxx> - #include <libbrep/build.hxx> #include <libbrep/common.hxx> #include <libbrep/build-odb.hxx> @@ -24,14 +21,13 @@ #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> -#include <mod/build-config.hxx> +#include <mod/build-target-config.hxx> #include <monitor/module-options.hxx> #include <monitor/monitor-options.hxx> using namespace std; using namespace butl; -using namespace bbot; using namespace odb::core; namespace brep @@ -43,18 +39,19 @@ namespace brep // We will collect and report build delays as separate steps not to hold // database locks while printing to stderr. Also we need to order delays // properly, so while printing reports we could group delays by toolchain - // and configuration. + // and target configuration. // // To achieve that, we will iterate through all possible package builds // creating the list of delays with the following sort priority: // // 1: toolchain name // 2: toolchain version (descending) - // 3: configuration name - // 4: configuration target + // 3: target configuration name + // 4: target // 5: tenant // 6: package name // 7: package version (descending) + // 8: package configuration name // struct compare_delay { @@ -68,7 +65,7 @@ namespace brep if (int r = x->toolchain_version.compare (y->toolchain_version)) return r > 0; - if (int r = x->configuration.compare (y->configuration)) + if (int r = x->target_config_name.compare (y->target_config_name)) return r < 0; if (int r = x->target.compare (y->target)) @@ -80,7 +77,10 @@ namespace brep if (int r = x->package_name.compare (y->package_name)) return r < 0; - return x->package_version.compare (y->package_version) > 0; + if (int r = x->package_version.compare (y->package_version)) + return r > 0; + + return x->package_config_name.compare (y->package_config_name) < 0; } }; @@ -90,21 +90,19 @@ namespace brep { public: // Note that in the brief mode we also need to print the total number of - // delays (reported or not) per configuration. Thus, we add all delays to - // the report object, marking them if we need to report them or not. + // delays (reported or not) per target configuration. Thus, we add all + // delays to the report object, marking them if we need to report them or + // not. // void add_delay (shared_ptr<build_delay>, bool report); bool - empty () const - { - return reported_delay_count_ == 0; - } + empty () const {return reported_delay_count_ == 0;} // In the brief mode (if full is false) print the number of reported/total - // (if total is true) delayed package builds per configuration rather than - // the packages themselves. + // (if total is true) delayed package configuration builds per target + // configuration rather than the package configurations themselves. // void print (const char* header, bool total, bool full) const; @@ -138,23 +136,23 @@ namespace brep cerr << "):" << endl; - // Group the printed delays by toolchain and configuration. + // Group the printed delays by toolchain and target configuration. // - const string* toolchain_name (nullptr); - const version* toolchain_version (nullptr); - const string* configuration (nullptr); - const target_triplet* target (nullptr); + const string* toolchain_name (nullptr); + const version* toolchain_version (nullptr); + const string* target_config_name (nullptr); + const target_triplet* target (nullptr); size_t config_reported_delay_count (0); size_t config_total_delay_count (0); - auto brief_config = [&configuration, + auto brief_config = [&target_config_name, &target, &config_reported_delay_count, &config_total_delay_count, total] () { - if (configuration != nullptr) + if (target_config_name != nullptr) { assert (target != nullptr); @@ -162,7 +160,7 @@ namespace brep // if (config_reported_delay_count != 0) { - cerr << " " << *configuration << '/' << *target << " (" + cerr << " " << *target_config_name << '/' << *target << " (" << config_reported_delay_count; if (total) @@ -204,44 +202,45 @@ namespace brep cerr << endl; - toolchain_name = &d->toolchain_name; - toolchain_version = &d->toolchain_version; - configuration = nullptr; - target = nullptr; + toolchain_name = &d->toolchain_name; + toolchain_version = &d->toolchain_version; + target_config_name = nullptr; + target = nullptr; } // Print the configuration, if changed. // - if (configuration == nullptr || - d->configuration != *configuration || + if (target_config_name == nullptr || + d->target_config_name != *target_config_name || d->target != *target) { if (full) { - if (configuration != nullptr) + if (target_config_name != nullptr) cerr << endl; - cerr << " " << d->configuration << '/' << d->target << endl; + cerr << " " << d->target_config_name << '/' << d->target << endl; } else brief_config (); - configuration = &d->configuration; - target = &d->target; + target_config_name = &d->target_config_name; + target = &d->target; } - // Print the delayed build package in the full report mode and count - // configuration builds otherwise. + // Print the delayed build package configuration in the full report mode + // and count configuration builds otherwise. // if (full) { // We can potentially extend this information with the archived flag // or the delay duration. // - cerr << " " << d->package_name << "/" << d->package_version; + cerr << " " << d->package_name << '/' << d->package_version + << ' ' << d->package_config_name; if (!d->tenant.empty ()) - cerr << " " << d->tenant; + cerr << ' ' << d->tenant; cerr << endl; } @@ -450,11 +449,11 @@ namespace brep return 0; } - build_configs configs; + build_target_configs configs; try { - configs = parse_buildtab (mod_ops.build_config ()); + configs = bbot::parse_buildtab (mod_ops.build_config ()); } catch (const tab_parsing& e) { @@ -505,11 +504,12 @@ namespace brep // if (ops.clean ()) { - using config_map = map<build_config_id, const build_config*>; + using config_map = map<build_target_config_id, + const build_target_config*>; config_map conf_map; - for (const build_config& c: configs) - conf_map[build_config_id {c.name, c.target}] = &c; + for (const build_target_config& c: configs) + conf_map[build_target_config_id {c.target, c.name}] = &c; // Prepare the build delay prepared query. // @@ -528,15 +528,16 @@ namespace brep size_t offset (0); query q ("ORDER BY" + - query::id.package.tenant + "," + - query::id.package.name + + query::id.package.tenant + "," + + query::id.package.name + order_by_version (query::id.package.version, false /* first */) + "," + - query::id.configuration + "," + - query::id.target + "," + - query::id.toolchain_name + + query::id.target + "," + + query::id.target_config_name + "," + + query::id.package_config_name + "," + + query::id.toolchain_name + order_by_version (query::id.toolchain_version, - false /* first */) + + false /* first */) + "OFFSET" + query::_ref (offset) + "LIMIT 100"); connection_ptr conn (db.connection ()); @@ -579,8 +580,9 @@ namespace brep // // Check that the build configuration is still present. // - (ci = conf_map.find (build_config_id {d.configuration, - d.target})) == + (ci = conf_map.find ( + build_target_config_id {d.target, + d.target_config_name})) == conf_map.end ()); // Check that the package still present, is buildable and doesn't @@ -594,9 +596,15 @@ namespace brep p = db.find<build_package> (pid); } - cleanup = (p == nullptr || + const build_package_config* pc (p != nullptr + ? find (d.package_config_name, + p->configs) + : nullptr); + + cleanup = (pc == nullptr || !p->buildable || - exclude (p->builds, + exclude (*pc, + p->builds, p->constraints, *ci->second, configs.class_inheritance_map)); @@ -642,7 +650,7 @@ namespace brep conn->prepare_query<buildable_package> ("buildable-package-query", pq)); - // Prepare the package build prepared query. + // Prepare the package configuration build prepared query. // // This query will only be used for toolchains that have no version // specified on the command line to obtain the latest completed build @@ -658,12 +666,16 @@ namespace brep using prep_bquery = prepared_query<package_build>; build_id id; + string package_config_name; const auto& bid (bquery::build::id); - bquery bq ((equal<package_build> (bid.package, id.package) && - bid.configuration == bquery::_ref (id.configuration) && - bid.target == bquery::_ref (id.target) && + bquery bq ((equal<package_build> (bid.package, id.package) && + bid.target == bquery::_ref (id.target) && + bid.target_config_name == + bquery::_ref (id.target_config_name) && + bid.package_config_name == + bquery::_ref (package_config_name) && bid.toolchain_name == bquery::_ref (id.toolchain_name)) + "ORDER BY" + bquery::build::soft_timestamp + "DESC, " + @@ -806,175 +818,184 @@ namespace brep { shared_ptr<build_package> p (db.load<build_package> (bp.id)); - for (const build_config& c: configs) + for (const build_package_config& pc: p->configs) { - if (exclude (p->builds, - p->constraints, - c, - configs.class_inheritance_map)) - continue; - - for (const pair<string, version>& t: toolchains) + for (const build_target_config& tc: configs) { - id = build_id (p->id, c.name, c.target, t.first, t.second); + if (exclude (pc, + p->builds, + p->constraints, + tc, + configs.class_inheritance_map)) + continue; + + for (const pair<string, version>& t: toolchains) + { + id = build_id (p->id, + tc.target, tc.name, + pc.name, + t.first, t.second); + + // If the toolchain version is unspecified then search for + // the latest build across all toolchain versions and search + // for a specific build otherwise. + // + shared_ptr<build> b; + + if (id.toolchain_version.empty ()) + { + package_config_name = pc.name; - // If the toolchain version is unspecified then search for the - // latest build across all toolchain versions and search for a - // specific build otherwise. - // - shared_ptr<build> b; + auto pbs (pbq.execute ()); - if (id.toolchain_version.empty ()) - { - auto pbs (pbq.execute ()); + if (!pbs.empty ()) + b = move (pbs.begin ()->build); + } + else + b = db.find<build> (id); - if (!pbs.empty ()) - b = move (pbs.begin ()->build); - } - else - b = db.find<build> (id); - - // Note that we consider a build as delayed if it is not - // completed in the expected timeframe. So even if the build - // task have been issued recently we may still consider the - // build as delayed. - // - timestamp bht (b != nullptr - ? b->hard_timestamp - : timestamp_nonexistent); - - timestamp bst (b != nullptr - ? b->soft_timestamp - : timestamp_nonexistent); - - // Create the delay object to record a timestamp when the - // package build could have potentially been started, unless - // it already exists. - // - shared_ptr<build_delay> d (db.find<build_delay> (id)); - - if (d == nullptr) - { - // If the archived package has no build nor build delay - // for this configuration, then we assume that the - // configuration was added after the package tenant has - // been archived and so the package could have never been - // built for this configuration. Thus, we don't consider - // this build as delayed and so skip it. + // Note that we consider a build as delayed if it is not + // completed in the expected timeframe. So even if the build + // task have been issued recently we may still consider the + // build as delayed. // - if (bp.archived && b == nullptr) - continue; - - // Use the build hard, soft, or status change timestamp (see - // the timestamps description for their ordering - // information) as the build delay tracking starting point - // and fallback to the current time if there is no build - // yet. + timestamp bht (b != nullptr + ? b->hard_timestamp + : timestamp_nonexistent); + + timestamp bst (b != nullptr + ? b->soft_timestamp + : timestamp_nonexistent); + + // Create the delay object to record a timestamp when the + // package configuration build could have potentially been + // started, unless it already exists. // - timestamp pts ( - b == nullptr ? now : - bht != timestamp_nonexistent ? bht : - bst != timestamp_nonexistent ? bst : - b->timestamp); - - d = make_shared<build_delay> (move (id.package.tenant), - move (id.package.name), - p->version, - move (id.configuration), - move (id.target), - move (id.toolchain_name), - t.second, - pts); - db.persist (d); - } + shared_ptr<build_delay> d (db.find<build_delay> (id)); - // Handle package builds differently based on their tenant's - // archive status. - // - // If the package is not archived then consider it as delayed - // if it is not (re-)built by the expiration time. Otherwise, - // consider it as delayed if it is unbuilt. - // - // We also don't need to report an unbuilt archived package - // twice, as both soft and hard build delays. - // - bool hard_delayed; - bool soft_delayed; - - if (!bp.archived) - { - auto delayed = [&d] (timestamp bt, timestamp be) + if (d == nullptr) { - timestamp t (bt != timestamp_nonexistent - ? bt - : d->package_timestamp); - return t <= be; - }; - - hard_delayed = delayed (bht, hard_rebuild_expiration); - soft_delayed = delayed (bst, soft_rebuild_expiration); - } - else - { - hard_delayed = (bst == timestamp_nonexistent); - soft_delayed = false; - } - - // Add hard/soft delays to the respective reports and collect - // the delay for update, if it is reported. - // - // Note that we update the delay objects persistent state - // later, after we successfully print the reports. - // - bool reported (false); + // If the archived package has no build nor build delay + // for this configuration, then we assume that the + // configuration was added after the package tenant has + // been archived and so the package could have never been + // built for this configuration. Thus, we don't consider + // this build as delayed and so skip it. + // + if (bp.archived && b == nullptr) + continue; + + // Use the build hard, soft, or status change timestamp + // (see the timestamps description for their ordering + // information) as the build delay tracking starting point + // and fallback to the current time if there is no build + // yet. + // + timestamp pts (b == nullptr ? now : + bht != timestamp_nonexistent ? bht : + bst != timestamp_nonexistent ? bst : + b->timestamp); + + d = make_shared<build_delay> (move (id.package.tenant), + move (id.package.name), + p->version, + move (id.target), + move (id.target_config_name), + move (id.package_config_name), + move (id.toolchain_name), + t.second, + pts); + db.persist (d); + } - if (hard_delayed) - { - // If the report timeout is zero then report the delay - // unconditionally. Otherwise, report the active package - // build delay if the report timeout is expired and the - // archived package build delay if it was never reported. - // Note that fixing the building infrastructure won't help - // building an archived package, so reporting its build - // delays repeatedly is meaningless. + // Handle package builds differently based on their tenant's + // archive status. // - bool report ( - ops.report_timeout () == 0 || - (!bp.archived - ? d->report_hard_timestamp <= report_expiration - : d->report_hard_timestamp == timestamp_nonexistent)); + // If the package is not archived then consider it as + // delayed if it is not (re-)built by the expiration + // time. Otherwise, consider it as delayed if it is unbuilt. + // + // We also don't need to report an unbuilt archived package + // twice, as both soft and hard build delays. + // + bool hard_delayed; + bool soft_delayed; - if (report) + if (!bp.archived) { - d->report_hard_timestamp = now; - reported = true; + auto delayed = [&d] (timestamp bt, timestamp be) + { + timestamp t (bt != timestamp_nonexistent + ? bt + : d->package_timestamp); + return t <= be; + }; + + hard_delayed = delayed (bht, hard_rebuild_expiration); + soft_delayed = delayed (bst, soft_rebuild_expiration); + } + else + { + hard_delayed = (bst == timestamp_nonexistent); + soft_delayed = false; } - hard_delays_report.add_delay (d, report); - } + // Add hard/soft delays to the respective reports and + // collect the delay for update, if it is reported. + // + // Note that we update the delay objects persistent state + // later, after we successfully print the reports. + // + bool reported (false); - if (soft_delayed) - { - bool report (ops.report_timeout () == 0 || - d->report_soft_timestamp <= report_expiration); + if (hard_delayed) + { + // If the report timeout is zero then report the delay + // unconditionally. Otherwise, report the active package + // build delay if the report timeout is expired and the + // archived package build delay if it was never reported. + // Note that fixing the building infrastructure won't help + // building an archived package, so reporting its build + // delays repeatedly is meaningless. + // + bool report ( + ops.report_timeout () == 0 || + (!bp.archived + ? d->report_hard_timestamp <= report_expiration + : d->report_hard_timestamp == timestamp_nonexistent)); + + if (report) + { + d->report_hard_timestamp = now; + reported = true; + } + + hard_delays_report.add_delay (d, report); + } - if (report) + if (soft_delayed) { - d->report_soft_timestamp = now; - reported = true; + bool report (ops.report_timeout () == 0 || + d->report_soft_timestamp <= report_expiration); + + if (report) + { + d->report_soft_timestamp = now; + reported = true; + } + + soft_delays_report.add_delay (d, report); } - soft_delays_report.add_delay (d, report); + // If we don't consider the report timestamps for reporting + // delays, it seems natural not to update these timestamps + // either. Note that reporting all delays and still updating + // the report timestamps can be achieved by specifying the + // zero report timeout. + // + if (reported && ops.report_timeout_specified ()) + update_delays.insert (move (d)); } - - // If we don't consider the report timestamps for reporting - // delays, it seems natural not to update these timestamps - // either. Note that reporting all delays and still updating - // the report timestamps can be achieved by specifying the - // zero report timeout. - // - if (reported && ops.report_timeout_specified ()) - update_delays.insert (move (d)); } } } diff --git a/tests/load/1/math/libexp-+2-1.2+1.tar.gz b/tests/load/1/math/libexp-+2-1.2+1.tar.gz Binary files differindex 5beeb84..b223d9f 100644 --- a/tests/load/1/math/libexp-+2-1.2+1.tar.gz +++ b/tests/load/1/math/libexp-+2-1.2+1.tar.gz diff --git a/tests/load/1/math/libfoo-+0-X.Y.tar.gz b/tests/load/1/math/libfoo-+0-X.Y.tar.gz Binary files differindex 6867d4f..95364bb 100644 --- a/tests/load/1/math/libfoo-+0-X.Y.tar.gz +++ b/tests/load/1/math/libfoo-+0-X.Y.tar.gz diff --git a/tests/load/1/math/libfoo-1.0.tar.gz b/tests/load/1/math/libfoo-1.0.tar.gz Binary files differindex 2d445ec..3f23ab9 100644 --- a/tests/load/1/math/libfoo-1.0.tar.gz +++ b/tests/load/1/math/libfoo-1.0.tar.gz diff --git a/tests/load/1/math/libfoo-1.2.4+1.tar.gz b/tests/load/1/math/libfoo-1.2.4+1.tar.gz Binary files differindex a52548c..2dde730 100644 --- a/tests/load/1/math/libfoo-1.2.4+1.tar.gz +++ b/tests/load/1/math/libfoo-1.2.4+1.tar.gz diff --git a/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz b/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz Binary files differindex 391eb6f..f1c9ba0 100644 --- a/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz +++ b/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz diff --git a/tests/load/1/math/libfoo-examples-1.2.4.tar.gz b/tests/load/1/math/libfoo-examples-1.2.4.tar.gz Binary files differindex eac5190..00164e6 100644 --- a/tests/load/1/math/libfoo-examples-1.2.4.tar.gz +++ b/tests/load/1/math/libfoo-examples-1.2.4.tar.gz diff --git a/tests/load/1/math/libfoo-tests-1.2.4.tar.gz b/tests/load/1/math/libfoo-tests-1.2.4.tar.gz Binary files differindex 223e24d..2efccd0 100644 --- a/tests/load/1/math/libfoo-tests-1.2.4.tar.gz +++ b/tests/load/1/math/libfoo-tests-1.2.4.tar.gz diff --git a/tests/load/1/math/libpq-0.tar.gz b/tests/load/1/math/libpq-0.tar.gz Binary files differindex b880520..d4beb18 100644 --- a/tests/load/1/math/libpq-0.tar.gz +++ b/tests/load/1/math/libpq-0.tar.gz diff --git a/tests/load/1/math/libstudxml-1.0.0+1.tar.gz b/tests/load/1/math/libstudxml-1.0.0+1.tar.gz Binary files differindex 41c9637..dcf0ee5 100644 --- a/tests/load/1/math/libstudxml-1.0.0+1.tar.gz +++ b/tests/load/1/math/libstudxml-1.0.0+1.tar.gz diff --git a/tests/load/1/math/packages.manifest b/tests/load/1/math/packages.manifest index 882aff6..8e66ddf 100644 --- a/tests/load/1/math/packages.manifest +++ b/tests/load/1/math/packages.manifest @@ -19,15 +19,23 @@ builds: default legacy build-include: windows**d/x86_64** build-include: windows-vc_13** build-exclude: **; Only supported on Windows. +bootstrap-build:\ +project = libexp + +\ location: libexp-+2-1.2+1.tar.gz -sha256sum: 317c8c6f45d9dfdfdef3a823411920cecd51729c7c4f58f9a0b0bbd681c07bd6 +sha256sum: d90cfe583890cd0c05cdfc204e69dd3b986c2da49851f7a87fa0ca870788ff79 : name: libfoo version: +0-X.Y summary: The Foo Library license: MIT +bootstrap-build:\ +project = libfoo + +\ location: libfoo-+0-X.Y.tar.gz -sha256sum: c994fd49f051ab7fb25f3a4e68ca878e484c5d3c2cb132b37d41224b0621b618 +sha256sum: c25e5cae2f72664a3961c3ef88a82e67150c4bcc2a5e1fb4d250e621c5574187 : name: libfoo version: 1.0 @@ -37,8 +45,12 @@ build-email: foo-builds@example.com builds: default legacy; Stable configurations only. builds: -32; 64-bit targets only builds: &msvc_13_up; Not too old MSVC. +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.0.tar.gz -sha256sum: e89c6d746f8b1ea3ec58d294946d2f683d133438d2ac8c88549ba24c19627e76 +sha256sum: 7382152bac5b4ce10215a5ecd6c94c490d0efc007031d3b03f407d068b74e624 : name: libfoo version: 1.2.4+1 @@ -88,32 +100,69 @@ requires: host tests: * libfoo-tests == 1.2.4 examples: libfoo-examples benchmarks: libfoo-benchmarks > 0.0.1 +builds: all +network-build-config: config.libfoo.network=true; Enable networking API. +network-build-include: windows-vc_14d/x86_64-microsoft-win32-msvc14.0 +network-build-exclude: ** +cache-build-config:\ +config.libfoo.cache=true +config.libfoo.buffer=4096 +; +Enable caching. +\ +cache-builds: default +cache-builds: -linux +cache-build-include: windows-vc_14d/x86_64-microsoft-win32-msvc14.0 +cache-build-exclude: ** +bootstrap-build:\ +project = libfoo + +\ +root-build:\ +config [bool] config.libfoo.network ?= false + +config [bool] config.libfoo.cache ?= false +config [uint64] config.libfoo.buffer ?= 1024 + +\ location: libfoo-1.2.4+1.tar.gz -sha256sum: d23a7ff116ab7264c3d423af97e4830bdaa8c9101cd95b210b19a97bb8512b74 +sha256sum: fe07978d72ab65c2ad72b0325aa56944cf093248d39edcb472a2fe5835defa3d : name: libfoo-benchmarks version: 1.2.4 summary: The Foo Math Library benchmarks license: MIT builds: 64; Fails building for 32 bits. +bootstrap-build:\ +project = libfoo-benchmarks + +\ location: libfoo-benchmarks-1.2.4.tar.gz -sha256sum: ba664343db5b9bd574450175834b0dd39d038dcff7387477b6eff0d5783a8ac4 +sha256sum: 8392db99b1ea0c78fe2c73d8c0ae35f8a31d798c8ed26ebf09b4bf557b4e3ce0 : name: libfoo-examples version: 1.2.4 summary: The Foo Math Library examples license: MIT builds: 64; Fails building for 32 bits. +bootstrap-build:\ +project = libfoo-examples + +\ location: libfoo-examples-1.2.4.tar.gz -sha256sum: 1343d1826c3ae5446ad965bc9aa7b1586e4238c7736c344e63a4a6bae3d57a88 +sha256sum: de1bf595994a63361262727594de94edbd77fff8234066da74672e44eb4349f2 : name: libfoo-tests version: 1.2.4 summary: The Foo Math Library tests license: MIT builds: 64; Fails building for 32 bits. +bootstrap-build:\ +project = libfoo-tests + +\ location: libfoo-tests-1.2.4.tar.gz -sha256sum: c5c0520b4e612fa2f8948c42824f3e199926c2395bf2c2f898e83f9eb19261a4 +sha256sum: bff1e47ef4f9658072dd7f412e97f80179d4257323568c3fa77ce6adbf89ebd1 : name: libpq version: 0 @@ -148,8 +197,12 @@ package-url: https://git.build2.org/cgit/packaging/postgresql/ email: pgsql-general@postgresql.org; Mailing list. package-email: packaging@build2.org; Mailing list. requires: build2 >= 0.4.0 +bootstrap-build:\ +project = libpq + +\ location: libpq-0.tar.gz -sha256sum: 367e72c8cc4aca1cdb99aacfefb0117ca333031775ff7435b0fa1f4988e33a31 +sha256sum: 2aee2bb1d58d51c657903bbab6253c5d4566b6f3f299ba118da24c7756caebfd : name: libstudxml version: 1.0.0+1 @@ -164,5 +217,9 @@ build-warning-email: studxml-warnings@example.com build-error-email: studxml-errors@example.com depends: libexpat >= 2.0.0 depends: libgenx +bootstrap-build:\ +project = libstudxml + +\ location: libstudxml-1.0.0+1.tar.gz -sha256sum: 1833906dd93ccc0cda832d6a1b3ef9ed7877bb9958b46d9b2666033d4a7919c9 +sha256sum: aa52d5b49ee1bad825cd6bca554f72636e8451f93c74f9a443bafce3c2bf82c0 diff --git a/tests/load/1/stable/libfoo-1.0.tar.gz b/tests/load/1/stable/libfoo-1.0.tar.gz Binary files differindex 2d445ec..3f23ab9 100644 --- a/tests/load/1/stable/libfoo-1.0.tar.gz +++ b/tests/load/1/stable/libfoo-1.0.tar.gz diff --git a/tests/load/1/stable/libfoo-1.2.2-alpha.1.tar.gz b/tests/load/1/stable/libfoo-1.2.2-alpha.1.tar.gz Binary files differindex aa5665e..1dfff70 100644 --- a/tests/load/1/stable/libfoo-1.2.2-alpha.1.tar.gz +++ b/tests/load/1/stable/libfoo-1.2.2-alpha.1.tar.gz diff --git a/tests/load/1/stable/libfoo-1.2.2.tar.gz b/tests/load/1/stable/libfoo-1.2.2.tar.gz Binary files differindex 94aca23..22eb89b 100644 --- a/tests/load/1/stable/libfoo-1.2.2.tar.gz +++ b/tests/load/1/stable/libfoo-1.2.2.tar.gz diff --git a/tests/load/1/stable/libfoo-1.2.3+4.tar.gz b/tests/load/1/stable/libfoo-1.2.3+4.tar.gz Binary files differindex 254f355..76439b0 100644 --- a/tests/load/1/stable/libfoo-1.2.3+4.tar.gz +++ b/tests/load/1/stable/libfoo-1.2.3+4.tar.gz diff --git a/tests/load/1/stable/libfoo-1.2.4.tar.gz b/tests/load/1/stable/libfoo-1.2.4.tar.gz Binary files differindex dc64431..da70cd3 100644 --- a/tests/load/1/stable/libfoo-1.2.4.tar.gz +++ b/tests/load/1/stable/libfoo-1.2.4.tar.gz diff --git a/tests/load/1/stable/packages.manifest b/tests/load/1/stable/packages.manifest index 1c5dd58..8d51838 100644 --- a/tests/load/1/stable/packages.manifest +++ b/tests/load/1/stable/packages.manifest @@ -9,8 +9,12 @@ build-email: foo-builds@example.com builds: default legacy; Stable configurations only. builds: -32; 64-bit targets only builds: &msvc_13_up; Not too old MSVC. +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.0.tar.gz -sha256sum: e89c6d746f8b1ea3ec58d294946d2f683d133438d2ac8c88549ba24c19627e76 +sha256sum: 7382152bac5b4ce10215a5ecd6c94c490d0efc007031d3b03f407d068b74e624 : name: libfoo version: 1.2.2-alpha.1 @@ -23,8 +27,12 @@ email: foo-users@example.com depends: libmisc [0.1 2.0-) | libmisc [2.0 5.0] depends: libgenx (0.2 3.0) depends: libexpat < 5.2 | libexpat (1 5.1] +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.2.2-alpha.1.tar.gz -sha256sum: f5d3e9e6e8f9621a638b1375d31f0eb50e6279d8066170b25da21e84198cfd82 +sha256sum: 71321f6616036380ac5c9c5dc81efa04b23577ef9dc18f1ce413587bb57677c9 : name: libfoo version: 1.2.2 @@ -35,8 +43,12 @@ url: http://www.example.com/foo/ email: foo-users@example.com depends: libbar <= 2.4.0 depends: libexp == +2-1.2 +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.2.2.tar.gz -sha256sum: 088068ea3d69542a153f829cf836013374763148fba0a43d8047974f58b5efd7 +sha256sum: 75d2a7d3eec62d63afd3d3a84d91bd02b05ecb16cd0907d5b0db1fc654e3753f : name: libfoo version: 1.2.3+4 @@ -47,8 +59,12 @@ keywords: c++ foo url: http://www.example.com/foo/ email: foo-users@example.com depends: libmisc >= 2.0.0 +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.2.3+4.tar.gz -sha256sum: f2ebecac6cac8addd7c623bc1becf055e76b13a0d2dd385832b92c38c58956d8 +sha256sum: 24c53899bd4dbfdde6a727e07724984bfb4ca7f20142291c40e30304f15434c3 : name: libfoo version: 1.2.4 @@ -62,5 +78,9 @@ changes: some changes 2 url: http://www.example.com/foo/ email: foo-users@example.com depends: libmisc >= 2.0.0 +bootstrap-build:\ +project = libfoo + +\ location: libfoo-1.2.4.tar.gz -sha256sum: aa1606323bfc59b70de642629dc5d8318cc5348e3646f90ed89406d975db1e1d +sha256sum: 98f80ca0cd1c053fd45ab37f72a6a31f1a0304747c636822df8d573420284642 diff --git a/tests/load/1/stable/signature.manifest b/tests/load/1/stable/signature.manifest index 8201745..9d3b944 100644 --- a/tests/load/1/stable/signature.manifest +++ b/tests/load/1/stable/signature.manifest @@ -1,13 +1,13 @@ : 1 -sha256sum: 2c710248e2654f57c0d572585195f2221a33a6fbe753a511874b2006db5c221a +sha256sum: 300aaa95b27809ca4bccf4b6eeb5f81701117457bd9e6018964004d4bbedbfb8 signature:\ -C8H2WkJR27bK35E0IAQxdLtR7vjYcQxvMxdPQSnk3GhpFQwRhpQ/dF0b81cC1a6ytd5/uPwM/L+x -Ki2GJvIuvvWurySofFGVd1iMYeOKMpHWbcYx+MvoDm2vxgUQxh3he5AAt5FPvvaGd72fmRWeoVdm -cpZmPKhGIQffGTcL4bifgXgCEQDPjArZY2g7Van/ZWbEhFQGRYVGvLbfZpeNG5OZSYV1kRrCI7lE -nGMsH/HfrkUivwPx5K0WyDF+ukj/DoZaY5PD+XHZ6hVVza++X5/0mH55yurrU1DG+jV/2HP1WGXL -tZq9sWOrdCG1zkfdefRY5ZbYVc82PeLzdONNLNaPYs2G/wFoPaYiXweiJ/rF6OGm6mcquupaBCeX -iR+c9IHiZBXhxGXkY0YsVzgsv/RxDi8LfUS6z/h1NN686F93jPwieKDJRBGl6voyH1h8O1j/aRCw -1MpeNFttF+U4GPDWkUadCoxpygPZOHJc7SW0EreJI8iePwrQDaJSdxo11DqCSpHs57YUnhJUT/q1 -C9RalWCKfAJgk5i83dbKBpTZqCq0oHoh5iHTr9y9IH9yAAvoUudppoM1IGNnT42frVNXPwjRJLB2 -j0NDKPkeBBg0Cgd2vZ7kvarKOP0GPiNUOUv17mFo6c1W9DEmZFoAfc0iyYOE76VgJsAK0UY8MZI= +kBjhtZKRx9FJmJmm+Bh8gUmvK99kQCWmA9qlyLWLfOiapvtliGn65eCh1uCbrjKWGri8SHPD1ABZ +aYcfPE5Cg6L40FltFqYF5qliZo2V8Um3JPd56Bm1S8/yBTftojEBNS4RYBLcLLmd6Blmb9/igTAQ +OOTep/zb3IGZuFzbiLHbfhP7VA4m7PUxV7AlZwSY8IhxzGurQWfBIpGskqkGk57mO+rQy3sq6bWH +IzbgA4hvfkiXyXzcuDIlfcSQaIAfzClqybNaEFgkqKeoZgUkPLbZiYCGasyVCSmCE0gthgOZL7gJ +sJU3iyQMeawCKuQhCr2wz1xhJjLdT41eQ1b4YbtKKneVKOhZDRXosw86xk0Ghi7SeNmWVa2y0UIg +fLIU8um23Gh5Tf37C4pyWjSLVJ4TwAvUGPhxBxd1E2Khuvak/x/+vo983tEikI14InLkdt/KdVpd +lprE9YEACNFaSJ1cwI6aX8exL4a7kku3YG65Mgg+JD5E1HGAOIfCZ/4FJxBF5Mq6cXDT0fIcVR7b +ioAJVa9q+EdRrMXlovCCDvmFUvwggTTL1N1/8o0gZrgJW76BTZrNA+MJ2pkwlsJDrNi6JhqbcbOn +ldZG1n5IwCEt4SvL2LV5iR/iH7iWrIPnkqJJu8rGKWWywz+mmOsl/TBpQylKKXmYLAnWA31htC4= \ diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx index 05efd15..ae3d03d 100644 --- a/tests/load/driver.cxx +++ b/tests/load/driver.cxx @@ -3,7 +3,6 @@ #include <iostream> #include <exception> -#include <algorithm> // sort(), find() #include <odb/session.hxx> #include <odb/transaction.hxx> @@ -397,7 +396,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpvxy)); assert (fpvxy->sha256sum && *fpvxy->sha256sum == - "c994fd49f051ab7fb25f3a4e68ca878e484c5d3c2cb132b37d41224b0621b618"); + "c25e5cae2f72664a3961c3ef88a82e67150c4bcc2a5e1fb4d250e621c5574187"); assert (fpvxy->buildable); @@ -433,7 +432,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv1)); assert (fpv1->sha256sum && *fpv1->sha256sum == - "e89c6d746f8b1ea3ec58d294946d2f683d133438d2ac8c88549ba24c19627e76"); + "7382152bac5b4ce10215a5ecd6c94c490d0efc007031d3b03f407d068b74e624"); assert (fpv1->buildable); @@ -478,7 +477,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv2)); assert (fpv2->sha256sum && *fpv2->sha256sum == - "088068ea3d69542a153f829cf836013374763148fba0a43d8047974f58b5efd7"); + "75d2a7d3eec62d63afd3d3a84d91bd02b05ecb16cd0907d5b0db1fc654e3753f"); assert (!fpv2->buildable); @@ -542,7 +541,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv2a)); assert (fpv2a->sha256sum && *fpv2a->sha256sum == - "f5d3e9e6e8f9621a638b1375d31f0eb50e6279d8066170b25da21e84198cfd82"); + "71321f6616036380ac5c9c5dc81efa04b23577ef9dc18f1ce413587bb57677c9"); assert (!fpv2a->buildable); @@ -580,7 +579,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv3)); assert (fpv3->sha256sum && *fpv3->sha256sum == - "f2ebecac6cac8addd7c623bc1becf055e76b13a0d2dd385832b92c38c58956d8"); + "24c53899bd4dbfdde6a727e07724984bfb4ca7f20142291c40e30304f15434c3"); assert (!fpv3->buildable); @@ -619,7 +618,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv4)); assert (fpv4->sha256sum && *fpv4->sha256sum == - "aa1606323bfc59b70de642629dc5d8318cc5348e3646f90ed89406d975db1e1d"); + "98f80ca0cd1c053fd45ab37f72a6a31f1a0304747c636822df8d573420284642"); assert (!fpv4->buildable); @@ -708,7 +707,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (xpv)); assert (xpv->sha256sum && *xpv->sha256sum == - "1833906dd93ccc0cda832d6a1b3ef9ed7877bb9958b46d9b2666033d4a7919c9"); + "aa52d5b49ee1bad825cd6bca554f72636e8451f93c74f9a443bafce3c2bf82c0"); assert (xpv->buildable); @@ -844,7 +843,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (fpv5)); assert (fpv5->sha256sum && *fpv5->sha256sum == - "d23a7ff116ab7264c3d423af97e4830bdaa8c9101cd95b210b19a97bb8512b74"); + "fe07978d72ab65c2ad72b0325aa56944cf093248d39edcb472a2fe5835defa3d"); assert (fpv5->buildable); @@ -903,7 +902,7 @@ test_pkg_repos (const cstrings& loader_args, assert (check_location (epv)); assert (epv->sha256sum && *epv->sha256sum == - "317c8c6f45d9dfdfdef3a823411920cecd51729c7c4f58f9a0b0bbd681c07bd6"); + "d90cfe583890cd0c05cdfc204e69dd3b986c2da49851f7a87fa0ca870788ff79"); // Verify libpq package version. // diff --git a/www/builds-body.css b/www/builds-body.css index 5ee4fe7..2706a1d 100644 --- a/www/builds-body.css +++ b/www/builds-body.css @@ -37,15 +37,15 @@ .build th, #filter th { - width: 7.0em; + width: 7.3em; } .build tr.name td .value, .build tr.version td .value, .build tr.toolchain td .value, -.build tr.config td .value, -.build tr.machine td .value, .build tr.target td .value, +.build tr.tgt-config td .value, +.build tr.pkg-config td .value, .build tr.timestamp td .value, .build tr.login td .value, .build tr.result td .value, diff --git a/www/package-version-details-body.css b/www/package-version-details-body.css index 34445df..7001d14 100644 --- a/www/package-version-details-body.css +++ b/www/package-version-details-body.css @@ -305,11 +305,12 @@ h1, h2, h3 .build th { - width: 7.0em; + width: 7.3em; } .build tr.toolchain td .value, -.build tr.config td .value, +.build tr.tgt-config td .value, +.build tr.pkg-config td .value, .build tr.timestamp td .value, .build tr.login td .value, .build tr.result td .value |