diff options
| -rw-r--r-- | etc/brep-module.conf | 40 | ||||
| -rw-r--r-- | libbrep/build.cxx | 8 | ||||
| -rw-r--r-- | libbrep/build.hxx | 65 | ||||
| -rw-r--r-- | libbrep/build.xml | 18 | ||||
| -rw-r--r-- | migrate/migrate.cxx | 32 | ||||
| -rw-r--r-- | mod/buildfile | 2 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 99 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 262 | ||||
| -rw-r--r-- | mod/module.cli | 90 | ||||
| -rw-r--r-- | monitor/monitor.cli | 36 | ||||
| -rw-r--r-- | monitor/monitor.cxx | 688 |
11 files changed, 940 insertions, 400 deletions
diff --git a/etc/brep-module.conf b/etc/brep-module.conf index 57d4865..9acef26 100644 --- a/etc/brep-module.conf +++ b/etc/brep-module.conf @@ -140,16 +140,20 @@ menu About=?about # build-forced-rebuild-timeout 600 -# Time to wait before considering a package for a normal rebuild. Must be -# specified in seconds. Default is 24 hours. +# Time to wait before considering a package for a soft rebuild (only to be +# performed if the build environment or any of the package dependencies have +# changed). Must be specified in seconds. The special zero value disables soft +# rebuilds. Default is 24 hours. # -# build-normal-rebuild-timeout 86400 +# build-soft-rebuild-timeout 86400 -# Alternative package rebuild timeout to use instead of the normal rebuild -# timeout (see the build-normal-rebuild-timeout option for details) during -# the specified time interval. Must be specified in seconds. Default is the -# time interval length. +# Alternative package soft rebuild timeout to use instead of the soft rebuild +# timeout (see the build-soft-rebuild-timeout option for details) during the +# specified time interval. Must be specified in seconds. Default is the time +# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild +# timeout is greater than 24 hours (thus the rebuild is only triggered within +# the last 24 hours of the build-soft-rebuild-timeout expiration). # # The alternative rebuild timeout can be used to "pull" the rebuild window to # the specified time of day, for example, to optimize load and/or power @@ -169,9 +173,25 @@ menu About=?about # times must both be either specified or absent. If unspecified, then no # alternative rebuild timeout will be used. # -# build-alt-rebuild-timeout -# build-alt-rebuild-start -# build-alt-rebuild-stop +# build-alt-soft-rebuild-timeout +# build-alt-soft-rebuild-start +# build-alt-soft-rebuild-stop + + +# Time to wait before considering a package for a hard rebuild (to be +# performed unconditionally). Must be specified in seconds. The special zero +# value disables hard rebuilds. Default is 7 days. +# +# build-hard-rebuild-timeout 604800 + + +# Alternative package hard rebuild timeout. The semantics is the same as for +# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout +# option. +# +# build-alt-hard-rebuild-timeout +# build-alt-hard-rebuild-start +# build-alt-hard-rebuild-stop # The maximum size of the build task request manifest accepted. Note that the diff --git a/libbrep/build.cxx b/libbrep/build.cxx index 5f8cd71..9cbad8f 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -62,7 +62,9 @@ namespace brep optional<string> inr, optional<string> afp, optional<string> ach, string mnm, string msm, - butl::target_triplet trg) + butl::target_triplet trg, + string ccs, + string mcs) : id (package_id (move (tnt), move (pnm), pvr), move (cfg), move (tnm), tvr), @@ -79,7 +81,9 @@ namespace brep agent_fingerprint (move (afp)), agent_challenge (move (ach)), machine (move (mnm)), machine_summary (move (msm)), - target (move (trg)) + target (move (trg)), + controller_checksum (move (ccs)), + machine_checksum (move (mcs)) { } diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 151f8dc..fbab13d 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -30,7 +30,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 12 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 14, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 15, closed) // We have to keep these mappings at the global scope instead of inside // the brep namespace because they need to be also effective in the @@ -194,7 +194,9 @@ namespace brep optional<string> agent_fingerprint, optional<string> agent_challenge, string machine, string machine_summary, - butl::target_triplet); + butl::target_triplet, + string controller_checksum, + string machine_checksum); build_id id; @@ -223,15 +225,26 @@ namespace brep // optional<result_status> status; - // Time of setting the result status that can be considered as the build - // task completion (currently all the result_status values). Initialized - // with timestamp_nonexistent by default. + // Times of the last soft/hard completed (re)builds. Used to decide when + // to perform soft and hard rebuilds, respectively. + // + // The soft timestamp is updated whenever we receive a task result. + // + // The hard timestamp is updated whenever we receive a task result with + // a status other than skip. + // + // Also note that whenever hard_timestamp is updated, soft_timestamp is + // updated as well and whenever soft_timestamp is updated, timestamp is + // updated as well. Thus the following condition is always true: // - // Note that in the future we may not consider abort and abnormal as the - // task completion and, for example, proceed with automatic rebuild (the - // flake monitor idea). + // hard_timestamp <= soft_timestamp <= timestamp // - timestamp_type completion_timestamp; + // Note that the "completed" above means that we may analyze the task + // result/log and deem it as not completed and proceed with automatic + // rebuild (the flake monitor idea). + // + timestamp_type soft_timestamp; + timestamp_type hard_timestamp; // May be present only for the building state. // @@ -248,6 +261,21 @@ namespace brep operation_results results; odb::section results_section; + // Checksums of entities involved in the build. + // + // Optional checksums are provided by the external entities (agent and + // worker). All are absent initially. + // + // Note that the agent checksum can also be absent after the hard rebuild + // task is issued and the worker and dependency checksums - after a failed + // rebuild (error result status or worse). + // + string controller_checksum; + string machine_checksum; + optional<string> agent_checksum; + optional<string> worker_checksum; + optional<string> dependency_checksum; + // Database mapping. // #pragma db member(id) id column("") @@ -265,15 +293,6 @@ namespace brep // #pragma db member(timestamp) index - // This is not required since 0.14.0. Note however, that just dropping - // this line won't pan out since this would require migration which odb is - // currently unable to handle automatically, advising to re-implement this - // change by adding a new data member with the desired default value, - // migrating the data, and deleting the old data member. This sounds a bit - // hairy, so let's keep it for now. - // - #pragma db member(completion_timestamp) default(0) - #pragma db member(results) id_column("") value_column("") \ section(results_section) @@ -396,10 +415,14 @@ namespace brep string& toolchain_name; // Tracks id.toolchain_name. upstream_version toolchain_version; // Original of id.toolchain_version. - // Time of the latest delay report. Initialized with timestamp_nonexistent - // by default. + // Times of the latest soft and hard rebuild delay reports. Initialized + // with timestamp_nonexistent by default. + // + // Note that both reports notify about initial build delays (at their + // respective time intervals). // - timestamp report_timestamp; + timestamp report_soft_timestamp; + timestamp report_hard_timestamp; // Time when the package is initially considered as buildable for this // configuration and toolchain. It is used to track the build delay if the diff --git a/libbrep/build.xml b/libbrep/build.xml index 0ca362a..6a2301b 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,22 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> + <changeset version="15"> + <alter-table name="build"> + <add-column name="soft_timestamp" type="BIGINT" null="false"/> + <add-column name="hard_timestamp" type="BIGINT" null="false"/> + <add-column name="controller_checksum" type="TEXT" null="false"/> + <add-column name="machine_checksum" type="TEXT" null="false"/> + <add-column name="agent_checksum" type="TEXT" null="true"/> + <add-column name="worker_checksum" type="TEXT" null="true"/> + <add-column name="dependency_checksum" type="TEXT" null="true"/> + <drop-column name="completion_timestamp"/> + </alter-table> + <alter-table name="build_delay"> + <add-column name="report_soft_timestamp" type="BIGINT" null="false"/> + <add-column name="report_hard_timestamp" type="BIGINT" null="false"/> + <drop-column name="report_timestamp"/> + </alter-table> + </changeset> + <changeset version="14"/> <changeset version="13"> diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx index 1093b7c..468d411 100644 --- a/migrate/migrate.cxx +++ b/migrate/migrate.cxx @@ -14,6 +14,8 @@ #include <libbutl/pager.hxx> +#include <libbrep/build.hxx> +#include <libbrep/build-odb.hxx> #include <libbrep/package.hxx> #include <libbrep/package-odb.hxx> #include <libbrep/database-lock.hxx> @@ -224,6 +226,36 @@ package_migrate_v20 ([] (database& db) }); #endif +// Register the data migration functions for the build database schema. +// +template <schema_version v> +using build_migration_entry_base = + data_migration_entry<v, LIBBREP_BUILD_SCHEMA_VERSION_BASE>; + +template <schema_version v> +struct build_migration_entry: build_migration_entry_base<v> +{ + build_migration_entry (void (*f) (database& db)) + : build_migration_entry_base<v> (f, "build") {} +}; + +static const build_migration_entry<15> +build_migrate_v15 ([] (database& db) +{ + // Setting proper checksums here feels a bit hairy. Let's assign them + // naturally on the first rebuild. + // + db.execute ("UPDATE build SET " + "soft_timestamp = completion_timestamp, " + "hard_timestamp = completion_timestamp, " + "controller_checksum = '', " + "machine_checksum = ''"); + + db.execute ("UPDATE build_delay SET " + "report_soft_timestamp = report_timestamp, " + "report_hard_timestamp = report_timestamp"); +}); + // main() function // int diff --git a/mod/buildfile b/mod/buildfile index ff9cd60..58a3caf 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -50,7 +50,7 @@ if $cli.configured cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix mod --guard-prefix MOD --generate-specifier \ --cxx-prologue "#include <mod/types-parsers.hxx>" \ ---cli-namespace brep::cli --generate-file-scanner --option-length 45 \ +--cli-namespace brep::cli --generate-file-scanner --option-length 46 \ --generate-modifier --generate-description --option-prefix "" # Include the generated cli files into the distribution and don't remove diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 3ae9f0f..1c46fc1 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -287,7 +287,6 @@ handle (request& rq, response&) // shared_ptr<build> bld; - optional<result_status> prev_status; bool build_notify (false); bool unforced (true); @@ -382,6 +381,58 @@ handle (request& rq, response&) if (auth) { + // Verify the result status/checksums. + // + // Specifically, if the result status is skip, then it can only be in + // response to the soft rebuild task (all checksums are present in the + // build object) and the result checksums must match the build object + // checksums. On verification failure respond with the bad request + // HTTP code (400). + // + if (rqm.result.status == result_status::skip) + { + if (!b->agent_checksum || + !b->worker_checksum || + !b->dependency_checksum) + throw invalid_request (400, "unexpected skip result status"); + + // Can only be absent for initial build, in which case the checksums + // are also absent and we would end up with the above 400 response. + // + assert (b->status); + + // Verify that the result checksum matches the build checksum and + // throw invalid_request(400) if that's not the case. + // + auto verify = [] (const string& build_checksum, + const optional<string>& result_checksum, + const char* what) + { + if (!result_checksum) + throw invalid_request ( + 400, + string (what) + + " checksum is expected for skip result status"); + + if (*result_checksum != build_checksum) + throw invalid_request ( + 400, + string (what) + " checksum '" + build_checksum + + "' is expected instead of '" + *result_checksum + + "' for skip result status"); + }; + + verify (*b->agent_checksum, rqm.agent_checksum, "agent"); + + verify (*b->worker_checksum, + rqm.result.worker_checksum, + "worker"); + + verify (*b->dependency_checksum, + rqm.result.dependency_checksum, + "dependency"); + } + unforced = b->force == force_state::unforced; // Don't send email to the build-email address for the @@ -392,10 +443,7 @@ handle (request& rq, response&) *b->status == rqm.result.status && unforced); - prev_status = move (b->status); - b->state = build_state::built; - b->status = rqm.result.status; b->force = force_state::unforced; // Cleanup the interactive build login information. @@ -407,22 +455,43 @@ handle (request& rq, response&) b->agent_fingerprint = nullopt; b->agent_challenge = nullopt; - // Mark the section as loaded, so results are updated. - // - b->results_section.load (); - b->results = move (rqm.result.results); - b->timestamp = system_clock::now (); - b->completion_timestamp = b->timestamp; + b->soft_timestamp = b->timestamp; + + // If the result status is other than skip, then save the status, + // results, and checksums and update the hard timestamp. + // + if (rqm.result.status != result_status::skip) + { + b->status = rqm.result.status; + b->hard_timestamp = b->soft_timestamp; + + // Mark the section as loaded, so results are updated. + // + b->results_section.load (); + b->results = move (rqm.result.results); + + // Save the checksums. + // + b->agent_checksum = move (rqm.agent_checksum); + b->worker_checksum = move (rqm.result.worker_checksum); + b->dependency_checksum = move (rqm.result.dependency_checksum); + } build_db_->update (b); - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); + // Don't send the build notification email if the task result is + // `skip`, the configuration is hidden, or is now excluded by the + // package. + // + if (rqm.result.status != result_status::skip && belongs (*cfg, "all")) + { + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); - if (belongs (*cfg, "all") && - !exclude (p->builds, p->constraints, *cfg)) - bld = move (b); + if (!exclude (p->builds, p->constraints, *cfg)) + bld = move (b); + } } } diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 8656f5e..22d0110 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -61,13 +61,23 @@ init (scanner& s) if (options_->build_config_specified ()) { - // Verify that build-alt-rebuild-{start,stop} are both either specified or - // not. + // Verify that build-alt-*-rebuild-{start,stop} are both either specified + // or not. // - if (options_->build_alt_rebuild_start_specified () != - options_->build_alt_rebuild_stop_specified ()) - fail << "build-alt-rebuild-start and build-alt-rebuild-stop " - << "configuration options must both be either specified or not"; + auto bad_alt = [&fail] (const char* what) + { + fail << "build-alt-" << what << "-rebuild-start and build-alt-" << what + << "-rebuild-stop configuration options must both be either " + << "specified or not"; + }; + + if (options_->build_alt_soft_rebuild_start_specified () != + options_->build_alt_soft_rebuild_stop_specified ()) + bad_alt ("soft"); + + if (options_->build_alt_hard_rebuild_start_specified () != + options_->build_alt_hard_rebuild_stop_specified ()) + bad_alt ("hard"); database_module::init (*options_, options_->build_db_retry ()); @@ -277,22 +287,25 @@ handle (request& rq, response& rs) move (fps), move (p->requirements), move (tests), + move (b->dependency_checksum), cm.machine->name, cm.config->target, cm.config->environment, cm.config->args, belongs (*cm.config, module_pkg ? "build2" : "host"), cm.config->warning_regexes, - move (t->interactive)); + move (t->interactive), + move (b->worker_checksum)); return task_response_manifest (move (session), move (b->agent_challenge), move (result_url), + move (b->agent_checksum), move (task)); }; - // Calculate the build (building state) or rebuild (built state) expiration - // time for package configurations + // Calculate the build (building state) or rebuild (built state) + // expiration time for package configurations. // timestamp now (system_clock::now ()); @@ -316,44 +329,95 @@ handle (request& rq, response& rs) timestamp forced_rebuild_expiration ( expiration (options_->build_forced_rebuild_timeout ())); - timestamp normal_rebuild_expiration; - - if (options_->build_alt_rebuild_start_specified ()) + // Calculate the soft/hard rebuild expiration time, based on the + // respective build-{soft,hard}-rebuild-timeout and + // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration + // options. + // + // If normal_timeout is zero, then return timestamp_unknown to indicate + // 'never expire'. Note that this value is less than any build timestamp + // value, including timestamp_nonexistent. + // + // NOTE: there is a similar code in monitor/monitor.cxx. + // + auto build_expiration = [&now] ( + const optional<pair<duration, duration>>& alt_interval, + optional<size_t> alt_timeout, + size_t normal_timeout) { - const duration& start (options_->build_alt_rebuild_start ()); - const duration& stop (options_->build_alt_rebuild_stop ()); + if (normal_timeout == 0) + return timestamp_unknown; - duration dt (daytime (now)); + timestamp r; + chrono::seconds nt (normal_timeout); - // Note that if the stop time is less than the start time then the - // interval extends through the midnight. - // - bool alt_timeout (start <= stop - ? dt >= start && dt < stop - : dt >= start || dt < stop); - - // If we out of the alternative rebuild timeout interval, then fall back - // to using the normal rebuild timeout. - // - if (alt_timeout) + if (alt_interval) { - if (!options_->build_alt_rebuild_timeout_specified ()) + const duration& start (alt_interval->first); + const duration& stop (alt_interval->second); + + duration dt (daytime (now)); + + // Note that if the stop time is less than the start time then the + // interval extends through the midnight. + // + bool use_alt_timeout (start <= stop + ? dt >= start && dt < stop + : dt >= start || dt < stop); + + // If we out of the alternative rebuild timeout interval, then fall + // back to using the normal rebuild timeout. + // + if (use_alt_timeout) { - duration interval_len (start <= stop - ? stop - start - : (24h - start) + stop); + // Calculate the alternative timeout, unless it is specified + // explicitly. + // + duration t; + + if (!alt_timeout) + { + t = start <= stop ? (stop - start) : ((24h - start) + stop); - normal_rebuild_expiration = now - interval_len; + // If the normal rebuild timeout is greater than 24 hours, then + // increase the default alternative timeout by (normal - 24h) (see + // build-alt-soft-rebuild-timeout configuration option for + // details). + // + if (nt > 24h) + t += nt - 24h; + } + else + t = chrono::seconds (*alt_timeout); + + r = now - t; } - else - normal_rebuild_expiration = - expiration (options_->build_alt_rebuild_timeout ()); } - } - if (normal_rebuild_expiration == timestamp_nonexistent) - normal_rebuild_expiration = - expiration (options_->build_normal_rebuild_timeout ()); + return r != timestamp_nonexistent ? r : (now - nt); + }; + + timestamp soft_rebuild_expiration ( + build_expiration ( + (options_->build_alt_soft_rebuild_start_specified () + ? make_pair (options_->build_alt_soft_rebuild_start (), + options_->build_alt_soft_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (options_->build_alt_soft_rebuild_timeout_specified () + ? options_->build_alt_soft_rebuild_timeout () + : optional<size_t> ()), + options_->build_soft_rebuild_timeout ())); + + timestamp hard_rebuild_expiration ( + build_expiration ( + (options_->build_alt_hard_rebuild_start_specified () + ? make_pair (options_->build_alt_hard_rebuild_start (), + options_->build_alt_hard_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (options_->build_alt_hard_rebuild_timeout_specified () + ? options_->build_alt_hard_rebuild_timeout () + : optional<size_t> ()), + options_->build_hard_rebuild_timeout ())); // Return the challenge (nonce) if brep is configured to authenticate bbot // agents. Return nullopt otherwise. @@ -555,6 +619,68 @@ handle (request& rq, response& rs) prep_bld_query bld_prep_query ( conn->prepare_query<build> ("mod-build-task-build-query", bq)); + // Return true if a package needs to be rebuilt. + // + auto needs_rebuild = [&forced_rebuild_expiration, + &soft_rebuild_expiration, + &hard_rebuild_expiration] (const build& b) + { + assert (b.state == build_state::built); + + return (b.force == force_state::forced && + b.soft_timestamp <= forced_rebuild_expiration) || + b.soft_timestamp <= soft_rebuild_expiration || + b.hard_timestamp <= hard_rebuild_expiration; + }; + + // Convert a build to the hard rebuild, resetting the agent checksum and + // dropping the previous build task result. + // + // Note that since the checksums are hierarchical, the agent checksum + // reset will trigger resets of the "subordinate" checksums up to the + // dependency checksum and so the package will be rebuilt. + // + // Also note that there is no sense to keep the build task result since we + // don't accept the skip result for the hard rebuild task. We, however, + // keep the status intact (see below for the reasoning). + // + auto convert_to_hard = [] (const shared_ptr<build>& b) + { + b->agent_checksum = nullopt; + + // Mark the section as loaded, so results are updated. + // + b->results_section.load (); + b->results.clear (); + }; + + // Return SHA256 checksum of the controller logic and the configuration + // target, environment, arguments, and warning-detecting regular + // expressions. + // + auto controller_checksum = [] (const build_config& c) + { + sha256 cs ("1"); // Hash the logic version. + + cs.append (c.target.string ()); + cs.append (c.environment ? *c.environment : ""); + + for (const string& a: c.args) + cs.append (a); + + for (const string& re: c.warning_regexes) + cs.append (re); + + return string (cs.string ()); + }; + + // Return the machine id as a machine checksum. + // + auto machine_checksum = [] (const machine_header_manifest& m) + { + return m.id; + }; + while (tsm.session.empty ()) { transaction t (conn->begin ()); @@ -604,9 +730,7 @@ handle (request& rq, response& rs) { assert (i->force != force_state::forcing); - if (i->timestamp <= (i->force == force_state::forced - ? forced_rebuild_expiration - : normal_rebuild_expiration)) + if (needs_rebuild (*i)) rebuilds.emplace_back (i.load ()); } } @@ -666,14 +790,15 @@ handle (request& rq, response& rs) move (cl), mh.name, move (mh.summary), - cm.config->target); + cm.config->target, + controller_checksum (*cm.config), + machine_checksum (*cm.machine)); build_db_->persist (b); } else { - // The package configuration is in the building state, and there - // are no results. + // The package configuration is in the building state. // // Note that in both cases we keep the status intact to be able // to compare it with the final one in the result request @@ -681,12 +806,7 @@ handle (request& rq, response& rs) // email. The same is true for the forced flag (in the sense // that we don't set the force state to unforced). // - // Load the section to assert the above statement. - // - build_db_->load (*b, b->results_section); - - assert (b->state == build_state::building && - b->results.empty ()); + assert (b->state == build_state::building); b->state = build_state::building; b->interactive = move (login); @@ -703,6 +823,22 @@ handle (request& rq, response& rs) b->machine = mh.name; b->machine_summary = move (mh.summary); b->target = cm.config->target; + + string ccs (controller_checksum (*cm.config)); + string mcs (machine_checksum (*cm.machine)); + + // Issue the hard rebuild if it is forced or the configuration + // or machine has changed. + // + if (b->hard_timestamp <= hard_rebuild_expiration || + b->force == force_state::forced || + b->controller_checksum != ccs || + b->machine_checksum != mcs) + convert_to_hard (b); + + b->controller_checksum = move (ccs); + b->machine_checksum = move (mcs); + b->timestamp = system_clock::now (); build_db_->update (b); @@ -770,10 +906,9 @@ handle (request& rq, response& rs) b = build_db_->find<build> (b->id); - if (b != nullptr && b->state == build_state::built && - b->timestamp <= (b->force == force_state::forced - ? forced_rebuild_expiration - : normal_rebuild_expiration)) + if (b != nullptr && + b->state == build_state::built && + needs_rebuild (*b)) { auto i (cfg_machines.find (b->id.configuration.c_str ())); @@ -828,10 +963,23 @@ handle (request& rq, response& rs) b->target = cm.config->target; - // Mark the section as loaded, so results are updated. + // Issue the hard rebuild if the timeout expired, rebuild is + // forced, or the configuration or machine has changed. // - b->results_section.load (); - b->results.clear (); + // Note that we never reset the build status (see above for the + // reasoning). + // + string ccs (controller_checksum (*cm.config)); + string mcs (machine_checksum (*cm.machine)); + + if (b->hard_timestamp <= hard_rebuild_expiration || + b->force == force_state::forced || + b->controller_checksum != ccs || + b->machine_checksum != mcs) + convert_to_hard (b); + + b->controller_checksum = move (ccs); + b->machine_checksum = move (mcs); b->timestamp = system_clock::now (); diff --git a/mod/module.cli b/mod/module.cli index c95c20c..c2dce5b 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -195,45 +195,87 @@ namespace brep be specified in seconds. Default is 10 minutes." } - size_t build-normal-rebuild-timeout = 86400 + size_t build-soft-rebuild-timeout = 86400 { "<seconds>", - "Time to wait before considering a package for a normal rebuild. Must - be specified in seconds. Default is 24 hours." + "Time to wait before considering a package for a soft rebuild (only to + be performed if the build environment or any of the package + dependencies have changed). Must be specified in seconds. The special + zero value disables soft rebuilds. Default is 24 hours" } - size_t build-alt-rebuild-timeout + size_t build-alt-soft-rebuild-timeout { "<seconds>", - "Alternative package rebuild timeout to use instead of the normal - rebuild timeout (see \cb{build-normal-rebuild-timeout} for details) + "Alternative package soft rebuild timeout to use instead of the soft + rebuild timeout (see \cb{build-soft-rebuild-timeout} for details) during the time interval specified with the - \cb{build-alt-rebuild-start} and \cb{build-alt-rebuild-stop} options. - Must be specified in seconds. Default is the time interval length." + \cb{build-alt-soft-rebuild-start} and + \cb{build-alt-soft-rebuild-stop} options. Must be specified in + seconds. Default is the time interval length plus + \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is + greater than 24 hours (thus the rebuild is only triggered within the + last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)." } - duration build-alt-rebuild-start + duration build-alt-soft-rebuild-start { "<hours>:<minutes>", - "The start time of the alternative package rebuild timeout (see - \cb{build-alt-rebuild-timeout} for details). Must be specified as - a time of day in the local timezone. The \cb{build-alt-rebuild-start} - and \cb{build-alt-rebuild-stop} options must be either both specified - or absent. If unspecified, then no alternative rebuild timeout will - be used." + "The start time of the alternative package soft rebuild timeout (see + \cb{build-alt-soft-rebuild-timeout} for details). Must be specified + as a time of day in the local timezone. The + \cb{build-alt-soft-rebuild-start} and + \cb{build-alt-soft-rebuild-stop} options must be either both + specified or absent. If unspecified, then no alternative rebuild + timeout will be used." } - duration build-alt-rebuild-stop + duration build-alt-soft-rebuild-stop { "<hours>:<minutes>", - "The end time of the alternative package rebuild timeout (see - \cb{build-alt-rebuild-timeout} for details). Must be specified as - a time of day in the local timezone. If it is less than the - \cb{build-alt-rebuild-start} option value, then the time interval - extends through midnight. The \cb{build-alt-rebuild-start} and - \cb{build-alt-rebuild-stop} options must be either both specified or - absent. If unspecified, then no alternative rebuild timeout will be - used." + "The end time of the alternative package soft rebuild timeout (see + \cb{build-alt-soft-rebuild-timeout} for details). Must be specified + as a time of day in the local timezone. If it is less than the + \cb{build-alt-soft-rebuild-start} option value, then the time + interval extends through midnight. The + \cb{build-alt-soft-rebuild-start} and + \cb{build-alt-soft-rebuild-stop} options must be either both + specified or absent. If unspecified, then no alternative rebuild + timeout will be used." + } + + size_t build-hard-rebuild-timeout = 604800 + { + "<seconds>", + "Time to wait before considering a package for a hard rebuild (to be + performed unconditionally). Must be specified in seconds. The special + zero value disables hard rebuilds. Default is 7 days." + } + + size_t build-alt-hard-rebuild-timeout + { + "<seconds>", + "Alternative package hard rebuild timeout. The semantics is the + same as for the \cb{build-alt-soft-rebuild-timeout} option but + for the \cb{build-hard-rebuild-timeout} option." + } + + duration build-alt-hard-rebuild-start + { + "<hours>:<minutes>", + "The start time of the alternative package hard rebuild timeout (see + \cb{build-alt-hard-rebuild-timeout} for details). The semantics is + the same as for the \cb{build-alt-soft-rebuild-start} option but + for the \cb{build-hard-rebuild-timeout} option." + } + + duration build-alt-hard-rebuild-stop + { + "<hours>:<minutes>", + "The end time of the alternative package hard rebuild timeout (see + \cb{build-alt-hard-rebuild-timeout} for details). The semantics is + the same as for the \cb{build-alt-soft-rebuild-stop} option but + for the \cb{build-hard-rebuild-timeout} option." } }; diff --git a/monitor/monitor.cli b/monitor/monitor.cli index edfc004..3a58a1d 100644 --- a/monitor/monitor.cli +++ b/monitor/monitor.cli @@ -60,18 +60,26 @@ namespace brep { "\h|OPTIONS|" - std::size_t --build-timeout + std::size_t --soft-rebuild-timeout { "<seconds>", - "Time to wait (in seconds) before considering a package build as + "Time to wait (in seconds) before considering a package soft (re)build as delayed. If unspecified, it is the sum of the package rebuild timeout - (normal rebuild timeout if the alternative timeout is unspecified and - the maximum of two otherwise) and the build result timeout (see the - \cb{build-normal-rebuild-timeout}, \cb{build-alt-rebuild-*}, and - \cb{build-result-timeout} \c{brep} module configuration options - for details). - - Note that a package that was not built before it was archived is + (soft rebuild timeout if the alternative timeout is unspecified and + the maximum of two otherwise) and the build result timeout (see + the \cb{build-soft-rebuild-timeout}, \cb{build-alt-soft-rebuild-*}, + and \cb{build-result-timeout} \cb{brep} module configuration options + for details). The special zero value disables monitoring of soft + rebuilds. + + Note that if both soft and hard rebuilds are disabled in the + \cb{brep} module configuration, then \cb{brep-monitor} is unable to + come up with a reasonable build timeout on its own. In this case, to + monitor the initial package build delays, you may need to specify + either \cb{--soft-rebuild-timeout} or \cb{--hard-rebuild-timeout} + explicitly. + + Also note that a package that was not built before it was archived is always considered as delayed. However, to distinguish this case from a situation where a package was archived before a configuration have been added, \cb{brep-monitor} needs to observe the package as @@ -81,6 +89,16 @@ namespace brep timeout." } + std::size_t --hard-rebuild-timeout + { + "<seconds>", + "Time to wait (in seconds) before considering a package hard (re)build + as delayed. If unspecified, it is calculated in the same way as for + \cb{--soft-rebuild-timeout} but using the + \cb{build-hard-rebuild-timeout} and \cb{build-alt-hard-rebuild-*} + \cb{brep} module configuration options." + } + std::size_t --report-timeout { "<seconds>", diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 5c4d451..7c20efe 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -41,6 +41,212 @@ namespace brep // struct failed {}; + // We will collect and report build delays as separate steps not to hold + // database locks while printing to stderr. Also we need to order delays + // properly, so while printing reports we could group delays by toolchain + // and configuration. + // + // To achieve that, we will iterate through all possible package builds + // creating the list of delays with the following sort priority: + // + // 1: toolchain name + // 2: toolchain version (descending) + // 3: configuration name + // 4: tenant + // 5: package name + // 6: package version (descending) + // + struct compare_delay + { + bool + operator() (const shared_ptr<const build_delay>& x, + const shared_ptr<const build_delay>& y) const + { + if (int r = x->toolchain_name.compare (y->toolchain_name)) + return r < 0; + + if (int r = x->toolchain_version.compare (y->toolchain_version)) + return r > 0; + + if (int r = x->configuration.compare (y->configuration)) + return r < 0; + + if (int r = x->tenant.compare (y->tenant)) + return r < 0; + + if (int r = x->package_name.compare (y->package_name)) + return r < 0; + + return x->package_version.compare (y->package_version) > 0; + } + }; + + // The ordered list of delays to report. + // + class delay_report + { + public: + // Note that in the brief mode we also need to print the total number of + // delays (reported or not) per configuration. Thus, we add all delays to + // the report object, marking them if we need to report them or not. + // + void + add_delay (shared_ptr<build_delay>, bool report); + + bool + empty () const + { + return reported_delay_count_ == 0; + } + + // In the brief mode (if full is false) print the number of reported/total + // (if total is true) delayed package builds per configuration rather than + // the packages themselves. + // + void + print (const char* header, bool total, bool full) const; + + private: + // Maps delays to the report flag. + // + map<shared_ptr<const build_delay>, bool, compare_delay> delays_; + size_t reported_delay_count_ = 0; + }; + + void delay_report:: + add_delay (shared_ptr<build_delay> delay, bool report) + { + delays_.emplace (move (delay), report); + + if (report) + ++reported_delay_count_; + } + + void delay_report:: + print (const char* header, bool total, bool full) const + { + if (empty ()) + return; + + cerr << header << " (" << reported_delay_count_; + + if (total) + cerr << '/' << delays_.size (); + + cerr << "):" << endl; + + // Group the printed delays by toolchain and configuration. + // + const string* toolchain_name (nullptr); + const version* toolchain_version (nullptr); + const string* configuration (nullptr); + + size_t config_reported_delay_count (0); + size_t config_total_delay_count (0); + + auto brief_config = [&configuration, + &config_reported_delay_count, + &config_total_delay_count, + total] () + { + if (configuration != nullptr) + { + // Only print configurations with delays that needs to be reported. + // + if (config_reported_delay_count != 0) + { + cerr << " " << *configuration << " (" + << config_reported_delay_count; + + if (total) + cerr << '/' << config_total_delay_count; + + cerr << ')' << endl; + } + + config_reported_delay_count = 0; + config_total_delay_count = 0; + } + }; + + for (const auto& dr: delays_) + { + bool report (dr.second); + + if (full && !report) + continue; + + const shared_ptr<const build_delay>& d (dr.first); + + // Print the toolchain, if changed. + // + if (toolchain_name == nullptr || + d->toolchain_name != *toolchain_name || + d->toolchain_version != *toolchain_version) + { + if (!full) + brief_config (); + + if (toolchain_name != nullptr) + cerr << endl; + + cerr << " " << d->toolchain_name; + + if (!d->toolchain_version.empty ()) + cerr << "/" << d->toolchain_version; + + cerr << endl; + + toolchain_name = &d->toolchain_name; + toolchain_version = &d->toolchain_version; + configuration = nullptr; + } + + // Print the configuration, if changed. + // + if (configuration == nullptr || d->configuration != *configuration) + { + if (full) + { + if (configuration != nullptr) + cerr << endl; + + cerr << " " << d->configuration << endl; + } + else + brief_config (); + + configuration = &d->configuration; + } + + // Print the delayed build package in the full report mode and count + // configuration builds otherwise. + // + if (full) + { + // We can potentially extend this information with the archived flag + // or the delay duration. + // + cerr << " " << d->package_name << "/" << d->package_version; + + if (!d->tenant.empty ()) + cerr << " " << d->tenant; + + cerr << endl; + } + else + { + if (report) + ++config_reported_delay_count; + + ++config_total_delay_count; + } + } + + if (!full) + brief_config (); + } + static const char* help_info ( " info: run 'brep-monitor --help' for more information"); @@ -141,12 +347,24 @@ namespace brep return 1; } - if (mod_ops.build_alt_rebuild_start_specified () != - mod_ops.build_alt_rebuild_stop_specified ()) + auto bad_alt = [&f] (const char* what) + { + cerr << "build-alt-" << what << "-rebuild-start and build-alt-" + << what << "-rebuild-stop configuration options must both be " + << "either specified or not in '" << f << "'" << endl; + }; + + if (mod_ops.build_alt_hard_rebuild_start_specified () != + mod_ops.build_alt_hard_rebuild_stop_specified ()) + { + bad_alt("hard"); + return 1; + } + + if (mod_ops.build_alt_soft_rebuild_start_specified () != + mod_ops.build_alt_soft_rebuild_stop_specified ()) { - cerr << "build-alt-rebuild-start and build-alt-rebuild-stop " - << "configuration options must both be either specified or not " - << "in '" << f << "'" << endl; + bad_alt("soft"); return 1; } } @@ -384,59 +602,16 @@ namespace brep } } - // Collect and report delays as separate steps not to hold database locks - // while printing to stderr. Also we need to properly order delays for - // printing. - // - // Iterate through all possible package builds creating the list of delays - // with the following sort priority: - // - // 1: toolchain name - // 2: toolchain version (descending) - // 3: configuration name - // 4: tenant - // 5: package name - // 6: package version (descending) - // - // Such ordering will allow us to group build delays by toolchain and - // configuration while printing the report. - // - struct compare_delay - { - bool - operator() (const shared_ptr<const build_delay>& x, - const shared_ptr<const build_delay>& y) const - { - if (int r = x->toolchain_name.compare (y->toolchain_name)) - return r < 0; - - if (int r = x->toolchain_version.compare (y->toolchain_version)) - return r > 0; - - if (int r = x->configuration.compare (y->configuration)) - return r < 0; - - if (int r = x->tenant.compare (y->tenant)) - return r < 0; - - if (int r = x->package_name.compare (y->package_name)) - return r < 0; - - return x->package_version.compare (y->package_version) > 0; - } - }; - - size_t reported_delay_count (0); - size_t total_delay_count (0); - - set<shared_ptr<const build_delay>, compare_delay> delays; + delay_report hard_delays_report; + delay_report soft_delays_report; + set<shared_ptr<const build_delay>, compare_delay> update_delays; { connection_ptr conn (db.connection ()); // Prepare the buildable package prepared query. // - // Query buildable packages in chunks in order not to hold locks for - // too long. + // Query buildable packages in chunks in order not to hold locks for too + // long. // using pquery = query<buildable_package>; using prep_pquery = prepared_query<buildable_package>; @@ -463,6 +638,11 @@ namespace brep // across all toolchain versions, if present, and the latest incomplete // build otherwise. // + // Why don't we pick the latest toolchain version? We don't want to + // stuck with it on the toolchain rollback. Instead we prefer the + // toolchain that built the package last and if there are none, pick the + // one for which the build task was issued last. + // using bquery = query<package_build>; using prep_bquery = prepared_query<package_build>; @@ -473,63 +653,126 @@ namespace brep bid.configuration == bquery::_ref (id.configuration) && bid.toolchain_name == bquery::_ref (id.toolchain_name)) + "ORDER BY" + - bquery::build::completion_timestamp + "DESC, " + + bquery::build::soft_timestamp + "DESC, " + bquery::build::timestamp + "DESC" + "LIMIT 1"); prep_bquery pbq ( conn->prepare_query<package_build> ("package-build-query", bq)); - duration build_timeout; + timestamp now (system_clock::now ()); - // If the build timeout is not specified explicitly, then calculate it - // as the sum of the package rebuild timeout (normal rebuild timeout if - // the alternative timeout is unspecified and the maximum of two - // otherwise) and the build result timeout. + // Calculate the build/rebuild expiration time, based on the respective + // --{soft,hard}-rebuild-timeout monitor options and the + // build-{soft,hard}-rebuild-timeout and + // build-alt-{soft,hard}-rebuild-{start,stop,timeout} brep module + // configuration options. + // + // If the --*-rebuild-timeout monitor option is zero or is not specified + // and the respective build-*-rebuild-timeout brep's configuration + // option is zero, then return timestamp_unknown to indicate 'never + // expire'. Note that this value is less than any build timestamp value, + // including timestamp_nonexistent. // - if (!ops.build_timeout_specified ()) + // NOTE: there is a similar code in mod/mod-build-task.cxx. + // + auto build_expiration = [&now, &mod_ops] ( + optional<size_t> rebuild_timeout, + const optional<pair<duration, duration>>& alt_interval, + optional<size_t> alt_timeout, + size_t normal_timeout) { - duration normal_rebuild_timeout ( - chrono::seconds (mod_ops.build_normal_rebuild_timeout ())); + duration t; - if (mod_ops.build_alt_rebuild_start_specified ()) + // If the rebuild timeout is not specified explicitly, then calculate + // it as the sum of the package rebuild timeout (normal rebuild + // timeout if the alternative timeout is unspecified and the maximum + // of two otherwise) and the build result timeout. + // + if (!rebuild_timeout) { - // Calculate the alternative rebuild timeout as the time interval - // lenght, unless it is specified explicitly. - // - if (!mod_ops.build_alt_rebuild_timeout_specified ()) + if (normal_timeout == 0) + return timestamp_unknown; + + chrono::seconds nt (normal_timeout); + + if (alt_interval) { - const duration& start (mod_ops.build_alt_rebuild_start ()); - const duration& stop (mod_ops.build_alt_rebuild_stop ()); + // Calculate the alternative timeout, unless it is specified + // explicitly. + // + if (!alt_timeout) + { + const duration& start (alt_interval->first); + const duration& stop (alt_interval->second); - // Note that if the stop time is less than the start time then the - // interval extends through the midnight. + // Note that if the stop time is less than the start time then + // the interval extends through the midnight. + // + t = start <= stop ? (stop - start) : ((24h - start) + stop); + + // If the normal rebuild time out is greater than 24 hours, then + // increase the default alternative timeout by (normal - 24h) + // (see build-alt-soft-rebuild-timeout configuration option for + // details). + // + if (nt > 24h) + t += nt - 24h; + } + else + t = chrono::seconds (*alt_timeout); + + // Take the maximum of the alternative and normal rebuild + // timeouts. // - build_timeout = start <= stop - ? stop - start - : (24h - start) + stop; + if (t < nt) + t = nt; } else - build_timeout = - chrono::seconds (mod_ops.build_alt_rebuild_timeout ()); + t = nt; - // Take the maximum of the alternative and normal rebuild timeouts. + // Summarize the rebuild and build result timeouts. // - if (build_timeout < normal_rebuild_timeout) - build_timeout = normal_rebuild_timeout; + t += chrono::seconds (mod_ops.build_result_timeout ()); } else - build_timeout = normal_rebuild_timeout; + { + if (*rebuild_timeout == 0) + return timestamp_unknown; - // Summarize the rebuild and build result timeouts. - // - build_timeout += chrono::seconds (mod_ops.build_result_timeout ()); - } - else - build_timeout = chrono::seconds (ops.build_timeout ()); + t = chrono::seconds (*rebuild_timeout); + } - timestamp now (system_clock::now ()); - timestamp build_expiration (now - build_timeout); + return now - t; + }; + + timestamp hard_rebuild_expiration ( + build_expiration ( + (ops.hard_rebuild_timeout_specified () + ? ops.hard_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_hard_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_hard_rebuild_start (), + mod_ops.build_alt_hard_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_hard_rebuild_timeout_specified () + ? mod_ops.build_alt_hard_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_hard_rebuild_timeout ())); + + timestamp soft_rebuild_expiration ( + build_expiration ( + (ops.soft_rebuild_timeout_specified () + ? ops.soft_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_soft_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_soft_rebuild_start (), + mod_ops.build_alt_soft_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_soft_rebuild_timeout_specified () + ? mod_ops.build_alt_soft_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_soft_rebuild_timeout ())); timestamp report_expiration ( now - chrono::seconds (ops.report_timeout ())); @@ -583,8 +826,12 @@ namespace brep // task have been issued recently we may still consider the // build as delayed. // - timestamp bct (b != nullptr - ? b->completion_timestamp + timestamp bht (b != nullptr + ? b->hard_timestamp + : timestamp_nonexistent); + + timestamp bst (b != nullptr + ? b->soft_timestamp : timestamp_nonexistent); // Create the delay object to record a timestamp when the @@ -605,15 +852,17 @@ namespace brep if (bp.archived && b == nullptr) continue; - // Use the build completion or build status change - // timestamp, whichever is earlier, as the build delay - // tracking starting point and fallback to the current time - // if there is no build yet. + // Use the build hard, soft, or status change timestamp (see + // the timestamps description for their ordering + // information) as the build delay tracking starting point + // and fallback to the current time if there is no build + // yet. // timestamp pts ( - b == nullptr ? now : - bct != timestamp_nonexistent && bct < b->timestamp ? bct : - b->timestamp); + b == nullptr ? now : + bht != timestamp_nonexistent ? bht : + bst != timestamp_nonexistent ? bst : + b->timestamp); d = make_shared<build_delay> (move (id.package.tenant), move (id.package.name), @@ -632,20 +881,40 @@ namespace brep // if it is not (re-)built by the expiration time. Otherwise, // consider it as delayed if it is unbuilt. // - bool delayed; + // We also don't need to report an unbuilt archived package + // twice, as both soft and hard build delays. + // + bool hard_delayed; + bool soft_delayed; if (!bp.archived) { - timestamp bts (bct != timestamp_nonexistent - ? bct + auto delayed = [&d] (timestamp bt, timestamp be) + { + timestamp t (bt != timestamp_nonexistent + ? bt : d->package_timestamp); + return t <= be; + }; - delayed = (bts <= build_expiration); + hard_delayed = delayed (bht, hard_rebuild_expiration); + soft_delayed = delayed (bst, soft_rebuild_expiration); } else - delayed = (bct == timestamp_nonexistent); + { + hard_delayed = (bst == timestamp_nonexistent); + soft_delayed = false; + } - if (delayed) + // Add hard/soft delays to the respective reports and collect + // the delay for update, if it is reported. + // + // Note that we update the delay objects persistent state + // later, after we successfully print the reports. + // + bool reported (false); + + if (hard_delayed) { // If the report timeout is zero then report the delay // unconditionally. Otherwise, report the active package @@ -655,33 +924,43 @@ namespace brep // building an archived package, so reporting its build // delays repeatedly is meaningless. // - if (ops.report_timeout () == 0 || - (!bp.archived - ? d->report_timestamp <= report_expiration - : d->report_timestamp == timestamp_nonexistent)) - { - // Note that we update the delay objects persistent state - // later, after we successfully print the report. - // - d->report_timestamp = now; - delays.insert (move (d)); + bool report ( + ops.report_timeout () == 0 || + (!bp.archived + ? d->report_hard_timestamp <= report_expiration + : d->report_hard_timestamp == timestamp_nonexistent)); - ++reported_delay_count; + if (report) + { + d->report_hard_timestamp = now; + reported = true; } - // - // In the brief mode also collect unreported delays to - // deduce and print the total number of delays per - // configuration. Mark such delays with the - // timestamp_nonexistent report timestamp. - // - else if (!ops.full_report ()) + + hard_delays_report.add_delay (d, report); + } + + if (soft_delayed) + { + bool report (ops.report_timeout () == 0 || + d->report_soft_timestamp <= report_expiration); + + if (report) { - d->report_timestamp = timestamp_nonexistent; - delays.insert (move (d)); + d->report_soft_timestamp = now; + reported = true; } - ++total_delay_count; + soft_delays_report.add_delay (d, report); } + + // If we don't consider the report timestamps for reporting + // delays, it seems natural not to update these timestamps + // either. Note that reporting all delays and still updating + // the report timestamps can be achieved by specifying the + // zero report timeout. + // + if (reported && ops.report_timeout_specified ()) + update_delays.insert (move (d)); } } } @@ -691,161 +970,48 @@ namespace brep } } - // Report package build delays, if any. + // Print delay reports, if not empty. // - if (reported_delay_count != 0) + if (!hard_delays_report.empty () || !soft_delays_report.empty ()) try { - // Print the report. - // cerr.exceptions (ostream::badbit | ostream::failbit); // Don't print the total delay count if the report timeout is zero since // all delays are reported in this case. // - bool print_total_delay_count (ops.report_timeout () != 0); - - cerr << "Package build delays (" << reported_delay_count; - - if (print_total_delay_count) - cerr << '/' << total_delay_count; - - cerr << "):" << endl; - - // Group the printed delays by toolchain and configuration. - // - const string* toolchain_name (nullptr); - const version* toolchain_version (nullptr); - const string* configuration (nullptr); - - // In the brief report mode print the number of reported/total delayed - // package builds per configuration rather than the packages themselves. - // - size_t config_reported_delay_count (0); - size_t config_total_delay_count (0); - - auto brief_config = [&configuration, - &config_reported_delay_count, - &config_total_delay_count, - print_total_delay_count] () - { - if (configuration != nullptr) - { - // Only print configurations with delays that needs to be reported. - // - if (config_reported_delay_count != 0) - { - cerr << " " << *configuration << " (" - << config_reported_delay_count; + bool total (ops.report_timeout () != 0); - if (print_total_delay_count) - cerr << '/' << config_total_delay_count; + hard_delays_report.print ("Package hard rebuild delays", + total, + ops.full_report ()); - cerr << ')' << endl; - } - - config_reported_delay_count = 0; - config_total_delay_count = 0; - } - }; - - for (shared_ptr<const build_delay> d: delays) - { - // Print the toolchain, if changed. - // - if (toolchain_name == nullptr || - d->toolchain_name != *toolchain_name || - d->toolchain_version != *toolchain_version) - { - if (!ops.full_report ()) - brief_config (); - - if (toolchain_name != nullptr) - cerr << endl; - - cerr << " " << d->toolchain_name; - - if (!d->toolchain_version.empty ()) - cerr << "/" << d->toolchain_version; - - cerr << endl; - - toolchain_name = &d->toolchain_name; - toolchain_version = &d->toolchain_version; - configuration = nullptr; - } - - // Print the configuration, if changed. - // - if (configuration == nullptr || d->configuration != *configuration) - { - if (ops.full_report ()) - { - if (configuration != nullptr) - cerr << endl; - - cerr << " " << d->configuration << endl; - } - else - brief_config (); - - configuration = &d->configuration; - } - - // Print the delayed build package in the full report mode and count - // configuration builds otherwise. - // - if (ops.full_report ()) - { - // We can potentially extend this information with the archived flag - // or the delay duration. - // - cerr << " " << d->package_name << "/" << d->package_version; - - if (!d->tenant.empty ()) - cerr << " " << d->tenant; - - cerr << endl; - } - else - { - if (d->report_timestamp != timestamp_nonexistent) - ++config_reported_delay_count; - - ++config_total_delay_count; - } - } - - if (!ops.full_report ()) - brief_config (); - - // Persist the delay report timestamps. - // - // If we don't consider the report timestamps for reporting delays, it - // seems natural not to update these timestamps either. Note that - // reporting all delays and still updating the report timestamps can be - // achieved by specifying the zero report timeout. + // Separate reports with an empty line. // - if (ops.report_timeout_specified ()) - { - transaction t (db.begin ()); + if (!hard_delays_report.empty () && !soft_delays_report.empty ()) + cerr << endl; - for (shared_ptr<const build_delay> d: delays) - { - // Only update timestamps for delays that needs to be reported. - // - if (d->report_timestamp != timestamp_nonexistent) - db.update (d); - } - - t.commit (); - } + soft_delays_report.print ("Package soft rebuild delays", + total, + ops.full_report ()); } catch (const io_error&) { return 1; // Not much we can do on stderr writing failure. } + // Persist the delay report timestamps. + // + if (!update_delays.empty ()) + { + transaction t (db.begin ()); + + for (shared_ptr<const build_delay> d: update_delays) + db.update (d); + + t.commit (); + } + return 0; } catch (const database_locked&) |