Add support for soft and hard rebuilds

11 files changed, 940 insertions, 400 deletions

diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 57d4865..9acef26 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -140,16 +140,20 @@ menu About=?about
 # build-forced-rebuild-timeout 600
 
 
-# Time to wait before considering a package for a normal rebuild. Must be
-# specified in seconds. Default is 24 hours.
+# Time to wait before considering a package for a soft rebuild (only to be
+# performed if the build environment or any of the package dependencies have
+# changed). Must be specified in seconds. The special zero value disables soft
+# rebuilds. Default is 24 hours.
 #
-# build-normal-rebuild-timeout 86400
+# build-soft-rebuild-timeout 86400
 
 
-# Alternative package rebuild timeout to use instead of the normal rebuild
-# timeout (see the build-normal-rebuild-timeout option for details) during
-# the specified time interval. Must be specified in seconds. Default is the
-# time interval length.
+# Alternative package soft rebuild timeout to use instead of the soft rebuild
+# timeout (see the build-soft-rebuild-timeout option for details) during the
+# specified time interval. Must be specified in seconds. Default is the time
+# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild
+# timeout is greater than 24 hours (thus the rebuild is only triggered within
+# the last 24 hours of the build-soft-rebuild-timeout expiration).
 #
 # The alternative rebuild timeout can be used to "pull" the rebuild window to
 # the specified time of day, for example, to optimize load and/or power
@@ -169,9 +173,25 @@ menu About=?about
 # times must both be either specified or absent. If unspecified, then no
 # alternative rebuild timeout will be used.
 #
-# build-alt-rebuild-timeout
-# build-alt-rebuild-start
-# build-alt-rebuild-stop
+# build-alt-soft-rebuild-timeout
+# build-alt-soft-rebuild-start
+# build-alt-soft-rebuild-stop
+
+
+# Time to wait before considering a package for a hard rebuild (to be
+# performed unconditionally). Must be specified in seconds. The special zero
+# value disables hard rebuilds. Default is 7 days.
+#
+# build-hard-rebuild-timeout 604800
+
+
+# Alternative package hard rebuild timeout. The semantics is the same as for
+# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout
+# option.
+#
+# build-alt-hard-rebuild-timeout
+# build-alt-hard-rebuild-start
+# build-alt-hard-rebuild-stop
 
 
 # The maximum size of the build task request manifest accepted. Note that the
diff --git a/libbrep/build.cxx b/libbrep/build.cxx
index 5f8cd71..9cbad8f 100644
--- a/libbrep/build.cxx
+++ b/libbrep/build.cxx
@@ -62,7 +62,9 @@ namespace brep
          optional<string> inr,
          optional<string> afp, optional<string> ach,
          string mnm, string msm,
-         butl::target_triplet trg)
+         butl::target_triplet trg,
+         string ccs,
+         string mcs)
       : id (package_id (move (tnt), move (pnm), pvr),
             move (cfg),
             move (tnm), tvr),
@@ -79,7 +81,9 @@ namespace brep
         agent_fingerprint (move (afp)), agent_challenge (move (ach)),
         machine (move (mnm)),
         machine_summary (move (msm)),
-        target (move (trg))
+        target (move (trg)),
+        controller_checksum (move (ccs)),
+        machine_checksum (move (mcs))
   {
   }
 
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index 151f8dc..fbab13d 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -30,7 +30,7 @@
 //
 #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 12
 
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 14, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 15, closed)
 
 // We have to keep these mappings at the global scope instead of inside
 // the brep namespace because they need to be also effective in the
@@ -194,7 +194,9 @@ namespace brep
            optional<string> agent_fingerprint,
            optional<string> agent_challenge,
            string machine, string machine_summary,
-           butl::target_triplet);
+           butl::target_triplet,
+           string controller_checksum,
+           string machine_checksum);
 
     build_id id;
 
@@ -223,15 +225,26 @@ namespace brep
     //
     optional<result_status> status;
 
-    // Time of setting the result status that can be considered as the build
-    // task completion (currently all the result_status values). Initialized
-    // with timestamp_nonexistent by default.
+    // Times of the last soft/hard completed (re)builds. Used to decide when
+    // to perform soft and hard rebuilds, respectively.
+    //
+    // The soft timestamp is updated whenever we receive a task result.
+    //
+    // The hard timestamp is updated whenever we receive a task result with
+    // a status other than skip.
+    //
+    // Also note that whenever hard_timestamp is updated, soft_timestamp is
+    // updated as well and whenever soft_timestamp is updated, timestamp is
+    // updated as well. Thus the following condition is always true:
     //
-    // Note that in the future we may not consider abort and abnormal as the
-    // task completion and, for example, proceed with automatic rebuild (the
-    // flake monitor idea).
+    // hard_timestamp <= soft_timestamp <= timestamp
     //
-    timestamp_type completion_timestamp;
+    // Note that the "completed" above means that we may analyze the task
+    // result/log and deem it as not completed and proceed with automatic
+    // rebuild (the flake monitor idea).
+    //
+    timestamp_type soft_timestamp;
+    timestamp_type hard_timestamp;
 
     // May be present only for the building state.
     //
@@ -248,6 +261,21 @@ namespace brep
     operation_results results;
     odb::section results_section;
 
+    // Checksums of entities involved in the build.
+    //
+    // Optional checksums are provided by the external entities (agent and
+    // worker). All are absent initially.
+    //
+    // Note that the agent checksum can also be absent after the hard rebuild
+    // task is issued and the worker and dependency checksums - after a failed
+    // rebuild (error result status or worse).
+    //
+    string           controller_checksum;
+    string           machine_checksum;
+    optional<string> agent_checksum;
+    optional<string> worker_checksum;
+    optional<string> dependency_checksum;
+
     // Database mapping.
     //
     #pragma db member(id) id column("")
@@ -265,15 +293,6 @@ namespace brep
     //
     #pragma db member(timestamp) index
 
-    // This is not required since 0.14.0. Note however, that just dropping
-    // this line won't pan out since this would require migration which odb is
-    // currently unable to handle automatically, advising to re-implement this
-    // change by adding a new data member with the desired default value,
-    // migrating the data, and deleting the old data member. This sounds a bit
-    // hairy, so let's keep it for now.
-    //
-    #pragma db member(completion_timestamp) default(0)
-
     #pragma db member(results) id_column("") value_column("") \
       section(results_section)
 
@@ -396,10 +415,14 @@ namespace brep
     string& toolchain_name;             // Tracks id.toolchain_name.
     upstream_version toolchain_version; // Original of id.toolchain_version.
 
-    // Time of the latest delay report. Initialized with timestamp_nonexistent
-    // by default.
+    // Times of the latest soft and hard rebuild delay reports. Initialized
+    // with timestamp_nonexistent by default.
+    //
+    // Note that both reports notify about initial build delays (at their
+    // respective time intervals).
     //
-    timestamp report_timestamp;
+    timestamp report_soft_timestamp;
+    timestamp report_hard_timestamp;
 
     // Time when the package is initially considered as buildable for this
     // configuration and toolchain. It is used to track the build delay if the
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 0ca362a..6a2301b 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,22 @@
 <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+  <changeset version="15">
+    <alter-table name="build">
+      <add-column name="soft_timestamp" type="BIGINT" null="false"/>
+      <add-column name="hard_timestamp" type="BIGINT" null="false"/>
+      <add-column name="controller_checksum" type="TEXT" null="false"/>
+      <add-column name="machine_checksum" type="TEXT" null="false"/>
+      <add-column name="agent_checksum" type="TEXT" null="true"/>
+      <add-column name="worker_checksum" type="TEXT" null="true"/>
+      <add-column name="dependency_checksum" type="TEXT" null="true"/>
+      <drop-column name="completion_timestamp"/>
+    </alter-table>
+    <alter-table name="build_delay">
+      <add-column name="report_soft_timestamp" type="BIGINT" null="false"/>
+      <add-column name="report_hard_timestamp" type="BIGINT" null="false"/>
+      <drop-column name="report_timestamp"/>
+    </alter-table>
+  </changeset>
+
   <changeset version="14"/>
 
   <changeset version="13">
diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx
index 1093b7c..468d411 100644
--- a/migrate/migrate.cxx
+++ b/migrate/migrate.cxx
@@ -14,6 +14,8 @@
 
 #include <libbutl/pager.hxx>
 
+#include <libbrep/build.hxx>
+#include <libbrep/build-odb.hxx>
 #include <libbrep/package.hxx>
 #include <libbrep/package-odb.hxx>
 #include <libbrep/database-lock.hxx>
@@ -224,6 +226,36 @@ package_migrate_v20 ([] (database& db)
 });
 #endif
 
+// Register the data migration functions for the build database schema.
+//
+template <schema_version v>
+using build_migration_entry_base =
+  data_migration_entry<v, LIBBREP_BUILD_SCHEMA_VERSION_BASE>;
+
+template <schema_version v>
+struct build_migration_entry: build_migration_entry_base<v>
+{
+  build_migration_entry (void (*f) (database& db))
+      : build_migration_entry_base<v> (f, "build") {}
+};
+
+static const build_migration_entry<15>
+build_migrate_v15 ([] (database& db)
+{
+  // Setting proper checksums here feels a bit hairy. Let's assign them
+  // naturally on the first rebuild.
+  //
+  db.execute ("UPDATE build SET "
+              "soft_timestamp = completion_timestamp, "
+              "hard_timestamp = completion_timestamp, "
+              "controller_checksum = '', "
+              "machine_checksum = ''");
+
+  db.execute ("UPDATE build_delay SET "
+              "report_soft_timestamp = report_timestamp, "
+              "report_hard_timestamp = report_timestamp");
+});
+
 // main() function
 //
 int
diff --git a/mod/buildfile b/mod/buildfile
index ff9cd60..58a3caf 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
   cli.options += --std c++11 -I $src_root --include-with-brackets    \
 --include-prefix mod --guard-prefix MOD --generate-specifier         \
 --cxx-prologue "#include <mod/types-parsers.hxx>"                    \
---cli-namespace brep::cli --generate-file-scanner --option-length 45 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 46 \
 --generate-modifier --generate-description --option-prefix ""
 
   # Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 3ae9f0f..1c46fc1 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -287,7 +287,6 @@ handle (request& rq, response&)
   //
   shared_ptr<build> bld;
 
-  optional<result_status> prev_status;
   bool build_notify (false);
   bool unforced (true);
 
@@ -382,6 +381,58 @@ handle (request& rq, response&)
 
       if (auth)
       {
+        // Verify the result status/checksums.
+        //
+        // Specifically, if the result status is skip, then it can only be in
+        // response to the soft rebuild task (all checksums are present in the
+        // build object) and the result checksums must match the build object
+        // checksums. On verification failure respond with the bad request
+        // HTTP code (400).
+        //
+        if (rqm.result.status == result_status::skip)
+        {
+          if (!b->agent_checksum  ||
+              !b->worker_checksum ||
+              !b->dependency_checksum)
+            throw invalid_request (400, "unexpected skip result status");
+
+          // Can only be absent for initial build, in which case the checksums
+          // are also absent and we would end up with the above 400 response.
+          //
+          assert (b->status);
+
+          // Verify that the result checksum matches the build checksum and
+          // throw invalid_request(400) if that's not the case.
+          //
+          auto verify = [] (const string& build_checksum,
+                            const optional<string>& result_checksum,
+                            const char* what)
+          {
+            if (!result_checksum)
+              throw invalid_request (
+                400,
+                string (what) +
+                " checksum is expected for skip result status");
+
+            if (*result_checksum != build_checksum)
+              throw invalid_request (
+                400,
+                string (what) + " checksum '" + build_checksum  +
+                "' is expected instead of '" + *result_checksum +
+                "' for skip result status");
+          };
+
+          verify (*b->agent_checksum, rqm.agent_checksum, "agent");
+
+          verify (*b->worker_checksum,
+                  rqm.result.worker_checksum,
+                  "worker");
+
+          verify (*b->dependency_checksum,
+                  rqm.result.dependency_checksum,
+                  "dependency");
+        }
+
         unforced = b->force == force_state::unforced;
 
         // Don't send email to the build-email address for the
@@ -392,10 +443,7 @@ handle (request& rq, response&)
                          *b->status == rqm.result.status             &&
                          unforced);
 
-        prev_status = move (b->status);
-
         b->state  = build_state::built;
-        b->status = rqm.result.status;
         b->force  = force_state::unforced;
 
         // Cleanup the interactive build login information.
@@ -407,22 +455,43 @@ handle (request& rq, response&)
         b->agent_fingerprint = nullopt;
         b->agent_challenge = nullopt;
 
-        // Mark the section as loaded, so results are updated.
-        //
-        b->results_section.load ();
-        b->results = move (rqm.result.results);
-
         b->timestamp = system_clock::now ();
-        b->completion_timestamp = b->timestamp;
+        b->soft_timestamp = b->timestamp;
+
+        // If the result status is other than skip, then save the status,
+        // results, and checksums and update the hard timestamp.
+        //
+        if (rqm.result.status != result_status::skip)
+        {
+          b->status = rqm.result.status;
+          b->hard_timestamp = b->soft_timestamp;
+
+          // Mark the section as loaded, so results are updated.
+          //
+          b->results_section.load ();
+          b->results = move (rqm.result.results);
+
+          // Save the checksums.
+          //
+          b->agent_checksum      = move (rqm.agent_checksum);
+          b->worker_checksum     = move (rqm.result.worker_checksum);
+          b->dependency_checksum = move (rqm.result.dependency_checksum);
+        }
 
         build_db_->update (b);
 
-        shared_ptr<build_package> p (
-          build_db_->load<build_package> (b->id.package));
+        // Don't send the build notification email if the task result is
+        // `skip`, the configuration is hidden, or is now excluded by the
+        // package.
+        //
+        if (rqm.result.status != result_status::skip && belongs (*cfg, "all"))
+        {
+          shared_ptr<build_package> p (
+            build_db_->load<build_package> (b->id.package));
 
-        if (belongs (*cfg, "all") &&
-            !exclude (p->builds, p->constraints, *cfg))
-          bld = move (b);
+          if (!exclude (p->builds, p->constraints, *cfg))
+            bld = move (b);
+        }
       }
     }
 
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 8656f5e..22d0110 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -61,13 +61,23 @@ init (scanner& s)
 
   if (options_->build_config_specified ())
   {
-    // Verify that build-alt-rebuild-{start,stop} are both either specified or
-    // not.
+    // Verify that build-alt-*-rebuild-{start,stop} are both either specified
+    // or not.
     //
-    if (options_->build_alt_rebuild_start_specified () !=
-        options_->build_alt_rebuild_stop_specified ())
-      fail << "build-alt-rebuild-start and build-alt-rebuild-stop "
-           << "configuration options must both be either specified or not";
+    auto bad_alt = [&fail] (const char* what)
+    {
+      fail << "build-alt-" << what << "-rebuild-start and build-alt-" << what
+           << "-rebuild-stop configuration options must both be either "
+           << "specified or not";
+    };
+
+    if (options_->build_alt_soft_rebuild_start_specified () !=
+        options_->build_alt_soft_rebuild_stop_specified ())
+      bad_alt ("soft");
+
+    if (options_->build_alt_hard_rebuild_start_specified () !=
+        options_->build_alt_hard_rebuild_stop_specified ())
+      bad_alt ("hard");
 
     database_module::init (*options_, options_->build_db_retry ());
 
@@ -277,22 +287,25 @@ handle (request& rq, response& rs)
                           move (fps),
                           move (p->requirements),
                           move (tests),
+                          move (b->dependency_checksum),
                           cm.machine->name,
                           cm.config->target,
                           cm.config->environment,
                           cm.config->args,
                           belongs (*cm.config, module_pkg ? "build2" : "host"),
                           cm.config->warning_regexes,
-                          move (t->interactive));
+                          move (t->interactive),
+                          move (b->worker_checksum));
 
       return task_response_manifest (move (session),
                                      move (b->agent_challenge),
                                      move (result_url),
+                                     move (b->agent_checksum),
                                      move (task));
     };
 
-    // Calculate the build (building state) or rebuild (built state) expiration
-    // time for package configurations
+    // Calculate the build (building state) or rebuild (built state)
+    // expiration time for package configurations.
     //
     timestamp now (system_clock::now ());
 
@@ -316,44 +329,95 @@ handle (request& rq, response& rs)
     timestamp forced_rebuild_expiration (
       expiration (options_->build_forced_rebuild_timeout ()));
 
-    timestamp normal_rebuild_expiration;
-
-    if (options_->build_alt_rebuild_start_specified ())
+    // Calculate the soft/hard rebuild expiration time, based on the
+    // respective build-{soft,hard}-rebuild-timeout and
+    // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration
+    // options.
+    //
+    // If normal_timeout is zero, then return timestamp_unknown to indicate
+    // 'never expire'. Note that this value is less than any build timestamp
+    // value, including timestamp_nonexistent.
+    //
+    // NOTE: there is a similar code in monitor/monitor.cxx.
+    //
+    auto build_expiration = [&now] (
+      const optional<pair<duration, duration>>& alt_interval,
+      optional<size_t> alt_timeout,
+      size_t normal_timeout)
     {
-      const duration& start (options_->build_alt_rebuild_start ());
-      const duration& stop  (options_->build_alt_rebuild_stop ());
+      if (normal_timeout == 0)
+        return timestamp_unknown;
 
-      duration dt (daytime (now));
+      timestamp r;
+      chrono::seconds nt (normal_timeout);
 
-      // Note that if the stop time is less than the start time then the
-      // interval extends through the midnight.
-      //
-      bool alt_timeout (start <= stop
-                        ? dt >= start && dt < stop
-                        : dt >= start || dt < stop);
-
-      // If we out of the alternative rebuild timeout interval, then fall back
-      // to using the normal rebuild timeout.
-      //
-      if (alt_timeout)
+      if (alt_interval)
       {
-        if (!options_->build_alt_rebuild_timeout_specified ())
+        const duration& start (alt_interval->first);
+        const duration& stop  (alt_interval->second);
+
+        duration dt (daytime (now));
+
+        // Note that if the stop time is less than the start time then the
+        // interval extends through the midnight.
+        //
+        bool use_alt_timeout (start <= stop
+                              ? dt >= start && dt < stop
+                              : dt >= start || dt < stop);
+
+        // If we out of the alternative rebuild timeout interval, then fall
+        // back to using the normal rebuild timeout.
+        //
+        if (use_alt_timeout)
         {
-          duration interval_len (start <= stop
-                                 ? stop - start
-                                 : (24h - start) + stop);
+          // Calculate the alternative timeout, unless it is specified
+          // explicitly.
+          //
+          duration t;
+
+          if (!alt_timeout)
+          {
+            t = start <= stop ? (stop - start) : ((24h - start) + stop);
 
-          normal_rebuild_expiration = now - interval_len;
+            // If the normal rebuild timeout is greater than 24 hours, then
+            // increase the default alternative timeout by (normal - 24h) (see
+            // build-alt-soft-rebuild-timeout configuration option for
+            // details).
+            //
+            if (nt > 24h)
+              t += nt - 24h;
+          }
+          else
+            t = chrono::seconds (*alt_timeout);
+
+          r = now - t;
         }
-        else
-          normal_rebuild_expiration =
-            expiration (options_->build_alt_rebuild_timeout ());
       }
-    }
 
-    if (normal_rebuild_expiration == timestamp_nonexistent)
-      normal_rebuild_expiration =
-        expiration (options_->build_normal_rebuild_timeout ());
+      return r != timestamp_nonexistent ? r : (now - nt);
+    };
+
+    timestamp soft_rebuild_expiration (
+      build_expiration (
+        (options_->build_alt_soft_rebuild_start_specified ()
+         ? make_pair (options_->build_alt_soft_rebuild_start (),
+                      options_->build_alt_soft_rebuild_stop ())
+         : optional<pair<duration, duration>> ()),
+        (options_->build_alt_soft_rebuild_timeout_specified ()
+         ? options_->build_alt_soft_rebuild_timeout ()
+         : optional<size_t> ()),
+        options_->build_soft_rebuild_timeout ()));
+
+    timestamp hard_rebuild_expiration (
+      build_expiration (
+        (options_->build_alt_hard_rebuild_start_specified ()
+         ? make_pair (options_->build_alt_hard_rebuild_start (),
+                      options_->build_alt_hard_rebuild_stop ())
+         : optional<pair<duration, duration>> ()),
+        (options_->build_alt_hard_rebuild_timeout_specified ()
+         ? options_->build_alt_hard_rebuild_timeout ()
+         : optional<size_t> ()),
+        options_->build_hard_rebuild_timeout ()));
 
     // Return the challenge (nonce) if brep is configured to authenticate bbot
     // agents. Return nullopt otherwise.
@@ -555,6 +619,68 @@ handle (request& rq, response& rs)
     prep_bld_query bld_prep_query (
       conn->prepare_query<build> ("mod-build-task-build-query", bq));
 
+    // Return true if a package needs to be rebuilt.
+    //
+    auto needs_rebuild = [&forced_rebuild_expiration,
+                          &soft_rebuild_expiration,
+                          &hard_rebuild_expiration] (const build& b)
+    {
+      assert (b.state == build_state::built);
+
+      return (b.force == force_state::forced &&
+              b.soft_timestamp <= forced_rebuild_expiration) ||
+             b.soft_timestamp <= soft_rebuild_expiration     ||
+             b.hard_timestamp <= hard_rebuild_expiration;
+    };
+
+    // Convert a build to the hard rebuild, resetting the agent checksum and
+    // dropping the previous build task result.
+    //
+    // Note that since the checksums are hierarchical, the agent checksum
+    // reset will trigger resets of the "subordinate" checksums up to the
+    // dependency checksum and so the package will be rebuilt.
+    //
+    // Also note that there is no sense to keep the build task result since we
+    // don't accept the skip result for the hard rebuild task. We, however,
+    // keep the status intact (see below for the reasoning).
+    //
+    auto convert_to_hard = [] (const shared_ptr<build>& b)
+    {
+      b->agent_checksum = nullopt;
+
+      // Mark the section as loaded, so results are updated.
+      //
+      b->results_section.load ();
+      b->results.clear ();
+    };
+
+    // Return SHA256 checksum of the controller logic and the configuration
+    // target, environment, arguments, and warning-detecting regular
+    // expressions.
+    //
+    auto controller_checksum = [] (const build_config& c)
+    {
+      sha256 cs ("1"); // Hash the logic version.
+
+      cs.append (c.target.string ());
+      cs.append (c.environment ? *c.environment : "");
+
+      for (const string& a: c.args)
+        cs.append (a);
+
+      for (const string& re: c.warning_regexes)
+        cs.append (re);
+
+      return string (cs.string ());
+    };
+
+    // Return the machine id as a machine checksum.
+    //
+    auto machine_checksum = [] (const machine_header_manifest& m)
+    {
+      return m.id;
+    };
+
     while (tsm.session.empty ())
     {
       transaction t (conn->begin ());
@@ -604,9 +730,7 @@ handle (request& rq, response& rs)
           {
             assert (i->force != force_state::forcing);
 
-            if (i->timestamp <= (i->force == force_state::forced
-                                 ? forced_rebuild_expiration
-                                 : normal_rebuild_expiration))
+            if (needs_rebuild (*i))
               rebuilds.emplace_back (i.load ());
           }
         }
@@ -666,14 +790,15 @@ handle (request& rq, response& rs)
                                       move (cl),
                                       mh.name,
                                       move (mh.summary),
-                                      cm.config->target);
+                                      cm.config->target,
+                                      controller_checksum (*cm.config),
+                                      machine_checksum (*cm.machine));
 
               build_db_->persist (b);
             }
             else
             {
-              // The package configuration is in the building state, and there
-              // are no results.
+              // The package configuration is in the building state.
               //
               // Note that in both cases we keep the status intact to be able
               // to compare it with the final one in the result request
@@ -681,12 +806,7 @@ handle (request& rq, response& rs)
               // email. The same is true for the forced flag (in the sense
               // that we don't set the force state to unforced).
               //
-              // Load the section to assert the above statement.
-              //
-              build_db_->load (*b, b->results_section);
-
-              assert (b->state == build_state::building &&
-                      b->results.empty ());
+              assert (b->state == build_state::building);
 
               b->state = build_state::building;
               b->interactive = move (login);
@@ -703,6 +823,22 @@ handle (request& rq, response& rs)
               b->machine = mh.name;
               b->machine_summary = move (mh.summary);
               b->target = cm.config->target;
+
+              string ccs (controller_checksum (*cm.config));
+              string mcs (machine_checksum (*cm.machine));
+
+              // Issue the hard rebuild if it is forced or the configuration
+              // or machine has changed.
+              //
+              if (b->hard_timestamp <= hard_rebuild_expiration ||
+                  b->force == force_state::forced              ||
+                  b->controller_checksum != ccs                ||
+                  b->machine_checksum != mcs)
+                convert_to_hard (b);
+
+              b->controller_checksum = move (ccs);
+              b->machine_checksum    = move (mcs);
+
               b->timestamp = system_clock::now ();
 
               build_db_->update (b);
@@ -770,10 +906,9 @@ handle (request& rq, response& rs)
 
           b = build_db_->find<build> (b->id);
 
-          if (b != nullptr && b->state == build_state::built &&
-              b->timestamp <= (b->force == force_state::forced
-                               ? forced_rebuild_expiration
-                               : normal_rebuild_expiration))
+          if (b != nullptr                   &&
+              b->state == build_state::built &&
+              needs_rebuild (*b))
           {
             auto i (cfg_machines.find (b->id.configuration.c_str ()));
 
@@ -828,10 +963,23 @@ handle (request& rq, response& rs)
 
               b->target = cm.config->target;
 
-              // Mark the section as loaded, so results are updated.
+              // Issue the hard rebuild if the timeout expired, rebuild is
+              // forced, or the configuration or machine has changed.
               //
-              b->results_section.load ();
-              b->results.clear ();
+              // Note that we never reset the build status (see above for the
+              // reasoning).
+              //
+              string ccs (controller_checksum (*cm.config));
+              string mcs (machine_checksum (*cm.machine));
+
+              if (b->hard_timestamp <= hard_rebuild_expiration ||
+                  b->force == force_state::forced              ||
+                  b->controller_checksum != ccs                ||
+                  b->machine_checksum != mcs)
+                convert_to_hard (b);
+
+              b->controller_checksum = move (ccs);
+              b->machine_checksum    = move (mcs);
 
               b->timestamp = system_clock::now ();
 
diff --git a/mod/module.cli b/mod/module.cli
index c95c20c..c2dce5b 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -195,45 +195,87 @@ namespace brep
          be specified in seconds. Default is 10 minutes."
       }
 
-      size_t build-normal-rebuild-timeout = 86400
+      size_t build-soft-rebuild-timeout = 86400
       {
         "<seconds>",
-        "Time to wait before considering a package for a normal rebuild. Must
-         be specified in seconds. Default is 24 hours."
+        "Time to wait before considering a package for a soft rebuild (only to
+         be performed if the build environment or any of the package
+         dependencies have changed). Must be specified in seconds. The special
+         zero value disables soft rebuilds. Default is 24 hours"
       }
 
-      size_t build-alt-rebuild-timeout
+      size_t build-alt-soft-rebuild-timeout
       {
         "<seconds>",
-        "Alternative package rebuild timeout to use instead of the normal
-         rebuild timeout (see \cb{build-normal-rebuild-timeout} for details)
+        "Alternative package soft rebuild timeout to use instead of the soft
+         rebuild timeout (see \cb{build-soft-rebuild-timeout} for details)
          during the time interval specified with the
-         \cb{build-alt-rebuild-start} and \cb{build-alt-rebuild-stop} options.
-         Must be specified in seconds. Default is the time interval length."
+         \cb{build-alt-soft-rebuild-start} and
+         \cb{build-alt-soft-rebuild-stop} options. Must be specified in
+         seconds. Default is the time interval length plus
+         \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is
+         greater than 24 hours (thus the rebuild is only triggered within the
+         last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)."
       }
 
-      duration build-alt-rebuild-start
+      duration build-alt-soft-rebuild-start
       {
         "<hours>:<minutes>",
-        "The start time of the alternative package rebuild timeout (see
-         \cb{build-alt-rebuild-timeout} for details). Must be specified as
-         a time of day in the local timezone. The \cb{build-alt-rebuild-start}
-         and \cb{build-alt-rebuild-stop} options must be either both specified
-         or absent. If unspecified, then no alternative rebuild timeout will
-         be used."
+        "The start time of the alternative package soft rebuild timeout (see
+         \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+         as a time of day in the local timezone. The
+         \cb{build-alt-soft-rebuild-start} and
+         \cb{build-alt-soft-rebuild-stop} options must be either both
+         specified or absent. If unspecified, then no alternative rebuild
+         timeout will be used."
       }
 
-      duration build-alt-rebuild-stop
+      duration build-alt-soft-rebuild-stop
       {
         "<hours>:<minutes>",
-        "The end time of the alternative package rebuild timeout (see
-         \cb{build-alt-rebuild-timeout} for details). Must be specified as
-         a time of day in the local timezone. If it is less than the
-         \cb{build-alt-rebuild-start} option value, then the time interval
-         extends through midnight. The \cb{build-alt-rebuild-start} and
-         \cb{build-alt-rebuild-stop} options must be either both specified or
-         absent. If unspecified, then no alternative rebuild timeout will be
-         used."
+        "The end time of the alternative package soft rebuild timeout (see
+         \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+         as a time of day in the local timezone. If it is less than the
+         \cb{build-alt-soft-rebuild-start} option value, then the time
+         interval extends through midnight. The
+         \cb{build-alt-soft-rebuild-start} and
+         \cb{build-alt-soft-rebuild-stop} options must be either both
+         specified or absent. If unspecified, then no alternative rebuild
+         timeout will be used."
+      }
+
+      size_t build-hard-rebuild-timeout = 604800
+      {
+        "<seconds>",
+        "Time to wait before considering a package for a hard rebuild (to be
+         performed unconditionally). Must be specified in seconds. The special
+         zero value disables hard rebuilds. Default is 7 days."
+      }
+
+      size_t build-alt-hard-rebuild-timeout
+      {
+        "<seconds>",
+        "Alternative package hard rebuild timeout. The semantics is the
+         same as for the \cb{build-alt-soft-rebuild-timeout} option but
+         for the \cb{build-hard-rebuild-timeout} option."
+      }
+
+      duration build-alt-hard-rebuild-start
+      {
+        "<hours>:<minutes>",
+        "The start time of the alternative package hard rebuild timeout (see
+         \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+         the same as for the \cb{build-alt-soft-rebuild-start} option but
+         for the \cb{build-hard-rebuild-timeout} option."
+      }
+
+      duration build-alt-hard-rebuild-stop
+      {
+        "<hours>:<minutes>",
+        "The end time of the alternative package hard rebuild timeout (see
+         \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+         the same as for the \cb{build-alt-soft-rebuild-stop} option but
+         for the \cb{build-hard-rebuild-timeout} option."
       }
     };
 
diff --git a/monitor/monitor.cli b/monitor/monitor.cli
index edfc004..3a58a1d 100644
--- a/monitor/monitor.cli
+++ b/monitor/monitor.cli
@@ -60,18 +60,26 @@ namespace brep
     {
       "\h|OPTIONS|"
 
-      std::size_t --build-timeout
+      std::size_t --soft-rebuild-timeout
       {
         "<seconds>",
-        "Time to wait (in seconds) before considering a package build as
+        "Time to wait (in seconds) before considering a package soft (re)build as
          delayed. If unspecified, it is the sum of the package rebuild timeout
-         (normal rebuild timeout if the alternative timeout is unspecified and
-         the maximum of two otherwise) and the build result timeout (see the
-         \cb{build-normal-rebuild-timeout}, \cb{build-alt-rebuild-*}, and
-         \cb{build-result-timeout} \c{brep} module configuration options
-         for details).
-
-         Note that a package that was not built before it was archived is
+         (soft rebuild timeout if the alternative timeout is unspecified and
+         the maximum of two otherwise) and the build result timeout (see
+         the \cb{build-soft-rebuild-timeout}, \cb{build-alt-soft-rebuild-*},
+         and \cb{build-result-timeout} \cb{brep} module configuration options
+         for details). The special zero value disables monitoring of soft
+         rebuilds.
+
+         Note that if both soft and hard rebuilds are disabled in the
+         \cb{brep} module configuration, then \cb{brep-monitor} is unable to
+         come up with a reasonable build timeout on its own. In this case, to
+         monitor the initial package build delays, you may need to specify
+         either \cb{--soft-rebuild-timeout} or \cb{--hard-rebuild-timeout}
+         explicitly.
+
+         Also note that a package that was not built before it was archived is
          always considered as delayed. However, to distinguish this case from
          a situation where a package was archived before a configuration have
          been added, \cb{brep-monitor} needs to observe the package as
@@ -81,6 +89,16 @@ namespace brep
          timeout."
       }
 
+      std::size_t --hard-rebuild-timeout
+      {
+        "<seconds>",
+        "Time to wait (in seconds) before considering a package hard (re)build
+         as delayed. If unspecified, it is calculated in the same way as for
+         \cb{--soft-rebuild-timeout} but using the
+         \cb{build-hard-rebuild-timeout} and \cb{build-alt-hard-rebuild-*}
+         \cb{brep} module configuration options."
+      }
+
       std::size_t --report-timeout
       {
         "<seconds>",
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index 5c4d451..7c20efe 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -41,6 +41,212 @@ namespace brep
   //
   struct failed {};
 
+  // We will collect and report build delays as separate steps not to hold
+  // database locks while printing to stderr. Also we need to order delays
+  // properly, so while printing reports we could group delays by toolchain
+  // and configuration.
+  //
+  // To achieve that, we will iterate through all possible package builds
+  // creating the list of delays with the following sort priority:
+  //
+  // 1: toolchain name
+  // 2: toolchain version (descending)
+  // 3: configuration name
+  // 4: tenant
+  // 5: package name
+  // 6: package version (descending)
+  //
+  struct compare_delay
+  {
+    bool
+    operator() (const shared_ptr<const build_delay>& x,
+                const shared_ptr<const build_delay>& y) const
+    {
+      if (int r = x->toolchain_name.compare (y->toolchain_name))
+        return r < 0;
+
+      if (int r = x->toolchain_version.compare (y->toolchain_version))
+        return r > 0;
+
+      if (int r = x->configuration.compare (y->configuration))
+        return r < 0;
+
+      if (int r = x->tenant.compare (y->tenant))
+        return r < 0;
+
+      if (int r = x->package_name.compare (y->package_name))
+        return r < 0;
+
+      return x->package_version.compare (y->package_version) > 0;
+    }
+  };
+
+  // The ordered list of delays to report.
+  //
+  class delay_report
+  {
+  public:
+    // Note that in the brief mode we also need to print the total number of
+    // delays (reported or not) per configuration. Thus, we add all delays to
+    // the report object, marking them if we need to report them or not.
+    //
+    void
+    add_delay (shared_ptr<build_delay>, bool report);
+
+    bool
+    empty () const
+    {
+      return reported_delay_count_ == 0;
+    }
+
+    // In the brief mode (if full is false) print the number of reported/total
+    // (if total is true) delayed package builds per configuration rather than
+    // the packages themselves.
+    //
+    void
+    print (const char* header, bool total, bool full) const;
+
+  private:
+    // Maps delays to the report flag.
+    //
+    map<shared_ptr<const build_delay>, bool, compare_delay> delays_;
+    size_t reported_delay_count_ = 0;
+  };
+
+  void delay_report::
+  add_delay (shared_ptr<build_delay> delay, bool report)
+  {
+    delays_.emplace (move (delay), report);
+
+    if (report)
+      ++reported_delay_count_;
+  }
+
+  void delay_report::
+  print (const char* header, bool total, bool full) const
+  {
+    if (empty ())
+      return;
+
+    cerr << header << " (" << reported_delay_count_;
+
+    if (total)
+      cerr << '/' << delays_.size ();
+
+    cerr << "):" << endl;
+
+    // Group the printed delays by toolchain and configuration.
+    //
+    const string*  toolchain_name    (nullptr);
+    const version* toolchain_version (nullptr);
+    const string*  configuration     (nullptr);
+
+    size_t config_reported_delay_count (0);
+    size_t config_total_delay_count (0);
+
+    auto brief_config = [&configuration,
+                         &config_reported_delay_count,
+                         &config_total_delay_count,
+                         total] ()
+    {
+      if (configuration != nullptr)
+      {
+        // Only print configurations with delays that needs to be reported.
+        //
+        if (config_reported_delay_count != 0)
+        {
+          cerr << "    " << *configuration << " ("
+               << config_reported_delay_count;
+
+          if (total)
+            cerr << '/' << config_total_delay_count;
+
+          cerr << ')' << endl;
+        }
+
+        config_reported_delay_count = 0;
+        config_total_delay_count = 0;
+      }
+    };
+
+    for (const auto& dr: delays_)
+    {
+      bool report (dr.second);
+
+      if (full && !report)
+        continue;
+
+      const shared_ptr<const build_delay>& d (dr.first);
+
+      // Print the toolchain, if changed.
+      //
+      if (toolchain_name == nullptr            ||
+          d->toolchain_name != *toolchain_name ||
+          d->toolchain_version != *toolchain_version)
+      {
+        if (!full)
+          brief_config ();
+
+        if (toolchain_name != nullptr)
+          cerr << endl;
+
+        cerr << "  " << d->toolchain_name;
+
+        if (!d->toolchain_version.empty ())
+          cerr << "/" << d->toolchain_version;
+
+        cerr << endl;
+
+        toolchain_name    = &d->toolchain_name;
+        toolchain_version = &d->toolchain_version;
+        configuration     = nullptr;
+      }
+
+      // Print the configuration, if changed.
+      //
+      if (configuration == nullptr || d->configuration != *configuration)
+      {
+        if (full)
+        {
+          if (configuration != nullptr)
+            cerr << endl;
+
+          cerr << "    " << d->configuration << endl;
+        }
+        else
+          brief_config ();
+
+        configuration = &d->configuration;
+      }
+
+      // Print the delayed build package in the full report mode and count
+      // configuration builds otherwise.
+      //
+      if (full)
+      {
+        // We can potentially extend this information with the archived flag
+        // or the delay duration.
+        //
+        cerr << "      " << d->package_name << "/" << d->package_version;
+
+        if (!d->tenant.empty ())
+          cerr << " " << d->tenant;
+
+        cerr << endl;
+      }
+      else
+      {
+        if (report)
+          ++config_reported_delay_count;
+
+        ++config_total_delay_count;
+      }
+    }
+
+    if (!full)
+      brief_config ();
+  }
+
   static const char* help_info (
     "  info: run 'brep-monitor --help' for more information");
 
@@ -141,12 +347,24 @@ namespace brep
         return 1;
       }
 
-      if (mod_ops.build_alt_rebuild_start_specified () !=
-          mod_ops.build_alt_rebuild_stop_specified ())
+      auto bad_alt = [&f] (const char* what)
+      {
+        cerr << "build-alt-" << what << "-rebuild-start and build-alt-"
+             << what << "-rebuild-stop configuration options must both be "
+             << "either specified or not in '" << f << "'" << endl;
+      };
+
+      if (mod_ops.build_alt_hard_rebuild_start_specified () !=
+          mod_ops.build_alt_hard_rebuild_stop_specified ())
+      {
+        bad_alt("hard");
+        return 1;
+      }
+
+      if (mod_ops.build_alt_soft_rebuild_start_specified () !=
+          mod_ops.build_alt_soft_rebuild_stop_specified ())
       {
-        cerr << "build-alt-rebuild-start and build-alt-rebuild-stop "
-             << "configuration options must both be either specified or not "
-             << "in '" << f << "'" << endl;
+        bad_alt("soft");
         return 1;
       }
     }
@@ -384,59 +602,16 @@ namespace brep
       }
     }
 
-    // Collect and report delays as separate steps not to hold database locks
-    // while printing to stderr. Also we need to properly order delays for
-    // printing.
-    //
-    // Iterate through all possible package builds creating the list of delays
-    // with the following sort priority:
-    //
-    // 1: toolchain name
-    // 2: toolchain version (descending)
-    // 3: configuration name
-    // 4: tenant
-    // 5: package name
-    // 6: package version (descending)
-    //
-    // Such ordering will allow us to group build delays by toolchain and
-    // configuration while printing the report.
-    //
-    struct compare_delay
-    {
-      bool
-      operator() (const shared_ptr<const build_delay>& x,
-                  const shared_ptr<const build_delay>& y) const
-      {
-        if (int r = x->toolchain_name.compare (y->toolchain_name))
-          return r < 0;
-
-        if (int r = x->toolchain_version.compare (y->toolchain_version))
-          return r > 0;
-
-        if (int r = x->configuration.compare (y->configuration))
-          return r < 0;
-
-        if (int r = x->tenant.compare (y->tenant))
-          return r < 0;
-
-        if (int r = x->package_name.compare (y->package_name))
-          return r < 0;
-
-        return x->package_version.compare (y->package_version) > 0;
-      }
-    };
-
-    size_t reported_delay_count (0);
-    size_t total_delay_count (0);
-
-    set<shared_ptr<const build_delay>, compare_delay> delays;
+    delay_report hard_delays_report;
+    delay_report soft_delays_report;
+    set<shared_ptr<const build_delay>, compare_delay> update_delays;
     {
       connection_ptr conn (db.connection ());
 
       // Prepare the buildable package prepared query.
       //
-      // Query buildable packages in chunks in order not to hold locks for
-      // too long.
+      // Query buildable packages in chunks in order not to hold locks for too
+      // long.
       //
       using pquery = query<buildable_package>;
       using prep_pquery = prepared_query<buildable_package>;
@@ -463,6 +638,11 @@ namespace brep
       // across all toolchain versions, if present, and the latest incomplete
       // build otherwise.
       //
+      // Why don't we pick the latest toolchain version? We don't want to
+      // stuck with it on the toolchain rollback. Instead we prefer the
+      // toolchain that built the package last and if there are none, pick the
+      // one for which the build task was issued last.
+      //
       using bquery = query<package_build>;
       using prep_bquery = prepared_query<package_build>;
 
@@ -473,63 +653,126 @@ namespace brep
                   bid.configuration == bquery::_ref (id.configuration) &&
                   bid.toolchain_name == bquery::_ref (id.toolchain_name)) +
                  "ORDER BY"                                               +
-                 bquery::build::completion_timestamp + "DESC, "           +
+                 bquery::build::soft_timestamp + "DESC, "                 +
                  bquery::build::timestamp + "DESC"                        +
                  "LIMIT 1");
 
       prep_bquery pbq (
         conn->prepare_query<package_build> ("package-build-query", bq));
 
-      duration build_timeout;
+      timestamp now (system_clock::now ());
 
-      // If the build timeout is not specified explicitly, then calculate it
-      // as the sum of the package rebuild timeout (normal rebuild timeout if
-      // the alternative timeout is unspecified and the maximum of two
-      // otherwise) and the build result timeout.
+      // Calculate the build/rebuild expiration time, based on the respective
+      // --{soft,hard}-rebuild-timeout monitor options and the
+      // build-{soft,hard}-rebuild-timeout and
+      // build-alt-{soft,hard}-rebuild-{start,stop,timeout} brep module
+      // configuration options.
+      //
+      // If the --*-rebuild-timeout monitor option is zero or is not specified
+      // and the respective build-*-rebuild-timeout brep's configuration
+      // option is zero, then return timestamp_unknown to indicate 'never
+      // expire'. Note that this value is less than any build timestamp value,
+      // including timestamp_nonexistent.
       //
-      if (!ops.build_timeout_specified ())
+      // NOTE: there is a similar code in mod/mod-build-task.cxx.
+      //
+      auto build_expiration = [&now, &mod_ops] (
+        optional<size_t> rebuild_timeout,
+        const optional<pair<duration, duration>>& alt_interval,
+        optional<size_t> alt_timeout,
+        size_t normal_timeout)
       {
-        duration normal_rebuild_timeout (
-          chrono::seconds (mod_ops.build_normal_rebuild_timeout ()));
+        duration t;
 
-        if (mod_ops.build_alt_rebuild_start_specified ())
+        // If the rebuild timeout is not specified explicitly, then calculate
+        // it as the sum of the package rebuild timeout (normal rebuild
+        // timeout if the alternative timeout is unspecified and the maximum
+        // of two otherwise) and the build result timeout.
+        //
+        if (!rebuild_timeout)
         {
-          // Calculate the alternative rebuild timeout as the time interval
-          // lenght, unless it is specified explicitly.
-          //
-          if (!mod_ops.build_alt_rebuild_timeout_specified ())
+          if (normal_timeout == 0)
+            return timestamp_unknown;
+
+          chrono::seconds nt (normal_timeout);
+
+          if (alt_interval)
           {
-            const duration& start (mod_ops.build_alt_rebuild_start ());
-            const duration& stop  (mod_ops.build_alt_rebuild_stop ());
+            // Calculate the alternative timeout, unless it is specified
+            // explicitly.
+            //
+            if (!alt_timeout)
+            {
+              const duration& start (alt_interval->first);
+              const duration& stop  (alt_interval->second);
 
-            // Note that if the stop time is less than the start time then the
-            // interval extends through the midnight.
+              // Note that if the stop time is less than the start time then
+              // the interval extends through the midnight.
+              //
+              t = start <= stop ? (stop - start) : ((24h - start) + stop);
+
+              // If the normal rebuild time out is greater than 24 hours, then
+              // increase the default alternative timeout by (normal - 24h)
+              // (see build-alt-soft-rebuild-timeout configuration option for
+              // details).
+              //
+              if (nt > 24h)
+                t += nt - 24h;
+            }
+            else
+              t = chrono::seconds (*alt_timeout);
+
+            // Take the maximum of the alternative and normal rebuild
+            // timeouts.
             //
-            build_timeout = start <= stop
-                            ? stop - start
-                            : (24h - start) + stop;
+            if (t < nt)
+              t = nt;
           }
           else
-            build_timeout =
-              chrono::seconds (mod_ops.build_alt_rebuild_timeout ());
+            t = nt;
 
-          // Take the maximum of the alternative and normal rebuild timeouts.
+          // Summarize the rebuild and build result timeouts.
           //
-          if (build_timeout < normal_rebuild_timeout)
-            build_timeout = normal_rebuild_timeout;
+          t += chrono::seconds (mod_ops.build_result_timeout ());
         }
         else
-          build_timeout = normal_rebuild_timeout;
+        {
+          if (*rebuild_timeout == 0)
+            return timestamp_unknown;
 
-        // Summarize the rebuild and build result timeouts.
-        //
-        build_timeout += chrono::seconds (mod_ops.build_result_timeout ());
-      }
-      else
-        build_timeout = chrono::seconds (ops.build_timeout ());
+          t = chrono::seconds (*rebuild_timeout);
+        }
 
-      timestamp now (system_clock::now ());
-      timestamp build_expiration (now - build_timeout);
+        return now - t;
+      };
+
+      timestamp hard_rebuild_expiration (
+        build_expiration (
+          (ops.hard_rebuild_timeout_specified ()
+           ? ops.hard_rebuild_timeout ()
+           : optional<size_t> ()),
+          (mod_ops.build_alt_hard_rebuild_start_specified ()
+           ? make_pair (mod_ops.build_alt_hard_rebuild_start (),
+                        mod_ops.build_alt_hard_rebuild_stop ())
+           : optional<pair<duration, duration>> ()),
+          (mod_ops.build_alt_hard_rebuild_timeout_specified ()
+           ? mod_ops.build_alt_hard_rebuild_timeout ()
+           : optional<size_t> ()),
+          mod_ops.build_hard_rebuild_timeout ()));
+
+      timestamp soft_rebuild_expiration (
+        build_expiration (
+          (ops.soft_rebuild_timeout_specified ()
+           ? ops.soft_rebuild_timeout ()
+           : optional<size_t> ()),
+          (mod_ops.build_alt_soft_rebuild_start_specified ()
+           ? make_pair (mod_ops.build_alt_soft_rebuild_start (),
+                        mod_ops.build_alt_soft_rebuild_stop ())
+           : optional<pair<duration, duration>> ()),
+          (mod_ops.build_alt_soft_rebuild_timeout_specified ()
+           ? mod_ops.build_alt_soft_rebuild_timeout ()
+           : optional<size_t> ()),
+          mod_ops.build_soft_rebuild_timeout ()));
 
       timestamp report_expiration (
         now - chrono::seconds (ops.report_timeout ()));
@@ -583,8 +826,12 @@ namespace brep
                 // task have been issued recently we may still consider the
                 // build as delayed.
                 //
-                timestamp bct (b != nullptr
-                               ? b->completion_timestamp
+                timestamp bht (b != nullptr
+                               ? b->hard_timestamp
+                               : timestamp_nonexistent);
+
+                timestamp bst (b != nullptr
+                               ? b->soft_timestamp
                                : timestamp_nonexistent);
 
                 // Create the delay object to record a timestamp when the
@@ -605,15 +852,17 @@ namespace brep
                   if (bp.archived && b == nullptr)
                     continue;
 
-                  // Use the build completion or build status change
-                  // timestamp, whichever is earlier, as the build delay
-                  // tracking starting point and fallback to the current time
-                  // if there is no build yet.
+                  // Use the build hard, soft, or status change timestamp (see
+                  // the timestamps description for their ordering
+                  // information) as the build delay tracking starting point
+                  // and fallback to the current time if there is no build
+                  // yet.
                   //
                   timestamp pts (
-                    b == nullptr                                       ? now :
-                    bct != timestamp_nonexistent && bct < b->timestamp ? bct :
-                    b->timestamp);
+                    b == nullptr                 ? now :
+                    bht != timestamp_nonexistent ? bht :
+                    bst != timestamp_nonexistent ? bst :
+                                                   b->timestamp);
 
                   d = make_shared<build_delay> (move (id.package.tenant),
                                                 move (id.package.name),
@@ -632,20 +881,40 @@ namespace brep
                 // if it is not (re-)built by the expiration time. Otherwise,
                 // consider it as delayed if it is unbuilt.
                 //
-                bool delayed;
+                // We also don't need to report an unbuilt archived package
+                // twice, as both soft and hard build delays.
+                //
+                bool hard_delayed;
+                bool soft_delayed;
 
                 if (!bp.archived)
                 {
-                  timestamp bts (bct != timestamp_nonexistent
-                                 ? bct
+                  auto delayed = [&d] (timestamp bt, timestamp be)
+                  {
+                    timestamp t (bt != timestamp_nonexistent
+                                 ? bt
                                  : d->package_timestamp);
+                    return t <= be;
+                  };
 
-                  delayed = (bts <= build_expiration);
+                  hard_delayed = delayed (bht, hard_rebuild_expiration);
+                  soft_delayed = delayed (bst, soft_rebuild_expiration);
                 }
                 else
-                  delayed = (bct == timestamp_nonexistent);
+                {
+                  hard_delayed = (bst == timestamp_nonexistent);
+                  soft_delayed = false;
+                }
 
-                if (delayed)
+                // Add hard/soft delays to the respective reports and collect
+                // the delay for update, if it is reported.
+                //
+                // Note that we update the delay objects persistent state
+                // later, after we successfully print the reports.
+                //
+                bool reported (false);
+
+                if (hard_delayed)
                 {
                   // If the report timeout is zero then report the delay
                   // unconditionally. Otherwise, report the active package
@@ -655,33 +924,43 @@ namespace brep
                   // building an archived package, so reporting its build
                   // delays repeatedly is meaningless.
                   //
-                  if (ops.report_timeout () == 0 ||
-                      (!bp.archived
-                       ? d->report_timestamp <= report_expiration
-                       : d->report_timestamp == timestamp_nonexistent))
-                  {
-                    // Note that we update the delay objects persistent state
-                    // later, after we successfully print the report.
-                    //
-                    d->report_timestamp = now;
-                    delays.insert (move (d));
+                  bool report (
+                    ops.report_timeout () == 0 ||
+                    (!bp.archived
+                     ? d->report_hard_timestamp <= report_expiration
+                     : d->report_hard_timestamp == timestamp_nonexistent));
 
-                    ++reported_delay_count;
+                  if (report)
+                  {
+                    d->report_hard_timestamp = now;
+                    reported = true;
                   }
-                  //
-                  // In the brief mode also collect unreported delays to
-                  // deduce and print the total number of delays per
-                  // configuration. Mark such delays with the
-                  // timestamp_nonexistent report timestamp.
-                  //
-                  else if (!ops.full_report ())
+
+                  hard_delays_report.add_delay (d, report);
+                }
+
+                if (soft_delayed)
+                {
+                  bool report (ops.report_timeout () == 0 ||
+                               d->report_soft_timestamp <= report_expiration);
+
+                  if (report)
                   {
-                    d->report_timestamp = timestamp_nonexistent;
-                    delays.insert (move (d));
+                    d->report_soft_timestamp = now;
+                    reported = true;
                   }
 
-                  ++total_delay_count;
+                  soft_delays_report.add_delay (d, report);
                 }
+
+                // If we don't consider the report timestamps for reporting
+                // delays, it seems natural not to update these timestamps
+                // either. Note that reporting all delays and still updating
+                // the report timestamps can be achieved by specifying the
+                // zero report timeout.
+                //
+                if (reported && ops.report_timeout_specified ())
+                  update_delays.insert (move (d));
               }
             }
           }
@@ -691,161 +970,48 @@ namespace brep
       }
     }
 
-    // Report package build delays, if any.
+    // Print delay reports, if not empty.
     //
-    if (reported_delay_count != 0)
+    if (!hard_delays_report.empty () || !soft_delays_report.empty ())
     try
     {
-      // Print the report.
-      //
       cerr.exceptions (ostream::badbit | ostream::failbit);
 
       // Don't print the total delay count if the report timeout is zero since
       // all delays are reported in this case.
       //
-      bool print_total_delay_count (ops.report_timeout () != 0);
-
-      cerr << "Package build delays (" << reported_delay_count;
-
-      if (print_total_delay_count)
-        cerr << '/' << total_delay_count;
-
-      cerr << "):" << endl;
-
-      // Group the printed delays by toolchain and configuration.
-      //
-      const string*  toolchain_name    (nullptr);
-      const version* toolchain_version (nullptr);
-      const string*  configuration     (nullptr);
-
-      // In the brief report mode print the number of reported/total delayed
-      // package builds per configuration rather than the packages themselves.
-      //
-      size_t config_reported_delay_count (0);
-      size_t config_total_delay_count (0);
-
-      auto brief_config = [&configuration,
-                           &config_reported_delay_count,
-                           &config_total_delay_count,
-                           print_total_delay_count] ()
-      {
-        if (configuration != nullptr)
-        {
-          // Only print configurations with delays that needs to be reported.
-          //
-          if (config_reported_delay_count != 0)
-          {
-            cerr << "    " << *configuration << " ("
-                 << config_reported_delay_count;
+      bool total (ops.report_timeout () != 0);
 
-            if (print_total_delay_count)
-              cerr << '/' << config_total_delay_count;
+      hard_delays_report.print ("Package hard rebuild delays",
+                                total,
+                                ops.full_report ());
 
-            cerr << ')' << endl;
-          }
-
-          config_reported_delay_count = 0;
-          config_total_delay_count = 0;
-        }
-      };
-
-      for (shared_ptr<const build_delay> d: delays)
-      {
-        // Print the toolchain, if changed.
-        //
-        if (toolchain_name == nullptr            ||
-            d->toolchain_name != *toolchain_name ||
-            d->toolchain_version != *toolchain_version)
-        {
-          if (!ops.full_report ())
-            brief_config ();
-
-          if (toolchain_name != nullptr)
-            cerr << endl;
-
-          cerr << "  " << d->toolchain_name;
-
-          if (!d->toolchain_version.empty ())
-            cerr << "/" << d->toolchain_version;
-
-          cerr << endl;
-
-          toolchain_name    = &d->toolchain_name;
-          toolchain_version = &d->toolchain_version;
-          configuration     = nullptr;
-        }
-
-        // Print the configuration, if changed.
-        //
-        if (configuration == nullptr || d->configuration != *configuration)
-        {
-          if (ops.full_report ())
-          {
-            if (configuration != nullptr)
-              cerr << endl;
-
-            cerr << "    " << d->configuration << endl;
-          }
-          else
-            brief_config ();
-
-          configuration = &d->configuration;
-        }
-
-        // Print the delayed build package in the full report mode and count
-        // configuration builds otherwise.
-        //
-        if (ops.full_report ())
-        {
-          // We can potentially extend this information with the archived flag
-          // or the delay duration.
-          //
-          cerr << "      " << d->package_name << "/" << d->package_version;
-
-          if (!d->tenant.empty ())
-            cerr << " " << d->tenant;
-
-          cerr << endl;
-        }
-        else
-        {
-          if (d->report_timestamp != timestamp_nonexistent)
-            ++config_reported_delay_count;
-
-          ++config_total_delay_count;
-        }
-      }
-
-      if (!ops.full_report ())
-        brief_config ();
-
-      // Persist the delay report timestamps.
-      //
-      // If we don't consider the report timestamps for reporting delays, it
-      // seems natural not to update these timestamps either. Note that
-      // reporting all delays and still updating the report timestamps can be
-      // achieved by specifying the zero report timeout.
+      // Separate reports with an empty line.
       //
-      if (ops.report_timeout_specified ())
-      {
-        transaction t (db.begin ());
+      if (!hard_delays_report.empty () && !soft_delays_report.empty ())
+        cerr << endl;
 
-        for (shared_ptr<const build_delay> d: delays)
-        {
-          // Only update timestamps for delays that needs to be reported.
-          //
-          if (d->report_timestamp != timestamp_nonexistent)
-            db.update (d);
-        }
-
-        t.commit ();
-      }
+      soft_delays_report.print ("Package soft rebuild delays",
+                                total,
+                                ops.full_report ());
     }
     catch (const io_error&)
     {
       return 1; // Not much we can do on stderr writing failure.
     }
 
+    // Persist the delay report timestamps.
+    //
+    if (!update_delays.empty ())
+    {
+      transaction t (db.begin ());
+
+      for (shared_ptr<const build_delay> d: update_delays)
+        db.update (d);
+
+      t.commit ();
+    }
+
     return 0;
   }
   catch (const database_locked&)