Invent alternative package rebuild timeout

9 files changed, 242 insertions, 29 deletions

diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 12e96cd..83d18da 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -134,6 +134,34 @@ menu About=?about
 # build-normal-rebuild-timeout 86400
 
 
+# Alternative package rebuild timeout to use instead of the normal rebuild
+# timeout (see the build-normal-rebuild-timeout option for details) during
+# the specified time interval. Must be specified in seconds. Default is the
+# time interval length.
+#
+# The alternative rebuild timeout can be used to "pull" the rebuild window to
+# the specified time of day, for example, to optimize load and/or power
+# consumption of the build infrastructure (off-work hours, solar, off-peak
+# electricity tariffs, etc). A shorter than the time interval rebuild timeout
+# can also be used to force continuous rebuilds, for example, to shake out
+# flaky tests. Note also that if the alternative rebuild timeout is greater
+# than the normal rebuild timeout, then this will result in slower rebuilds
+# during the alternative time interval. In this case, if the build
+# infrastructure is monitored for delayed package builds, then the alternative
+# rebuild timeout should only be made slightly greater than the normal timeout
+# (see brep-monitor(1) for details).
+#
+# The time interval boundaries must be specified as times of day (in the local
+# timezone) in the <hours>:<minutes> form. If the stop time is less than the
+# start time then the interval extends through midnight. The start and stop
+# times must both be either specified or absent. If unspecified, then no
+# alternative rebuild timeout will be used.
+#
+# build-alt-rebuild-timeout
+# build-alt-rebuild-start
+# build-alt-rebuild-stop
+
+
 # The maximum size of the build task request manifest accepted. Note that the
 # HTTP POST request body is cached to retry database transactions in the face
 # of recoverable failures (deadlock, loss of connection, etc). Default is
diff --git a/libbrep/types.hxx b/libbrep/types.hxx
index c270d60..9d63b8c 100644
--- a/libbrep/types.hxx
+++ b/libbrep/types.hxx
@@ -94,6 +94,7 @@ namespace brep
   //
   using butl::system_clock;
   using butl::timestamp;
+  using butl::duration;
   using butl::timestamp_nonexistent;
 }
 
diff --git a/mod/buildfile b/mod/buildfile
index ca46bc4..191d966 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
   cli.options += --std c++11 -I $src_root --include-with-brackets    \
 --include-prefix mod --guard-prefix MOD --generate-specifier         \
 --cxx-prologue "#include <mod/types-parsers.hxx>"                    \
---cli-namespace brep::cli --generate-file-scanner --option-length 38 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 41 \
 --generate-modifier --generate-description --option-prefix ""
 
   # Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 17bc15e..edddb89 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -59,6 +59,14 @@ init (scanner& s)
 
   if (options_->build_config_specified ())
   {
+    // Verify that build-alt-rebuild-{start,stop} are both either specified or
+    // not.
+    //
+    if (options_->build_alt_rebuild_start_specified () !=
+        options_->build_alt_rebuild_stop_specified ())
+      fail << "build-alt-rebuild-start and build-alt-rebuild-stop "
+           << "configuration options must both be either specified or not";
+
     database_module::init (*options_, options_->build_db_retry ());
 
     // Check that the database 'build' schema matches the current one. It's
@@ -254,12 +262,48 @@ handle (request& rq, response& rs)
     uint64_t forced_result_expiration_ns (
       expiration_ns (options_->build_forced_rebuild_timeout ()));
 
-    timestamp normal_rebuild_expiration (
-      expiration (options_->build_normal_rebuild_timeout ()));
-
     timestamp forced_rebuild_expiration (
       expiration (options_->build_forced_rebuild_timeout ()));
 
+    timestamp normal_rebuild_expiration;
+
+    if (options_->build_alt_rebuild_start_specified ())
+    {
+      const duration& start (options_->build_alt_rebuild_start ());
+      const duration& stop  (options_->build_alt_rebuild_stop ());
+
+      duration dt (daytime (now));
+
+      // Note that if the stop time is less than the start time then the
+      // interval extends through the midnight.
+      //
+      bool alt_timeout (start <= stop
+                        ? dt >= start && dt < stop
+                        : dt >= start || dt < stop);
+
+      // If we out of the alternative rebuild timeout interval, then fall back
+      // to using the normal rebuild timeout.
+      //
+      if (alt_timeout)
+      {
+        if (!options_->build_alt_rebuild_timeout_specified ())
+        {
+          duration interval_len (start <= stop
+                                 ? stop - start
+                                 : (24h - start) + stop);
+
+          normal_rebuild_expiration = now - interval_len;
+        }
+        else
+          normal_rebuild_expiration =
+            expiration (options_->build_alt_rebuild_timeout ());
+      }
+    }
+
+    if (normal_rebuild_expiration == timestamp_nonexistent)
+      normal_rebuild_expiration =
+        expiration (options_->build_normal_rebuild_timeout ());
+
     // Return the challenge (nonce) if brep is configured to authenticate bbot
     // agents. Return nullopt otherwise.
     //
@@ -301,7 +345,7 @@ handle (request& rq, response& rs)
           if (!os.wait () || nonce.size () != 64)
             fail << "unable to generate nonce";
 
-          uint64_t t (chrono::duration_cast<std::chrono::nanoseconds> (
+          uint64_t t (chrono::duration_cast<chrono::nanoseconds> (
                         now.time_since_epoch ()).count ());
 
           sha256 cs (nonce.data (), nonce.size ());
@@ -397,11 +441,11 @@ handle (request& rq, response& rs)
                           canonical_version (toolchain_version),
                           true /* revision */)                   &&
 
-      (bld_query::state == "built" ||
-       ((bld_query::force == "forcing" &&
-         bld_query::timestamp > forced_result_expiration_ns) ||
-        (bld_query::force != "forcing" && // Unforced or forced.
-         bld_query::timestamp > normal_result_expiration_ns))));
+      (bld_query::state == "built"                          ||
+       (bld_query::force == "forcing" &&
+        bld_query::timestamp > forced_result_expiration_ns) ||
+       (bld_query::force != "forcing" && // Unforced or forced.
+        bld_query::timestamp > normal_result_expiration_ns)));
 
     prep_bld_query bld_prep_query (
       conn->prepare_query<build> ("mod-build-task-build-query", bq));
diff --git a/mod/module.cli b/mod/module.cli
index fa1d2cc..b59158a 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -199,6 +199,40 @@ namespace brep
         "Time to wait before considering a package for a normal rebuild. Must
          be specified in seconds. Default is 24 hours."
       }
+
+      size_t build-alt-rebuild-timeout
+      {
+        "<seconds>",
+        "Alternative package rebuild timeout to use instead of the normal
+         rebuild timeout (see \cb{build-normal-rebuild-timeout} for details)
+         during the time interval specified with the
+         \cb{build-alt-rebuild-start} and \cb{build-alt-rebuild-stop} options.
+         Must be specified in seconds. Default is the time interval length."
+      }
+
+      duration build-alt-rebuild-start
+      {
+        "<hours>:<minutes>",
+        "The start time of the alternative package rebuild timeout (see
+         \cb{build-alt-rebuild-timeout} for details). Must be specified as
+         a time of day in the local timezone. The \cb{build-alt-rebuild-start}
+         and \cb{build-alt-rebuild-stop} options must be either both specified
+         or absent. If unspecified, then no alternative rebuild timeout will
+         be used."
+      }
+
+      duration build-alt-rebuild-stop
+      {
+        "<hours>:<minutes>",
+        "The end time of the alternative package rebuild timeout (see
+         \cb{build-alt-rebuild-timeout} for details). Must be specified as
+         a time of day in the local timezone. If it is less than the
+         \cb{build-alt-rebuild-start} option value, then the time interval
+         extends through midnight. The \cb{build-alt-rebuild-start} and
+         \cb{build-alt-rebuild-stop} options must be either both specified or
+         absent. If unspecified, then no alternative rebuild timeout will be
+         used."
+      }
     };
 
     class build_db
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index ceaab29..dc21e97 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -3,6 +3,8 @@
 
 #include <mod/types-parsers.hxx>
 
+#include <libbutl/timestamp.mxx> // from_string()
+
 #include <mod/module-options.hxx>
 
 using namespace std;
@@ -50,6 +52,40 @@ namespace brep
       parse_path (x, s);
     }
 
+    // Parse time of day.
+    //
+    void parser<duration>::
+    parse (duration& x, bool& xs, scanner& s)
+    {
+      xs = true;
+
+      const char* o (s.next ());
+
+      if (!s.more ())
+        throw missing_value (o);
+
+      const char* v (s.next ());
+
+      // To avoid the manual time of day parsing and validation, let's parse
+      // it as the first Epoch day time and convert the result (timestamp) to
+      // the time elapsed since Epoch (duration).
+      //
+      try
+      {
+        string t ("1970-01-01 ");
+        t += v;
+
+        x = butl::from_string (t.c_str (),
+                               "%Y-%m-%d %H:%M",
+                               false /* local */).time_since_epoch ();
+        return;
+      }
+      catch (const invalid_argument&) {}
+      catch (const system_error&) {}
+
+      throw invalid_value (o, v);
+    }
+
     // Parse repository_location.
     //
     void parser<repository_location>::
diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx
index 091c868..6b851eb 100644
--- a/mod/types-parsers.hxx
+++ b/mod/types-parsers.hxx
@@ -39,6 +39,15 @@ namespace brep
       parse (dir_path&, bool&, scanner&);
     };
 
+    // Parse time of day specified in the `hh:mm` form.
+    //
+    template <>
+    struct parser<duration>
+    {
+      static void
+      parse (duration&, bool&, scanner&);
+    };
+
     template <>
     struct parser<bpkg::repository_location>
     {
diff --git a/monitor/monitor.cli b/monitor/monitor.cli
index e4d228f..b3687a9 100644
--- a/monitor/monitor.cli
+++ b/monitor/monitor.cli
@@ -32,10 +32,10 @@ namespace brep
        \cb{brep-monitor} analyzes the \cb{brep} internal state and reports the
        infrastructure issues printing their descriptions to \cb{stderr}.
 
-       The specified \cb{brep} configuration file (<brep-config>) is used to
-       retrieve information required to access the databases and deduce the
-       expected behavior. Most of this information can be overridden via the
-       command line options.
+       The specified \cb{brep} module configuration file (<brep-config>) is
+       used to retrieve information required to access the databases and
+       deduce the expected behavior. Most of this information can be
+       overridden via the command line options.
 
        Currently, only delayed package builds for the specified toolchains are
        reported. If toolchain version is omitted then all package builds with
@@ -64,10 +64,21 @@ namespace brep
       {
         "<seconds>",
         "Time to wait (in seconds) before considering a package build as
-         delayed. If unspecified, the sum of \cb{brep}'s
-         \cb{build-normal-rebuild-timeout} and \cb{build-result-timeout}
-         configuration option values is used. Note also that an archived
-         package that is unbuilt is always considered delayed."
+         delayed. If unspecified, it is the sum of the package rebuild timeout
+         (normal rebuild timeout if the alternative timeout is unspecified and
+         the maximum of two otherwise) and the build result timeout (see the
+         \cb{build-normal-rebuild-timeout}, \cb{build-alt-rebuild-*}, and
+         \cb{build-result-timeout} \c{brep} module configuration options
+         for details).
+
+         Note that a package that was not built before it was archived is
+         always considered as delayed. However, to distinguish this case from
+         a situation where a package was archived before a configuration have
+         been added, \cb{brep-monitor} needs to observe the package as
+         buildable for this configuration before it is archived. As result, if
+         you run \cb{brep-monitor} periodically (for example, as a cron job),
+         then make sure its running period is less than the tenant archive
+         timeout."
       }
 
       std::size_t --report-timeout
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index d1442d5..bbab0a5 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -140,12 +140,15 @@ namespace brep
              << "': " << e << endl;
         return 1;
       }
-    }
 
-    if (!mod_ops.build_config_specified ())
-    {
-      cerr << "warning: package building functionality is disabled" << endl;
-      return 0;
+      if (mod_ops.build_alt_rebuild_start_specified () !=
+          mod_ops.build_alt_rebuild_stop_specified ())
+      {
+        cerr << "build-alt-rebuild-start and build-alt-rebuild-stop "
+             << "configuration options must both be either specified or not "
+             << "in '" << f << "'" << endl;
+        return 1;
+      }
     }
 
     // Parse the toolchains suppressing duplicates.
@@ -212,6 +215,12 @@ namespace brep
 
     // Parse buildtab.
     //
+    if (!mod_ops.build_config_specified ())
+    {
+      cerr << "warning: package building functionality is disabled" << endl;
+      return 0;
+    }
+
     build_configs configs;
 
     try
@@ -471,14 +480,55 @@ namespace brep
       prep_bquery pbq (
         conn->prepare_query<package_build> ("package-build-query", bq));
 
-      timestamp::duration build_timeout (
-        ops.build_timeout_specified ()
-        ? chrono::seconds (ops.build_timeout ())
-        : chrono::seconds (mod_ops.build_normal_rebuild_timeout () +
-                           mod_ops.build_result_timeout ()));
+      duration build_timeout;
 
-      timestamp now (system_clock::now ());
+      // If the build timeout is not specified explicitly, then calculate it
+      // as the sum of the package rebuild timeout (normal rebuild timeout if
+      // the alternative timeout is unspecified and the maximum of two
+      // otherwise) and the build result timeout.
+      //
+      if (!ops.build_timeout_specified ())
+      {
+        duration normal_rebuild_timeout (
+          chrono::seconds (mod_ops.build_normal_rebuild_timeout ()));
 
+        if (mod_ops.build_alt_rebuild_start_specified ())
+        {
+          // Calculate the alternative rebuild timeout as the time interval
+          // lenght, unless it is specified explicitly.
+          //
+          if (!mod_ops.build_alt_rebuild_timeout_specified ())
+          {
+            const duration& start (mod_ops.build_alt_rebuild_start ());
+            const duration& stop  (mod_ops.build_alt_rebuild_stop ());
+
+            // Note that if the stop time is less than the start time then the
+            // interval extends through the midnight.
+            //
+            build_timeout = start <= stop
+                            ? stop - start
+                            : (24h - start) + stop;
+          }
+          else
+            build_timeout =
+              chrono::seconds (mod_ops.build_alt_rebuild_timeout ());
+
+          // Take the maximum of the alternative and normal rebuild timeouts.
+          //
+          if (build_timeout < normal_rebuild_timeout)
+            build_timeout = normal_rebuild_timeout;
+        }
+        else
+          build_timeout = normal_rebuild_timeout;
+
+        // Summarize the rebuild and build result timeouts.
+        //
+        build_timeout += chrono::seconds (mod_ops.build_result_timeout ());
+      }
+      else
+        build_timeout = chrono::seconds (ops.build_timeout ());
+
+      timestamp now (system_clock::now ());
       timestamp build_expiration (now - build_timeout);
 
       timestamp report_expiration (