Add support for random package ordering in build task module

6 files changed, 655 insertions, 522 deletions

diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 30bff0d..bd5034b 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -6,11 +6,14 @@
 #include <map>
 #include <regex>
 #include <chrono>
+#include <random>
 
 #include <odb/database.hxx>
 #include <odb/transaction.hxx>
 #include <odb/schema-catalog.hxx>
 
+#include <libbutl/ft/lang.hxx> // thread_local
+
 #include <libbutl/regex.hxx>
 #include <libbutl/sha256.hxx>
 #include <libbutl/openssl.hxx>
@@ -39,6 +42,22 @@ using namespace bbot;
 using namespace brep::cli;
 using namespace odb::core;
 
+static thread_local mt19937 rand_gen (random_device {} ());
+
+// Generate a random number in the specified range (max value is included).
+//
+static inline size_t
+rand (size_t min_val, size_t max_val)
+{
+  // Note that size_t is not whitelisted as a type the
+  // uniform_int_distribution class template can be instantiated with.
+  //
+  return static_cast<size_t> (
+    uniform_int_distribution<unsigned long long> (
+      static_cast<unsigned long long> (min_val),
+      static_cast<unsigned long long> (max_val)) (rand_gen));
+}
+
 // While currently the user-defined copy constructor is not required (we don't
 // need to deep copy nullptr's), it is a good idea to keep the placeholder
 // ready for less trivial cases.
@@ -98,6 +117,45 @@ init (scanner& s)
     options_->root (dir_path ("/"));
 }
 
+template <typename T>
+static inline query<T>
+package_query (brep::params::build_task& params, interactive_mode imode)
+{
+  using namespace brep;
+  using query = query<T>;
+
+  query q (!query::build_tenant::archived);
+
+  // Filter by repositories canonical names (if requested).
+  //
+  const vector<string>& rp (params.repository ());
+
+  if (!rp.empty ())
+    q = q &&
+      query::build_repository::id.canonical_name.in_range (rp.begin (),
+                                                           rp.end ());
+
+  // If the interactive mode is false or true, then filter out the respective
+  // packages.
+  //
+  switch (imode)
+  {
+  case interactive_mode::false_:
+    {
+      q = q && query::build_tenant::interactive.is_null ();
+      break;
+      }
+  case interactive_mode::true_:
+    {
+      q = q && query::build_tenant::interactive.is_not_null ();
+      break;
+    }
+  case interactive_mode::both: break;
+  }
+
+  return q;
+}
+
 bool brep::build_task::
 handle (request& rq, response& rs)
 {
@@ -513,18 +571,8 @@ handle (request& rq, response& rs)
     using pkg_query = query<buildable_package>;
     using prep_pkg_query = prepared_query<buildable_package>;
 
-    // Exclude archived tenants.
-    //
-    pkg_query pq (!pkg_query::build_tenant::archived);
-
-    // Filter by repositories canonical names (if requested).
-    //
-    const vector<string>& rp (params.repository ());
-
-    if (!rp.empty ())
-      pq = pq &&
-        pkg_query::build_repository::id.canonical_name.in_range (rp.begin (),
-                                                                 rp.end ());
+    interactive_mode imode (tqm.effective_interactive_mode ());
+    pkg_query pq (package_query<buildable_package> (params, imode));
 
     // Transform (in-place) the interactive login information into the actual
     // login command, if specified in the manifest and the transformation
@@ -555,622 +603,665 @@ handle (request& rq, response& rs)
       tqm.interactive_login = move (lc);
     }
 
-    // If the interactive mode if false or true, then filter out the
-    // respective packages. Otherwise, order them so that packages from the
-    // interactive build tenants appear first.
+    // In the random package ordering mode iterate over the packages list by
+    // starting from the random offset and wrapping around when reaching the
+    // end.
     //
-    interactive_mode imode (tqm.effective_interactive_mode ());
+    size_t start_offset (0);
+    optional<size_t> package_count;
 
-    switch (imode)
+    if (options_->build_package_order () == build_order::random)
     {
-    case interactive_mode::false_:
-      {
-        pq = pq && pkg_query::build_tenant::interactive.is_null ();
-        break;
-      }
-    case interactive_mode::true_:
-      {
-        pq = pq && pkg_query::build_tenant::interactive.is_not_null ();
-        break;
-      }
-    case interactive_mode::both: break; // See below.
-    }
-
-    // Specify the portion.
-    //
-    size_t offset (0);
-
-    pq += "ORDER BY";
+      using query = query<buildable_package_count>;
 
-    if (imode == interactive_mode::both)
-      pq += pkg_query::build_tenant::interactive + "NULLS LAST,";
+      query q (package_query<buildable_package_count> (params, imode));
 
-    pq += pkg_query::build_package::id.tenant + "," +
-      pkg_query::build_package::id.name +
-      order_by_version (pkg_query::build_package::id.version, false) +
-      "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50";
+      transaction t (build_db_->begin ());
 
-    connection_ptr conn (build_db_->connection ());
-
-    prep_pkg_query pkg_prep_query (
-      conn->prepare_query<buildable_package> (
-        "mod-build-task-package-query", pq));
-
-    // Prepare the build prepared query.
-    //
-    // Note that we can not query the database for configurations that a
-    // package was not built with, as the database contains only those build
-    // configurations that have already been acted upon (initially empty).
-    //
-    // This is why we query the database for configurations that should not be
-    // built (in the built state, or in the building state and not expired).
-    // Having such a list we will select the first build configuration that is
-    // not in the list (if available) for the response.
-    //
-    using bld_query = query<build>;
-    using prep_bld_query = prepared_query<build>;
+      // If there are any non-archived interactive build tennants, then we
+      // need to start from one of packages they contain. Note that packages
+      // from the interactive build tenants appear first (see below for the
+      // package ordering details).
+      //
+      package_count =
+        build_db_->query_value<buildable_package_count> (
+          q && query::build_tenant::interactive.is_not_null ());
 
-    package_id id;
-    string pkg_config_name;
+      if (*package_count == 0)
+        package_count = build_db_->query_value<buildable_package_count> (q);
 
-    bld_query sq (false);
-    for (const auto& cm: conf_machines)
-      sq = sq || (bld_query::id.target == cm.first.target             &&
-                  bld_query::id.target_config_name == cm.first.config &&
-                  bld_query::id.package_config_name ==
-                    bld_query::_ref (pkg_config_name));
+      t.commit ();
 
-    bld_query bq (
-      equal<build> (bld_query::id.package, id)                   &&
-      sq                                                         &&
-      bld_query::id.toolchain_name == tqm.toolchain_name         &&
+      if (*package_count != 0)
+        start_offset = rand (0, *package_count - 1);
+    }
 
-      compare_version_eq (bld_query::id.toolchain_version,
-                          canonical_version (toolchain_version),
-                          true /* revision */)                   &&
+    if (!package_count || *package_count != 0)
+    {
+      // Specify the portion.
+      //
+      size_t offset (start_offset);
+      size_t limit  (50);
 
-      (bld_query::state == "built"                          ||
-       (bld_query::force == "forcing" &&
-        bld_query::timestamp > forced_result_expiration_ns) ||
-       (bld_query::force != "forcing" && // Unforced or forced.
-        bld_query::timestamp > normal_result_expiration_ns)));
+      pq += "ORDER BY";
 
-    prep_bld_query bld_prep_query (
-      conn->prepare_query<build> ("mod-build-task-build-query", bq));
+      // If the interactive mode is both, then order the packages so that ones
+      // from the interactive build tenants appear first.
+      //
+      if (imode == interactive_mode::both)
+        pq += pkg_query::build_tenant::interactive + "NULLS LAST,";
 
-    // Return true if a package needs to be rebuilt.
-    //
-    auto needs_rebuild = [&forced_rebuild_expiration,
-                          &soft_rebuild_expiration,
-                          &hard_rebuild_expiration] (const build& b)
-    {
-      assert (b.state == build_state::built);
+      pq += pkg_query::build_package::id.tenant + ","                  +
+        pkg_query::build_package::id.name                              +
+        order_by_version (pkg_query::build_package::id.version, false) +
+        "OFFSET" + pkg_query::_ref (offset)                            +
+        "LIMIT" + pkg_query::_ref (limit);
 
-      return (b.force == force_state::forced &&
-              b.timestamp <= forced_rebuild_expiration)  ||
-             b.soft_timestamp <= soft_rebuild_expiration ||
-             b.hard_timestamp <= hard_rebuild_expiration;
-    };
+      connection_ptr conn (build_db_->connection ());
 
-    // Convert a build to the hard rebuild, resetting the agent checksum and
-    // dropping the previous build task result.
-    //
-    // Note that since the checksums are hierarchical, the agent checksum
-    // reset will trigger resets of the "subordinate" checksums up to the
-    // dependency checksum and so the package will be rebuilt.
-    //
-    // Also note that there is no sense to keep the build task result since we
-    // don't accept the skip result for the hard rebuild task. We, however,
-    // keep the status intact (see below for the reasoning).
-    //
-    auto convert_to_hard = [] (const shared_ptr<build>& b)
-    {
-      b->agent_checksum = nullopt;
+      prep_pkg_query pkg_prep_query (
+        conn->prepare_query<buildable_package> (
+          "mod-build-task-package-query", pq));
 
-      // Mark the section as loaded, so results are updated.
+      // Prepare the build prepared query.
       //
-      b->results_section.load ();
-      b->results.clear ();
-    };
+      // Note that we can not query the database for configurations that a
+      // package was not built with, as the database contains only those build
+      // configurations that have already been acted upon (initially empty).
+      //
+      // This is why we query the database for configurations that should not
+      // be built (in the built state, or in the building state and not
+      // expired).  Having such a list we will select the first build
+      // configuration that is not in the list (if available) for the
+      // response.
+      //
+      using bld_query = query<build>;
+      using prep_bld_query = prepared_query<build>;
 
-    // Return SHA256 checksum of the controller logic and the configuration
-    // target, environment, arguments, and warning-detecting regular
-    // expressions.
-    //
-    auto controller_checksum = [] (const build_target_config& c)
-    {
-      sha256 cs ("1"); // Hash the logic version.
+      package_id id;
+      string pkg_config_name;
 
-      cs.append (c.target.string ());
-      cs.append (c.environment ? *c.environment : "");
+      bld_query sq (false);
+      for (const auto& cm: conf_machines)
+        sq = sq || (bld_query::id.target == cm.first.target             &&
+                    bld_query::id.target_config_name == cm.first.config &&
+                    bld_query::id.package_config_name ==
+                    bld_query::_ref (pkg_config_name));
 
-      for (const string& a: c.args)
-        cs.append (a);
+      bld_query bq (
+        equal<build> (bld_query::id.package, id)                   &&
+        sq                                                         &&
+        bld_query::id.toolchain_name == tqm.toolchain_name         &&
 
-      for (const string& re: c.warning_regexes)
-        cs.append (re);
+        compare_version_eq (bld_query::id.toolchain_version,
+                            canonical_version (toolchain_version),
+                            true /* revision */)                   &&
 
-      return string (cs.string ());
-    };
+        (bld_query::state == "built"                          ||
+         (bld_query::force == "forcing" &&
+          bld_query::timestamp > forced_result_expiration_ns) ||
+         (bld_query::force != "forcing" && // Unforced or forced.
+          bld_query::timestamp > normal_result_expiration_ns)));
 
-    // Return the machine id as a machine checksum.
-    //
-    auto machine_checksum = [] (const machine_header_manifest& m)
-    {
-      return m.id;
-    };
+      prep_bld_query bld_prep_query (
+        conn->prepare_query<build> ("mod-build-task-build-query", bq));
 
-    while (tsm.session.empty ())
-    {
-      transaction t (conn->begin ());
+      // Return true if a package needs to be rebuilt.
+      //
+      auto needs_rebuild = [&forced_rebuild_expiration,
+                            &soft_rebuild_expiration,
+                            &hard_rebuild_expiration] (const build& b)
+      {
+        assert (b.state == build_state::built);
 
-      // Query (and cache) buildable packages.
+        return (b.force == force_state::forced &&
+                b.timestamp <= forced_rebuild_expiration)  ||
+        b.soft_timestamp <= soft_rebuild_expiration ||
+        b.hard_timestamp <= hard_rebuild_expiration;
+      };
+
+      // Convert a build to the hard rebuild, resetting the agent checksum and
+      // dropping the previous build task result.
+      //
+      // Note that since the checksums are hierarchical, the agent checksum
+      // reset will trigger resets of the "subordinate" checksums up to the
+      // dependency checksum and so the package will be rebuilt.
+      //
+      // Also note that there is no sense to keep the build task result since we
+      // don't accept the skip result for the hard rebuild task. We, however,
+      // keep the status intact (see below for the reasoning).
       //
-      auto packages (pkg_prep_query.execute ());
+      auto convert_to_hard = [] (const shared_ptr<build>& b)
+      {
+        b->agent_checksum = nullopt;
+
+        // Mark the section as loaded, so results are updated.
+        //
+        b->results_section.load ();
+        b->results.clear ();
+      };
 
-      // Bail out if there is nothing left.
+      // Return SHA256 checksum of the controller logic and the configuration
+      // target, environment, arguments, and warning-detecting regular
+      // expressions.
       //
-      if (packages.empty ())
+      auto controller_checksum = [] (const build_target_config& c)
       {
-        t.commit ();
-        break;
-      }
+        sha256 cs ("1"); // Hash the logic version.
+
+        cs.append (c.target.string ());
+        cs.append (c.environment ? *c.environment : "");
 
-      offset += packages.size ();
+        for (const string& a: c.args)
+          cs.append (a);
 
-      // Iterate over packages until we find one that needs building.
+        for (const string& re: c.warning_regexes)
+          cs.append (re);
+
+        return string (cs.string ());
+      };
+
+      // Return the machine id as a machine checksum.
       //
-      for (auto& bp: packages)
+      auto machine_checksum = [] (const machine_header_manifest& m)
       {
-        id = move (bp.id);
+        return m.id;
+      };
 
-        shared_ptr<build_package> p (build_db_->load<build_package> (id));
+      for (bool done (false); tsm.session.empty () && !done; )
+      {
+        transaction t (conn->begin ());
 
-        // Note that a request to interactively build a package in multiple
-        // configurations is most likely a mistake than a deliberate choice.
-        // Thus, for the interactive tenant let's check if the package can be
-        // built in multiple configurations. If that's the case then we will
-        // put all the potential builds into the aborted state and continue
-        // iterating looking for another package. Otherwise, just proceed for
-        // this package normally.
+        // We need to be careful in the random package ordering mode not to
+        // miss the end after having wrapped around.
         //
-        // It also feels like a good idea to archive an interactive tenant
-        // after a build object is created for it, regardless if the build
-        // task is issued or not. This way we make sure that an interactive
-        // build is never performed multiple times for such a tenant for any
-        // reason (multiple toolchains, buildtab change, etc). Note that the
-        // build result will still be accepted for an archived build.
+        done = (start_offset != 0     &&
+                offset < start_offset &&
+                offset + limit >= start_offset);
+
+        if (done)
+          limit = start_offset - offset;
+
+        // Query (and cache) buildable packages.
         //
-        shared_ptr<build_tenant> t (build_db_->load<build_tenant> (id.tenant));
+        auto packages (pkg_prep_query.execute ());
 
-        if (t->interactive)
+        // Bail out if there is nothing left, unless we need to wrap around in
+        // the random package ordering mode.
+        //
+        if (packages.empty ())
         {
-          // Note that the tenant can be archived via some other package on
-          // some previous iteration. Skip the package if that's the case.
-          //
-          if (t->archived)
-            continue;
+          t.commit ();
 
-          // Collect the potential build configurations as all combinations of
-          // the tenant's packages build configurations and the non-excluded
-          // (by the packages) build target configurations. Note that here we
-          // ignore the machines from the task request.
-          //
-          struct build_config
-          {
-            package_id                 pid;
-            string                     pc;
-            const build_target_config* tc;
-          };
+          if (start_offset != 0 && offset >= start_offset)
+            offset = 0;
+          else
+            done = true;
+
+          continue;
+        }
+
+        offset += packages.size ();
+
+        // Iterate over packages until we find one that needs building.
+        //
+        for (auto& bp: packages)
+        {
+          id = move (bp.id);
 
-          small_vector<build_config, 1> build_configs;
+          shared_ptr<build_package> p (build_db_->load<build_package> (id));
 
-          // Note that we don't bother creating a prepared query here, since
-          // its highly unlikely to encounter multiple interactive tenants per
-          // task request. Given that we archive such tenants immediately, as
-          // a common case there will be none.
+          // Note that a request to interactively build a package in multiple
+          // configurations is most likely a mistake than a deliberate choice.
+          // Thus, for the interactive tenant let's check if the package can
+          // be built in multiple configurations. If that's the case then we
+          // will put all the potential builds into the aborted state and
+          // continue iterating looking for another package. Otherwise, just
+          // proceed for this package normally.
           //
-          pkg_query pq (pkg_query::build_tenant::id == t->id);
-          for (auto& tp: build_db_->query<buildable_package> (pq))
+          // It also feels like a good idea to archive an interactive tenant
+          // after a build object is created for it, regardless if the build
+          // task is issued or not. This way we make sure that an interactive
+          // build is never performed multiple times for such a tenant for any
+          // reason (multiple toolchains, buildtab change, etc). Note that the
+          // build result will still be accepted for an archived build.
+          //
+          shared_ptr<build_tenant> t (
+            build_db_->load<build_tenant> (id.tenant));
+
+          if (t->interactive)
           {
-            shared_ptr<build_package> p (
-              build_db_->load<build_package> (tp.id));
+            // Note that the tenant can be archived via some other package on
+            // some previous iteration. Skip the package if that's the case.
+            //
+            if (t->archived)
+              continue;
+
+            // Collect the potential build configurations as all combinations
+            // of the tenant's packages build configurations and the
+            // non-excluded (by the packages) build target
+            // configurations. Note that here we ignore the machines from the
+            // task request.
+            //
+            struct build_config
+            {
+              package_id                 pid;
+              string                     pc;
+              const build_target_config* tc;
+            };
+
+            small_vector<build_config, 1> build_configs;
 
-            for (build_package_config& pc: p->configs)
+            // Note that we don't bother creating a prepared query here, since
+            // its highly unlikely to encounter multiple interactive tenants
+            // per task request. Given that we archive such tenants
+            // immediately, as a common case there will be none.
+            //
+            pkg_query pq (pkg_query::build_tenant::id == t->id);
+            for (auto& tp: build_db_->query<buildable_package> (pq))
             {
-              for (const auto& tc: *target_conf_)
+              shared_ptr<build_package> p (
+                build_db_->load<build_package> (tp.id));
+
+              for (build_package_config& pc: p->configs)
               {
-                if (!exclude (pc, p->builds, p->constraints, tc))
-                  build_configs.push_back (build_config {p->id, pc.name, &tc});
+                for (const auto& tc: *target_conf_)
+                {
+                  if (!exclude (pc, p->builds, p->constraints, tc))
+                    build_configs.push_back (
+                      build_config {p->id, pc.name, &tc});
+                }
               }
             }
-          }
 
-          // If multiple build configurations are collected, then abort all
-          // the potential builds and continue iterating over the packages.
-          //
-          if (build_configs.size () > 1)
-          {
-            // Abort the builds.
+            // If multiple build configurations are collected, then abort all
+            // the potential builds and continue iterating over the packages.
             //
-            for (build_config& c: build_configs)
+            if (build_configs.size () > 1)
             {
-              const string& pc (c.pc);
-              const build_target_config& tc (*c.tc);
-
-              build_id bid (c.pid,
-                            tc.target,
-                            tc.name,
-                            pc,
-                            tqm.toolchain_name,
-                            toolchain_version);
-
-              // Can there be any existing builds for such a tenant? Doesn't
-              // seem so, unless due to some manual intervention into the
-              // database. Anyway, let's just leave such a build alone.
+              // Abort the builds.
               //
-              shared_ptr<build> b (build_db_->find<build> (bid));
-
-              if (b == nullptr)
+              for (build_config& c: build_configs)
               {
-                b = make_shared<build> (
-                  move (bid.package.tenant),
-                  move (bid.package.name),
-                  bp.version,
-                  move (bid.target),
-                  move (bid.target_config_name),
-                  move (bid.package_config_name),
-                  move (bid.toolchain_name),
-                  toolchain_version,
-                  nullopt             /* interactive */,
-                  nullopt             /* agent_fp */,
-                  nullopt             /* agent_challenge */,
-                  "brep"              /* machine */,
-                  "build task module" /* machine_summary */,
-                  ""                  /* controller_checksum */,
-                  ""                  /* machine_checksum */);
-
-                b->state  = build_state::built;
-                b->status = result_status::abort;
-
-                b->soft_timestamp = b->timestamp;
-                b->hard_timestamp = b->soft_timestamp;
-
-                // Mark the section as loaded, so results are updated.
+                const string& pc (c.pc);
+                const build_target_config& tc (*c.tc);
+
+                build_id bid (c.pid,
+                              tc.target,
+                              tc.name,
+                              pc,
+                              tqm.toolchain_name,
+                              toolchain_version);
+
+                // Can there be any existing builds for such a tenant? Doesn't
+                // seem so, unless due to some manual intervention into the
+                // database. Anyway, let's just leave such a build alone.
                 //
-                b->results_section.load ();
-
-                b->results.push_back (
-                  operation_result {
-                    "configure",
+                shared_ptr<build> b (build_db_->find<build> (bid));
+
+                if (b == nullptr)
+                {
+                  b = make_shared<build> (
+                    move (bid.package.tenant),
+                    move (bid.package.name),
+                    bp.version,
+                    move (bid.target),
+                    move (bid.target_config_name),
+                    move (bid.package_config_name),
+                    move (bid.toolchain_name),
+                    toolchain_version,
+                    nullopt             /* interactive */,
+                    nullopt             /* agent_fp */,
+                    nullopt             /* agent_challenge */,
+                    "brep"              /* machine */,
+                    "build task module" /* machine_summary */,
+                    ""                  /* controller_checksum */,
+                    ""                  /* machine_checksum */);
+
+                  b->state  = build_state::built;
+                  b->status = result_status::abort;
+
+                  b->soft_timestamp = b->timestamp;
+                  b->hard_timestamp = b->soft_timestamp;
+
+                  // Mark the section as loaded, so results are updated.
+                  //
+                  b->results_section.load ();
+
+                  b->results.push_back (
+                    operation_result {
+                      "configure",
                       result_status::abort,
                       "error: multiple configurations for interactive build\n"});
 
-                build_db_->persist (b);
+                  build_db_->persist (b);
+                }
               }
-            }
 
-            // Archive the tenant.
-            //
-            t->archived = true;
-            build_db_->update (t);
+              // Archive the tenant.
+              //
+              t->archived = true;
+              build_db_->update (t);
 
-            continue; // Skip the package.
+              continue; // Skip the package.
+            }
           }
-        }
 
-        for (build_package_config& pc: p->configs)
-        {
-          pkg_config_name = pc.name;
-
-          // Iterate through the built configurations and erase them from the
-          // build configuration map. All those configurations that remained
-          // can be built. We will take the first one, if present.
-          //
-          // Also save the built configurations for which it's time to be
-          // rebuilt.
-          //
-          config_machines configs (conf_machines); // Make a copy for this pkg.
-          auto pkg_builds (bld_prep_query.execute ());
-
-          for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i)
+          for (build_package_config& pc: p->configs)
           {
-            auto j (
-              configs.find (build_target_config_id {i->id.target,
-                                                    i->id.target_config_name}));
+            pkg_config_name = pc.name;
 
-            // Outdated configurations are already excluded with the database
-            // query.
+            // Iterate through the built configurations and erase them from the
+            // build configuration map. All those configurations that remained
+            // can be built. We will take the first one, if present.
             //
-            assert (j != configs.end ());
-            configs.erase (j);
-
-            if (i->state == build_state::built)
-            {
-              assert (i->force != force_state::forcing);
-
-              if (needs_rebuild (*i))
-                rebuilds.emplace_back (i.load ());
-            }
-          }
-
-          if (!configs.empty ())
-          {
-            // Find the first build configuration that is not excluded by the
-            // package configuration.
+            // Also save the built configurations for which it's time to be
+            // rebuilt.
             //
-            auto i (configs.begin ());
-            auto e (configs.end ());
-
-            for (;
-                 i != e &&
-                 exclude (pc, p->builds, p->constraints, *i->second.config);
-                 ++i) ;
+            config_machines configs (conf_machines); // Make copy for this pkg.
+            auto pkg_builds (bld_prep_query.execute ());
 
-            if (i != e)
+            for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i)
             {
-              config_machine& cm (i->second);
-              machine_header_manifest& mh (*cm.machine);
-
-              build_id bid (move (id),
-                            cm.config->target,
-                            cm.config->name,
-                            move (pkg_config_name),
-                            move (tqm.toolchain_name),
-                            toolchain_version);
-
-              shared_ptr<build> b (build_db_->find<build> (bid));
-              optional<string> cl (challenge ());
+              auto j (
+                configs.find (build_target_config_id {i->id.target,
+                      i->id.target_config_name}));
 
-              shared_ptr<build_tenant> t (
-                build_db_->load<build_tenant> (bid.package.tenant));
-
-              // Move the interactive build login information into the build
-              // object, if the package to be built interactively.
+              // Outdated configurations are already excluded with the
+              // database query.
               //
-              optional<string> login (t->interactive
-                                      ? move (tqm.interactive_login)
-                                      : nullopt);
+              assert (j != configs.end ());
+              configs.erase (j);
 
-              // If build configuration doesn't exist then create the new one
-              // and persist. Otherwise put it into the building state, refresh
-              // the timestamp and update.
-              //
-              if (b == nullptr)
+              if (i->state == build_state::built)
               {
-                b = make_shared<build> (move (bid.package.tenant),
-                                        move (bid.package.name),
-                                        move (bp.version),
-                                        move (bid.target),
-                                        move (bid.target_config_name),
-                                        move (bid.package_config_name),
-                                        move (bid.toolchain_name),
-                                        move (toolchain_version),
-                                        move (login),
-                                        move (agent_fp),
-                                        move (cl),
-                                        mh.name,
-                                        move (mh.summary),
-                                        controller_checksum (*cm.config),
-                                        machine_checksum (*cm.machine));
-
-                build_db_->persist (b);
+                assert (i->force != force_state::forcing);
+
+                if (needs_rebuild (*i))
+                  rebuilds.emplace_back (i.load ());
               }
-              else
-              {
-                // The build configuration is in the building state.
-                //
-                // Note that in both cases we keep the status intact to be
-                // able to compare it with the final one in the result request
-                // handling in order to decide if to send the notification
-                // email. The same is true for the forced flag (in the sense
-                // that we don't set the force state to unforced).
-                //
-                assert (b->state == build_state::building);
+            }
 
-                b->state = build_state::building;
-                b->interactive = move (login);
+            if (!configs.empty ())
+            {
+              // Find the first build configuration that is not excluded by
+              // the package configuration.
+              //
+              auto i (configs.begin ());
+              auto e (configs.end ());
 
-                // Switch the force state not to reissue the task after the
-                // forced rebuild timeout. Note that the result handler will
-                // still recognize that the rebuild was forced.
-                //
-                if (b->force == force_state::forcing)
-                  b->force = force_state::forced;
+              for (;
+                   i != e &&
+                   exclude (pc, p->builds, p->constraints, *i->second.config);
+                   ++i) ;
 
-                b->agent_fingerprint = move (agent_fp);
-                b->agent_challenge = move (cl);
-                b->machine = mh.name;
-                b->machine_summary = move (mh.summary);
+              if (i != e)
+              {
+                config_machine& cm (i->second);
+                machine_header_manifest& mh (*cm.machine);
 
-                string ccs (controller_checksum (*cm.config));
-                string mcs (machine_checksum (*cm.machine));
+                build_id bid (move (id),
+                              cm.config->target,
+                              cm.config->name,
+                              move (pkg_config_name),
+                              move (tqm.toolchain_name),
+                              toolchain_version);
 
-                // Issue the hard rebuild if it is forced or the configuration
-                // or machine has changed.
-                //
-                if (b->hard_timestamp <= hard_rebuild_expiration ||
-                    b->force == force_state::forced              ||
-                    b->controller_checksum != ccs                ||
-                    b->machine_checksum != mcs)
-                  convert_to_hard (b);
+                shared_ptr<build> b (build_db_->find<build> (bid));
+                optional<string> cl (challenge ());
 
-                b->controller_checksum = move (ccs);
-                b->machine_checksum    = move (mcs);
+                shared_ptr<build_tenant> t (
+                  build_db_->load<build_tenant> (bid.package.tenant));
 
-                b->timestamp = system_clock::now ();
+                // Move the interactive build login information into the build
+                // object, if the package to be built interactively.
+                //
+                optional<string> login (t->interactive
+                                        ? move (tqm.interactive_login)
+                                        : nullopt);
 
-                build_db_->update (b);
-              }
+                // If build configuration doesn't exist then create the new
+                // one and persist. Otherwise put it into the building state,
+                // refresh the timestamp and update.
+                //
+                if (b == nullptr)
+                {
+                  b = make_shared<build> (move (bid.package.tenant),
+                                          move (bid.package.name),
+                                          move (bp.version),
+                                          move (bid.target),
+                                          move (bid.target_config_name),
+                                          move (bid.package_config_name),
+                                          move (bid.toolchain_name),
+                                          move (toolchain_version),
+                                          move (login),
+                                          move (agent_fp),
+                                          move (cl),
+                                          mh.name,
+                                          move (mh.summary),
+                                          controller_checksum (*cm.config),
+                                          machine_checksum (*cm.machine));
+
+                  build_db_->persist (b);
+                }
+                else
+                {
+                  // The build configuration is in the building state.
+                  //
+                  // Note that in both cases we keep the status intact to be
+                  // able to compare it with the final one in the result
+                  // request handling in order to decide if to send the
+                  // email. The same is true for the forced flag (in the sense
+                  // that we don't set the force state to unforced).
+                  //
+                  assert (b->state == build_state::building);
+
+                  b->state = build_state::building;
+                  b->interactive = move (login);
+
+                  // Switch the force state not to reissue the task after the
+                  // forced rebuild timeout. Note that the result handler will
+                  // still recognize that the rebuild was forced.
+                  //
+                  if (b->force == force_state::forcing)
+                    b->force = force_state::forced;
+
+                  b->agent_fingerprint = move (agent_fp);
+                  b->agent_challenge = move (cl);
+                  b->machine = mh.name;
+                  b->machine_summary = move (mh.summary);
+
+                  string ccs (controller_checksum (*cm.config));
+                  string mcs (machine_checksum (*cm.machine));
+
+                  // Issue the hard rebuild if it is forced or the
+                  // configuration or machine has changed.
+                  //
+                  if (b->hard_timestamp <= hard_rebuild_expiration ||
+                      b->force == force_state::forced              ||
+                      b->controller_checksum != ccs                ||
+                      b->machine_checksum != mcs)
+                    convert_to_hard (b);
+
+                  b->controller_checksum = move (ccs);
+                  b->machine_checksum    = move (mcs);
+
+                  b->timestamp = system_clock::now ();
+
+                  build_db_->update (b);
+                }
+
+                // Archive an interactive tenant.
+                //
+                if (t->interactive)
+                {
+                  t->archived = true;
+                  build_db_->update (t);
+                }
 
-              // Archive an interactive tenant.
-              //
-              if (t->interactive)
-              {
-                t->archived = true;
-                build_db_->update (t);
+                // Finally, prepare the task response manifest.
+                //
+                tsm = task (move (b), move (p), move (pc), move (t), cm);
+                break; // Bail out from the package configurations loop.
               }
-
-              // Finally, prepare the task response manifest.
-              //
-              tsm = task (move (b), move (p), move (pc), move (t), cm);
-              break; // Bail out from the package configurations loop.
             }
           }
+
+          // If the task response manifest is prepared, then bail out from the
+          // package loop, commit the transaction and respond.
+          //
+          if (!tsm.session.empty ())
+            break;
         }
 
-        // If the task response manifest is prepared, then bail out from the
-        // package loop, commit the transaction and respond.
-        //
-        if (!tsm.session.empty ())
-          break;
+        t.commit ();
       }
 
-      t.commit ();
-    }
-
-    // If we don't have an unbuilt package, then let's see if we have a build
-    // configuration to rebuild.
-    //
-    if (tsm.session.empty () && !rebuilds.empty ())
-    {
-      // Sort the configuration rebuild list with the following sort priority:
-      //
-      // 1: force state
-      // 2: overall status
-      // 3: timestamp (less is preferred)
+      // If we don't have an unbuilt package, then let's see if we have a
+      // build configuration to rebuild.
       //
-      auto cmp = [] (const shared_ptr<build>& x, const shared_ptr<build>& y)
+      if (tsm.session.empty () && !rebuilds.empty ())
       {
-        if (x->force != y->force)
-          return x->force > y->force;       // Forced goes first.
+        // Sort the configuration rebuild list with the following sort
+        // priority:
+        //
+        // 1: force state
+        // 2: overall status
+        // 3: timestamp (less is preferred)
+        //
+        auto cmp = [] (const shared_ptr<build>& x, const shared_ptr<build>& y)
+        {
+          if (x->force != y->force)
+            return x->force > y->force;       // Forced goes first.
 
-        assert (x->status && y->status);    // Both built.
+          assert (x->status && y->status);    // Both built.
 
-        if (x->status != y->status)
-          return x->status > y->status;     // Larger status goes first.
+          if (x->status != y->status)
+            return x->status > y->status;     // Larger status goes first.
 
-        return x->timestamp < y->timestamp; // Older goes first.
-      };
+          return x->timestamp < y->timestamp; // Older goes first.
+        };
 
-      sort (rebuilds.begin (), rebuilds.end (), cmp);
+        sort (rebuilds.begin (), rebuilds.end (), cmp);
 
-      optional<string> cl (challenge ());
+        optional<string> cl (challenge ());
 
-      // Pick the first build configuration from the ordered list.
-      //
-      // Note that the configurations and packages may not match the required
-      // criteria anymore (as we have committed the database transactions that
-      // were used to collect this data) so we recheck. If we find one that
-      // matches then put it into the building state, refresh the timestamp and
-      // update. Note that we don't amend the status and the force state to
-      // have them available in the result request handling (see above).
-      //
-      for (auto& b: rebuilds)
-      {
-        try
+        // Pick the first build configuration from the ordered list.
+        //
+        // Note that the configurations and packages may not match the
+        // required criteria anymore (as we have committed the database
+        // transactions that were used to collect this data) so we recheck. If
+        // we find one that matches then put it into the building state,
+        // refresh the timestamp and update. Note that we don't amend the
+        // status and the force state to have them available in the result
+        // request handling (see above).
+        //
+        for (auto& b: rebuilds)
         {
-          transaction t (build_db_->begin ());
-
-          b = build_db_->find<build> (b->id);
-
-          if (b != nullptr                   &&
-              b->state == build_state::built &&
-              needs_rebuild (*b))
+          try
           {
-            auto i (conf_machines.find (
-                      build_target_config_id {b->target,
-                                              b->target_config_name}));
+            transaction t (build_db_->begin ());
 
-            // Only actual package configurations are loaded (see above).
-            //
-            assert (i != conf_machines.end ());
-            const config_machine& cm (i->second);
+            b = build_db_->find<build> (b->id);
 
-            // Rebuild the package if still present, is buildable, doesn't
-            // exclude the configuration, and matches the request's
-            // interactive mode.
-            //
-            // Note that while change of the latter seems rather far fetched,
-            // let's check it for good measure.
-            //
-            shared_ptr<build_package> p (
-              build_db_->find<build_package> (b->id.package));
-
-            shared_ptr<build_tenant> t (
-              p != nullptr
-              ? build_db_->load<build_tenant> (p->id.tenant)
-              : nullptr);
-
-            build_package_config* pc (p != nullptr
-                                      ? find (b->package_config_name,
-                                              p->configs)
-                                      : nullptr);
-
-            if (pc != nullptr                          &&
-                p->buildable                           &&
-                (imode == interactive_mode::both ||
-                 (t->interactive.has_value () ==
-                  (imode == interactive_mode::true_))) &&
-                !exclude (*pc, p->builds, p->constraints, *cm.config))
+            if (b != nullptr                   &&
+                b->state == build_state::built &&
+                needs_rebuild (*b))
             {
-              assert (b->status);
+              auto i (conf_machines.find (
+                        build_target_config_id {
+                          b->target, b->target_config_name}));
 
-              b->state = build_state::building;
+              // Only actual package configurations are loaded (see above).
+              //
+              assert (i != conf_machines.end ());
+              const config_machine& cm (i->second);
 
-              // Save the interactive build login information into the build
-              // object, if the package to be built interactively.
+              // Rebuild the package if still present, is buildable, doesn't
+              // exclude the configuration, and matches the request's
+              // interactive mode.
               //
-              // Can't move from, as may need it on the next iteration.
+              // Note that while change of the latter seems rather far fetched,
+              // let's check it for good measure.
               //
-              b->interactive = t->interactive
-                               ? tqm.interactive_login
-                               : nullopt;
+              shared_ptr<build_package> p (
+                build_db_->find<build_package> (b->id.package));
 
-              // Can't move from, as may need them on the next iteration.
-              //
-              b->agent_fingerprint = agent_fp;
-              b->agent_challenge = cl;
+              shared_ptr<build_tenant> t (
+                p != nullptr
+                ? build_db_->load<build_tenant> (p->id.tenant)
+                : nullptr);
+
+              build_package_config* pc (p != nullptr
+                                        ? find (b->package_config_name,
+                                                p->configs)
+                                        : nullptr);
+
+              if (pc != nullptr                          &&
+                  p->buildable                           &&
+                  (imode == interactive_mode::both ||
+                   (t->interactive.has_value () ==
+                    (imode == interactive_mode::true_))) &&
+                  !exclude (*pc, p->builds, p->constraints, *cm.config))
+              {
+                assert (b->status);
 
-              const machine_header_manifest& mh (*cm.machine);
-              b->machine = mh.name;
-              b->machine_summary = mh.summary;
+                b->state = build_state::building;
 
-              // Issue the hard rebuild if the timeout expired, rebuild is
-              // forced, or the configuration or machine has changed.
-              //
-              // Note that we never reset the build status (see above for the
-              // reasoning).
-              //
-              string ccs (controller_checksum (*cm.config));
-              string mcs (machine_checksum (*cm.machine));
+                // Save the interactive build login information into the build
+                // object, if the package to be built interactively.
+                //
+                // Can't move from, as may need it on the next iteration.
+                //
+                b->interactive = t->interactive
+                  ? tqm.interactive_login
+                  : nullopt;
+
+                // Can't move from, as may need them on the next iteration.
+                //
+                b->agent_fingerprint = agent_fp;
+                b->agent_challenge = cl;
+
+                const machine_header_manifest& mh (*cm.machine);
+                b->machine = mh.name;
+                b->machine_summary = mh.summary;
 
-              if (b->hard_timestamp <= hard_rebuild_expiration ||
-                  b->force == force_state::forced              ||
-                  b->controller_checksum != ccs                ||
-                  b->machine_checksum != mcs)
-                convert_to_hard (b);
+                // Issue the hard rebuild if the timeout expired, rebuild is
+                // forced, or the configuration or machine has changed.
+                //
+                // Note that we never reset the build status (see above for
+                // the reasoning).
+                //
+                string ccs (controller_checksum (*cm.config));
+                string mcs (machine_checksum (*cm.machine));
+
+                if (b->hard_timestamp <= hard_rebuild_expiration ||
+                    b->force == force_state::forced              ||
+                    b->controller_checksum != ccs                ||
+                    b->machine_checksum != mcs)
+                  convert_to_hard (b);
 
-              b->controller_checksum = move (ccs);
-              b->machine_checksum    = move (mcs);
+                b->controller_checksum = move (ccs);
+                b->machine_checksum    = move (mcs);
 
-              b->timestamp = system_clock::now ();
+                b->timestamp = system_clock::now ();
 
-              build_db_->update (b);
+                build_db_->update (b);
 
-              tsm = task (move (b), move (p), move (*pc), move (t), cm);
+                tsm = task (move (b), move (p), move (*pc), move (t), cm);
+              }
             }
+
+            t.commit ();
           }
+          catch (const odb::deadlock&) {} // Just try with the next rebuild.
 
-          t.commit ();
+          // If the task response manifest is prepared, then bail out from the
+          // package configuration rebuilds loop and respond.
+          //
+          if (!tsm.session.empty ())
+            break;
         }
-        catch (const odb::deadlock&) {} // Just try with the next rebuild.
-
-        // If the task response manifest is prepared, then bail out from the
-        // package configuration rebuilds loop and respond.
-        //
-        if (!tsm.session.empty ())
-          break;
       }
     }
   }
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index b0ff61a..bad13c1 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -825,8 +825,8 @@ handle (request& rq, response& rs)
         //
         size_t npos (0);
 
-        size_t ncur = build_db_->query_value<package_build_count> (
-          build_query<package_build_count> (&conf_ids, bld_params, tn));
+        size_t ncur (build_db_->query_value<package_build_count> (
+          build_query<package_build_count> (&conf_ids, bld_params, tn)));
 
         // From now we will be using specific values for the below filters for
         // each build database query. Note that the toolchain is the only
diff --git a/mod/module.cli b/mod/module.cli
index 7c6e0b4..3d4ea00 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -105,14 +105,14 @@ namespace brep
       {
         "<user>",
         "Package database login user name. If not specified, then operating
-         system (login) name is used. See also \c{package-db-role}."
+         system (login) name is used. See also \cb{package-db-role}."
       }
 
       string package-db-role = "brep"
       {
         "<user>",
         "Package database execution user name. If not empty then the login
-         user will be switched (with \c{SET ROLE}) to this user prior to
+         user will be switched (with \cb{SET ROLE}) to this user prior to
          executing any statements. If not specified, then \cb{brep} is used."
       }
 
@@ -214,9 +214,9 @@ namespace brep
          \cb{build-alt-soft-rebuild-start} and
          \cb{build-alt-soft-rebuild-stop} options. Must be specified in
          seconds. Default is the time interval length plus
-         \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is
-         greater than 24 hours (thus the rebuild is only triggered within the
-         last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)."
+         \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout
+         is greater than 24 hours (thus the rebuild is only triggered within
+         the last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)."
       }
 
       duration build-alt-soft-rebuild-start
@@ -286,14 +286,14 @@ namespace brep
       {
         "<user>",
         "Build database login user name. If not specified, then operating
-         system (login) name is used. See also \c{build-db-role}."
+         system (login) name is used. See also \cb{build-db-role}."
       }
 
       string build-db-role = "brep"
       {
         "<user>",
         "Build database execution user name. If not empty then the login
-         user will be switched (with \c{SET ROLE}) to this user prior to
+         user will be switched (with \cb{SET ROLE}) to this user prior to
          executing any statements. If not specified, then \cb{brep} is used."
       }
 
@@ -452,6 +452,15 @@ namespace brep
          request is considered invalid, unless no expressions are specified.
          Repeat this option to specify multiple expressions."
       }
+
+      build_order build-package-order = build_order::stable
+      {
+        "<order>",
+        "Order in which packages are considered for build. The valid <order>
+         values are \cb{stable} and \cb{random}. If not specified, then
+         \cb{stable} is assumed. Note that interactive builds are always
+         preferred."
+      }
     };
 
     class build_result: build, package_db, build_db, handler
@@ -663,8 +672,8 @@ namespace brep
       {
         "<service>",
         "The default view to display for the global repository root. The
-         <service> argument is one of the supported services (\c{packages},
-         \c{builds}, \c{submit}, \c{ci}, etc). The default service is
+         <service> argument is one of the supported services (\cb{packages},
+         \cb{builds}, \cb{submit}, \cb{ci}, etc). The default service is
          packages."
       }
 
@@ -672,8 +681,8 @@ namespace brep
       {
         "<service>",
         "The default view to display for the tenant repository root. The
-         <service> argument is one of the supported services (\c{packages},
-         \c{builds}, \c{submit}, \c{ci}, etc). The default service is
+         <service> argument is one of the supported services (\cb{packages},
+         \cb{builds}, \cb{submit}, \cb{ci}, etc). The default service is
          packages."
       }
     };
diff --git a/mod/options-types.hxx b/mod/options-types.hxx
index 4aa573f..dc1047d 100644
--- a/mod/options-types.hxx
+++ b/mod/options-types.hxx
@@ -25,6 +25,12 @@ namespace brep
     page_menu () = default;
     page_menu (string b, string l): label (move (b)), link (move (l)) {}
   };
+
+  enum class build_order
+  {
+    stable,
+    random
+  };
 }
 
 #endif // MOD_OPTIONS_TYPES_HXX
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index 3868c32..fd087bc 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -217,5 +217,25 @@ namespace brep
         throw invalid_value (o, v, os.str ());
       }
     }
+
+    // Parse build_order.
+    //
+    void parser<build_order>::
+    parse (build_order& x, bool& xs, scanner& s)
+    {
+      xs = true;
+      const char* o (s.next ());
+
+      if (!s.more ())
+        throw missing_value (o);
+
+      const string v (s.next ());
+      if (v == "stable")
+        x = build_order::stable;
+      else if (v == "random")
+        x = build_order::random;
+      else
+        throw invalid_value (o, v);
+    }
   }
 }
diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx
index 05a7263..c6da9f6 100644
--- a/mod/types-parsers.hxx
+++ b/mod/types-parsers.hxx
@@ -84,6 +84,13 @@ namespace brep
       static void
       parse (pair<std::regex, string>&, bool&, scanner&);
     };
+
+    template <>
+    struct parser<build_order>
+    {
+      static void
+      parse (build_order&, bool&, scanner&);
+    };
   }
 }