Add --instance-max agent option

This is the ground work for the task priority/interrupt support.

3 files changed, 105 insertions, 32 deletions

diff --git a/bbot/agent/agent.cli b/bbot/agent/agent.cli
index aa7eb59..3d028fd 100644
--- a/bbot/agent/agent.cli
+++ b/bbot/agent/agent.cli
@@ -111,6 +111,20 @@ namespace bbot
        network ports, interfaces, etc."
     }
 
+    uint16_t --instance-max = 0
+    {
+      "<num>",
+      "Maximum number of instances that can perform tasks concurrently. If the
+       number of instances that have been started is greater than this number
+       (normally by just one), then when the maximum number of tasks is
+       already being performed, the extra instances operate in the \i{priority
+       monitor} mode: they only query controller URLs with priorities higher
+       than of the existing tasks and can only perform a task by interrupting
+       one of them. If the maximum number of instances is \cb{0} (default),
+       then it is assumed the number of instances started is the maximum
+       number, essentially disabling the priority monitor functionality."
+    }
+
     size_t --cpu = 1
     {
       "<num>",
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 8f54346..a8b7c77 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -61,7 +61,8 @@ namespace bbot
   standard_version tc_ver;
   string           tc_id;
 
-  uint16_t inst;
+  uint16_t inst;     // 1-based.
+  uint16_t inst_max; // 0 if priority monitoring is disabled.
 
   uint16_t offset;
 
@@ -682,10 +683,13 @@ snapshot_path (const dir_path& tp)
 // (re-)bootstrapping them if necessary.
 //
 // Note that this function returns both machines that this process managed to
-// lock as well as the machines locked by other processes (except those that
-// are being bootstrapped), in case the caller needs to interrupt one of them
-// for a higher-priority task. In the latter case, the manifest only has the
-// machine_manifest information.
+// lock as well as the machines locked by other processes (including those
+// that are being bootstrapped), in case the caller needs to interrupt one of
+// them for a higher-priority task. In the latter case, the manifest is empty
+// if the machine is being bootstrapped and only has the machine_manifest
+// information otherwise. (The bootstrapped machines have to be returned to
+// get the correct count of currently active instances for the inst_max
+// comparison.)
 //
 struct bootstrapped_machine
 {
@@ -836,7 +840,7 @@ try
 
             none = none && sp.empty ();
 
-            // Try to lock the machine, skipping it if being bootstrapped.
+            // Try to lock the machine.
             //
             machine_lock ml (lock_machine (tl, tp));
 
@@ -844,37 +848,39 @@ try
             {
               // @@ TMP: restore l4 tracing.
 
-              if (!ml.prio) // Being bootstrapped.
+              machine_manifest mm;
+              if (ml.prio)
               {
-                l1 ([&]{trace << "skipping " << md << ": being bootstrapped "
-                              << "by " << ml.pid;});
-                break;
-              }
+                // Get the machine manifest (subset of the steps performed for
+                // the locked case below).
+                //
+                // Note that it's possible the machine we get is not what was
+                // originally locked by the other process (e.g., it has been
+                // upgraded since). It's also possible that if and when we
+                // interrupt and lock this machine, it will be a different
+                // machine (e.g., it has been upgraded since we read this
+                // machine manifest). To deal with all of that we will be
+                // reloading this information if/when we acquire the lock to
+                // this machine.
+                //
+                if (sp.empty ())
+                {
+                  l3 ([&]{trace << "skipping " << md << ": no subvolume link";});
+                  break;
+                }
 
-              // Get the machine manifest (subset of the steps performed for
-              // the locked case below).
-              //
-              // Note that it's possible the machine we get is not what was
-              // originally locked by the other process (e.g., it has been
-              // upgraded since). It's also possible that if and when we
-              // interrupt and lock this machine, it will be a different
-              // machine (e.g., it has been upgraded since we read this
-              // machine manifest). To deal with all of that we will be
-              // reloading this information if/when we acquire the lock to
-              // this machine.
-              //
-              if (sp.empty ())
+                l1 ([&]{trace << "keeping " << md << ": locked by " << ml.pid
+                              << " with priority " << *ml.prio;});
+
+                mm = parse_manifest<machine_manifest> (
+                  sp / "manifest", "machine");
+              }
+              else // Being bootstrapped.
               {
-                l3 ([&]{trace << "skipping " << md << ": no subvolume link";});
-                break;
+                l1 ([&]{trace << "keeping " << md << ": being bootstrapped "
+                              << "by " << ml.pid;});
               }
 
-              l1 ([&]{trace << "keeping " << md << ": locked by " << ml.pid
-                            << " with priority " << *ml.prio;});
-
-              auto mm (
-                parse_manifest<machine_manifest> (sp / "manifest", "machine"));
-
               // Add the machine to the lists and bail out.
               //
               r.push_back (bootstrapped_machine {
@@ -1625,6 +1631,8 @@ try
   if (inst == 0 || inst > 99)
     fail << "invalid --instance value " << inst;
 
+  inst_max = ops.instance_max ();
+
   offset = (tc_num - 1) * 100 + inst;
 
   // Controller URLs.
@@ -1699,6 +1707,9 @@ try
       info << "toolchain id   " << tc_id <<
       info << "instance  num  " << inst;
 
+    if (inst_max != 0)
+      dr << info << "instance  max  " << inst_max;
+
     for (const string& u: controllers)
       dr << info << "controller url " << u;
   }
@@ -1761,6 +1772,49 @@ try
     toolchain_lock& tl (er.first);
     bootstrapped_machines& ms (er.second);
 
+    // Determine if we should operate in the priority monitor mode and, if so,
+    // the lower bound on the priorities that we should consider.
+    //
+    optional<uint64_t> prio_mon;
+    if (inst_max != 0)
+    {
+      uint16_t           busy (0);
+      optional<uint64_t> prio;
+
+      for (const bootstrapped_machine& m: ms)
+      {
+        if (!m.lock.locked ())
+        {
+          ++busy;
+          if (m.lock.prio && (!prio || *m.lock.prio < *prio))
+            prio = *m.lock.prio;
+        }
+      }
+
+      assert (busy <= inst_max);
+
+      if (busy == inst_max)
+      {
+        if (!prio) // All being bootstrapped.
+        {
+          sleep = rand_sleep ();
+          continue;
+        }
+
+        prio_mon = *prio;
+      }
+    }
+
+    // @@ For now bail out if in the priority monitor mode.
+    //
+    if (prio_mon)
+    {
+      l1 ([&]{trace << "priority monitor, lower bound " << *prio_mon;});
+
+      sleep = rand_sleep () / 2;
+      continue;
+    }
+
     // Prepare task request.
     //
     task_request_manifest tq {
@@ -1779,6 +1833,8 @@ try
     {
       // @@ For now skip machines locked by other processes.
       //
+      // @@ Note: skip machines being bootstrapped.
+      //
       if (m.lock.locked ())
         tq.machines.emplace_back (m.manifest.machine.id,
                                   m.manifest.machine.name,
diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service
index e938126..18b7c9e 100644
--- a/bbot/bbot-agent@.service
+++ b/bbot/bbot-agent@.service
@@ -38,6 +38,8 @@ Environment=TOOLCHAIN_NUM=1
 Environment=TOOLCHAIN_VER=
 Environment=TOOLCHAIN_ID=
 
+Environment=INSTANCE_MAX=0
+
 Environment="CONTROLLER_URL="
 Environment="CONTROLLER_TRUST="
 
@@ -63,6 +65,7 @@ ExecStart=/build/bots/default/bin/bbot-agent \
   --toolchain-num ${TOOLCHAIN_NUM} \
   --toolchain-ver ${TOOLCHAIN_VER} \
   --toolchain-id ${TOOLCHAIN_ID} \
+  --instance-max ${INSTANCE_MAX} \
   --instance %i \
   $CONTROLLER_TRUST \
   $CONTROLLER_URL