Add support for controller request retries, connection timeout

3 files changed, 36 insertions, 4 deletions

diff --git a/bbot/agent/agent.cli b/bbot/agent/agent.cli
index a5dbe01..0f306fb 100644
--- a/bbot/agent/agent.cli
+++ b/bbot/agent/agent.cli
@@ -150,7 +150,8 @@ namespace bbot
     size_t --bootstrap-retries = 2
     {
       "<num>",
-      "Number of time to retry a mis-booted bootstrap, 2 by default."
+      "Number of times to retry a mis-booted bootstrap, 2 (3 attempts total)
+       by default."
     }
 
     size_t --build-timeout = 1800
@@ -163,7 +164,15 @@ namespace bbot
     size_t --build-retries = 2
     {
       "<num>",
-      "Number of time to retry a mis-booted build, 2 by default."
+      "Number of times to retry a mis-booted build, 2 (3 attempts total) by
+       default."
+    }
+
+    size_t --connect-timeout = 60
+    {
+      "<sec>",
+      "Maximum number of seconds to wait for controller request connection,
+       60 (1 minute) by default."
     }
 
     size_t --request-timeout = 300
@@ -173,6 +182,17 @@ namespace bbot
        300 (5 minutes) by default."
     }
 
+    size_t --request-retries = 4
+    {
+      "<num>",
+      "Number of times to retry a controller request, 4 (5 attempts total) by
+       default. Note that both the total time for all retries as well as the
+       time of each retry are limited by the same \c{--request-timeout}
+       value. This means that a successful request may take up to twice as
+       long if a connection was established at the end of the retry window
+       and took just as long to complete."
+    }
+
     uint16_t --verbose = 1
     {
       "<level>",
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index d345472..c8e316a 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -1029,7 +1029,10 @@ try
                        curl::post,
                        u,
                        "--header", "Content-Type: text/manifest",
-                       "--max-time", ops.request_timeout ());
+                       "--retry", ops.request_retries (),
+                       "--retry-max-time", ops.request_timeout (),
+                       "--max-time", ops.request_timeout (),
+                       "--connect-timeout", ops.connect_timeout ());
 
           // This is tricky/hairy: we may fail hard parsing the output before
           // seeing that curl exited with an error and failing softly.
@@ -1184,7 +1187,10 @@ try
                      curl::post,
                      u,
                      "--header", "Content-Type: text/manifest",
-                     "--max-time", ops.request_timeout ());
+                     "--retry", ops.request_retries (),
+                     "--retry-max-time", ops.request_timeout (),
+                     "--max-time", ops.request_timeout (),
+                     "--connect-timeout", ops.connect_timeout ());
 
         // This is tricky/hairy: we may fail hard writing the input before
         // seeing that curl exited with an error and failing softly.
diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service
index 41486e2..3f41abc 100644
--- a/bbot/bbot-agent@.service
+++ b/bbot/bbot-agent@.service
@@ -14,9 +14,13 @@ Environment=RAM=1048576
 
 Environment=BOOTSTRAP_TIMEOUT=900
 Environment=BOOTSTRAP_RETRIES=2
+
 Environment=BUILD_TIMEOUT=1800
 Environment=BUILD_RETRIES=2
+
+Environment=CONNECT_TIMEOUT=60
 Environment=REQUEST_TIMEOUT=300
+Environment=REQUEST_RETRIES=4
 
 Environment=TOOLCHAIN_NAME=%i
 Environment=TOOLCHAIN_NUM=1
@@ -35,7 +39,9 @@ ExecStart=/build/bots/%i/bin/bbot-agent --systemd-daemon \
   --bootstrap-retries ${BOOTSTRAP_RETRIES} \
   --build-timeout ${BUILD_TIMEOUT} \
   --build-retries ${BUILD_RETRIES} \
+  --connect-timeout ${CONNECT_TIMEOUT} \
   --request-timeout ${REQUEST_TIMEOUT} \
+  --request-retries ${REQUEST_RETRIES} \
   --toolchain-name ${TOOLCHAIN_NAME} \
   --toolchain-num ${TOOLCHAIN_NUM} \
   --toolchain-ver ${TOOLCHAIN_VER} \