Add support for bbot.bindist.upload and bbot.upload steps in worker

2 files changed, 106 insertions, 8 deletions

diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 7dc6794..ed90da0 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -840,6 +840,7 @@ try
 
   path tf (gd / "task.manifest");       // Task manifest file.
   path rf (pd / "result.manifest.lz4"); // Result manifest file.
+  path af (pd / "upload.tar");          // Archive of build artifacts to upload.
 
   serialize_manifest (tm, tf, "task");
 
@@ -862,10 +863,43 @@ try
     }
 
     r = parse_manifest<result_manifest> (rf, "result");
+
+    // If archive of build artifacts is present, then just list its content as
+    // a sanity check.
+    //
+    bool err (!r.status);
+    if (!err && file_exists (af))
+    {
+      try
+      {
+        auto_fd null (fdopen_null ());
+
+        // Redirect stdout to stderr if the command is traced and to /dev/null
+        // otherwise.
+        //
+        process_exit pe (
+          process_run_callback (
+            trace,
+            null.get (), // Don't expect to read from stdin.
+            verb >= 3 ? 2 : null.get (),
+            2,
+            "tar",
+            "-tf", af));
+
+        if (!pe)
+          fail << "tar " << pe;
+      }
+      catch (const process_error& e)
+      {
+        fail << "unable execute tar: " << e;
+      }
+    }
   }
   else
   {
     try_rmfile (rf);
+    try_rmfile (af);
+    try_rmdir_r (pd / dir_path ("upload"));
 
     // <name>-<toolchain>-<xxx>
     //
@@ -985,11 +1019,12 @@ try
 
         l3 ([&]{trace << "completed startup in " << startup_to - to << "s";});
 
-        // Next the worker builds things and then uploads the result manifest.
-        // So on our side we serve TFTP requests while checking for the
-        // manifest file. To workaround some obscure filesystem races (the
-        // file's mtime/size is updated several seconds later; maybe tmpfs
-        // issue?), we periodically re-check.
+        // Next the worker builds things and then uploads optional archive of
+        // build artifacts and the result manifest afterwards. So on our side
+        // we serve TFTP requests while checking for the manifest file. To
+        // workaround some obscure filesystem races (the file's mtime/size is
+        // updated several seconds later; maybe tmpfs issue?), we periodically
+        // re-check.
         //
         for (to = build_to; to != 0; )
         {
@@ -1015,14 +1050,77 @@ try
 
           // Parse the result manifest.
           //
+          optional<result_manifest> rm;
+
           try
           {
-            r = parse_manifest<result_manifest> (rf, "result", false);
+            rm = parse_manifest<result_manifest> (rf, "result", false);
           }
           catch (const failed&)
           {
             r.status = result_status::abnormal; // Soft-fail below.
           }
+
+          // Upload the build artifacts if the result manifest is parsed
+          // successfully, the result status is not an error, and upload.tar
+          // exists.
+          //
+          // Note that while the worker doesn't upload the build artifacts
+          // archives on errors, there can be the case when the error occurred
+          // while uploading the archive and so the partially uploaded file
+          // may exist. Thus, we check if the result status is not an error.
+          //
+          bool err (!rm || !rm->status);
+          if (!err && file_exists (af))
+          {
+            // Extract the build artifacts from the archive and upload them to
+            // the controller. On error keep the result status as abort for
+            // transient errors (network failure, etc) and set it to abnormal
+            // otherwise (for subsequent machine suspension and
+            // investigation).
+            //
+            optional<bool> err; // True if the error is transient.
+
+            try
+            {
+              process_exit pe (
+                process_run_callback (
+                  trace,
+                  fdopen_null (), // Don't expect to read from stdin.
+                  2,              // Redirect stdout to stderr.
+                  2,
+                  "tar",
+                  "-xf", af,
+                  "-C", pd));
+
+              if (!pe)
+              {
+                err = false;
+                error << "tar " << pe;
+              }
+            }
+            catch (const process_error& e)
+            {
+              err = false;
+              error << "unable execute tar: " << e;
+            }
+
+            if (!err)
+            {
+              // @@ Upload the extracted artifacts.
+            }
+
+            if (err)
+            {
+              if (!*err)                            // Non-transient?
+                r.status = result_status::abnormal; // Soft-fail below.
+
+              rm = nullopt; // Drop the parsed manifest.
+            }
+          }
+
+          if (rm)
+            r = move (*rm);
         }
         else
         {
@@ -1089,7 +1187,7 @@ handle_signal (int sig)
   }
 }
 
-static const string agent_checksum ("1"); // Logic version.
+static const string agent_checksum ("2"); // Logic version.
 
 int
 main (int argc, char* argv[])
diff --git a/bbot/agent/tftp.cxx b/bbot/agent/tftp.cxx
index b671060..58aaabc 100644
--- a/bbot/agent/tftp.cxx
+++ b/bbot/agent/tftp.cxx
@@ -126,7 +126,7 @@ namespace bbot
                 ops.tftp ());
 
         // This is not really accurate since tftpd will, for example, serve
-        // an upload request until it is complete. But it's close anough for
+        // an upload request until it is complete. But it's close enough for
         // our needs.
         //
         sec -= (inc - static_cast<size_t> (timeout.tv_sec));