Generate bindist-result.manifest if bbot.bindist.upload worker step is enabled

3 files changed, 198 insertions, 21 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index f9550f8..4895c76 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -17,10 +17,11 @@
 #include <libbutl/b.hxx>
 #include <libbutl/pager.hxx>
 #include <libbutl/prompt.hxx>
-#include <libbutl/utility.hxx>       // to_utf8(), eof()
+#include <libbutl/utility.hxx>              // to_utf8(), eof()
 #include <libbutl/timestamp.hxx>
 #include <libbutl/filesystem.hxx>
 #include <libbutl/string-parser.hxx>
+#include <libbutl/manifest-serializer.hxx>
 
 #include <libbutl/json/parser.hxx>
 
@@ -1189,12 +1190,15 @@ static const string worker_checksum ("5"); // Logic version.
 static int bbot::
 build (size_t argc, const char* argv[])
 {
-  using namespace bpkg;
+  using std::map;
+  using std::multimap;
 
   using string_parser::unquote;
 
-  using std::map;
-  using std::multimap;
+  using serializer    = manifest_serializer;
+  using serialization = manifest_serialization;
+
+  using namespace bpkg;
 
   tracer trace ("build");
 
@@ -3818,7 +3822,7 @@ build (size_t argc, const char* argv[])
       // Generate the binary distribution package.
       //
       // Note that if bbot.bindist.upload step is enabled, it makes sense to
-      // only copy all the generated binary distribution files to the
+      // only copy the generated binary distribution files to the
       // upload/bindist/<distribution>/ directory after the binary
       // distribution packages are testes, i.e. after the potential
       // bbot.sys-uninstall.* steps.
@@ -3828,20 +3832,32 @@ build (size_t argc, const char* argv[])
       // of the bpkg-pkg-bindist(1) man page. Note: needed later for
       // uninstall, upload.
       //
+      struct bindist_os_release
+      {
+        string name_id;
+        optional<string> version_id;
+      };
+
       struct bindist_file
       {
-        bbot::path path;    // Absolute and normalized.
-        string system_name;
+        string type;
+        bbot::path path;              // Absolute and normalized.
+        optional<string> system_name;
       };
 
       struct bindist_package
       {
+        string name;
+        string version;
+        optional<string> system_version;
         vector<bindist_file> files;
       };
 
       struct bindist_result_type
       {
         string distribution;
+        string architecture;
+        bindist_os_release os_release;
         bindist_package package;
         vector<bindist_package> dependencies;
       };
@@ -3958,6 +3974,36 @@ build (size_t argc, const char* argv[])
                                             move (d));
           };
 
+          // Parse bindist_os_release object.
+          //
+          auto parse_os_release = [&p] ()
+          {
+            // enter: after begin_object
+            // leave: after end_object
+
+            bindist_os_release r;
+
+            // Skip unknown/uninteresting members.
+            //
+            while (p.next_expect (event::name, event::end_object))
+            {
+              const string& n (p.name ());
+
+              if (n == "name_id")
+              {
+                r.name_id = p.next_expect_string ();
+              }
+              else if (n == "version_id")
+              {
+                r.version_id = p.next_expect_string ();
+              }
+              else
+                p.next_expect_value_skip ();
+            }
+
+            return r;
+          };
+
           // Parse a bindist_file object.
           //
           auto parse_file = [&p, &bad_json] ()
@@ -3973,7 +4019,11 @@ build (size_t argc, const char* argv[])
             {
               const string& n (p.name ());
 
-              if (n == "path")
+              if (n == "type")
+              {
+                r.type = p.next_expect_string ();
+              }
+              else if (n == "path")
               {
                 try
                 {
@@ -4011,7 +4061,19 @@ build (size_t argc, const char* argv[])
             {
               const string& n (p.name ());
 
-              if (n == "files")
+              if (n == "name")
+              {
+                r.name = p.next_expect_string ();
+              }
+              else if (n == "version")
+              {
+                r.version = p.next_expect_string ();
+              }
+              else if (n == "system_version")
+              {
+                r.system_version = p.next_expect_string ();
+              }
+              else if (n == "files")
               {
                 p.next_expect (event::begin_array);
 
@@ -4027,6 +4089,11 @@ build (size_t argc, const char* argv[])
 
           // Parse the bindist_result.
           //
+          // Note that if the bbot.bindist.upload step is enabled, then we
+          // require bindist_result.os_release.version_id to be present. This
+          // way the uploaded binary package can be published for a specific
+          // version of the distribution.
+          //
           p.next_expect (event::begin_object);
 
           while (p.next_expect (event::name, event::end_object))
@@ -4041,6 +4108,19 @@ build (size_t argc, const char* argv[])
                 bad_json ("expected distribution '" + distribution +
                           "' instead of '" + bindist_result.distribution + "'");
             }
+            else if (n == "architecture")
+            {
+              bindist_result.architecture = p.next_expect_string ();
+            }
+            else if (n == "os_release")
+            {
+              p.next_expect (event::begin_object);
+              bindist_result.os_release = parse_os_release ();
+
+              if (!bindist_result.os_release.version_id && bindist_upload)
+                bad_json ("version_id must be present if bbot.bindist.upload "
+                          "step is enabled");
+            }
             else if (n == "package")
             {
               p.next_expect (event::begin_object);
@@ -4131,7 +4211,7 @@ build (size_t argc, const char* argv[])
         {
           for (const bindist_file& f: bfs)
           {
-            if (!f.system_name.empty ())
+            if (f.system_name)
               pfs.push_back (f.path.string ().c_str ());
           }
         };
@@ -5205,8 +5285,8 @@ build (size_t argc, const char* argv[])
         {
           for (const bindist_file& f: bfs)
           {
-            if (!f.system_name.empty ())
-              pns.push_back (f.system_name.c_str ());
+            if (f.system_name)
+              pns.push_back (f.system_name->c_str ());
           }
         };
 
@@ -5352,8 +5432,10 @@ build (size_t argc, const char* argv[])
 
       // Prepare the bindist artifacts.
       //
-      // Move all the generated binary distribution files and
+      // Move the binary distribution files generated for the main package and
       // bindist-result.json to the upload/bindist/<distribution>/ directory.
+      // Also serialize the subset of the bindist result as
+      // bindist-result.manifest.
       //
       // Fail if the breakpoint refers to the bbot.bindist.upload step since
       // it has no specific command associated.
@@ -5401,18 +5483,79 @@ build (size_t argc, const char* argv[])
             cp_into (trace, &r.log, rp, d);
         };
 
-        auto move_files = [&mv] (const vector<bindist_file>& bfs)
+        // Main package files.
+        //
+        for (const bindist_file& f: bindist_result.package.files)
+          mv (f.path);
+
+        // Bindist result JSON.
+        //
+        mv (bindist_result_file);
+
+        // Bindist result manifest.
+        //
+        path mf (d / "bindist-result.manifest");
+
+        try
         {
-          for (const bindist_file& f: bfs)
-            mv (f.path);
-        };
+          ofdstream os (mf);
+          serializer s (os, mf.string ());
 
-        move_files (bindist_result.package.files);
+          // Serialize package manifest.
+          //
+          s.next ("", "1"); // Start of manifest.
 
-        for (const bindist_package& d: bindist_result.dependencies)
-          move_files (d.files);
+          s.next ("distribution", bindist_result.distribution);
+          s.next ("architecture", bindist_result.architecture);
 
-        mv (bindist_result_file);
+          s.next ("os-release-name-id", bindist_result.os_release.name_id);
+
+          // Should have failed earlier.
+          //
+          assert (bindist_result.os_release.version_id);
+
+          s.next ("os-release-version-id",
+                  *bindist_result.os_release.version_id);
+
+          s.next ("package-name", bindist_result.package.name);
+          s.next ("package-version", bindist_result.package.version);
+
+          if (bindist_result.package.system_version)
+            s.next ("package-system-version",
+                    *bindist_result.package.system_version);
+
+          s.next ("", "");  // End of manifest.
+
+          // Serialize package file manifests.
+          //
+          for (const bindist_file& f: bindist_result.package.files)
+          {
+            s.next ("", "1"); // Start of manifest.
+
+            s.next ("package-file-type", f.type);
+
+            // Note: the simple path representation is POSIX.
+            //
+            s.next ("package-file-path", f.path.leaf ().string ());
+
+            if (f.system_name)
+              s.next ("package-file-system-name", *f.system_name);
+
+            s.next ("", "");  // End of manifest.
+          }
+
+          s.next ("", ""); // End of stream.
+
+          os.close ();
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write to '" << mf << "': " << e;
+        }
+        catch (const serialization& e)
+        {
+          fail << "unable to serialize bindist result: " << e;
+        }
       }
 
       // Create the archive of the build artifacts for subsequent upload, if
diff --git a/doc/manual.cli b/doc/manual.cli
index f916049..eb48a31 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -2101,6 +2101,39 @@ shift
 exec \"$@\" cc config.c=\"gcc-9 $mode\" config.cxx=\"g++-9 $mode\"
 \
 
+\h2#arch-worker-bindist-result|Bindist Result Manifest|
+
+At the \c{bbot.bindist.upload} step the worker also creates the
+\c{bindist-result.json} and \c{bindist-result.manifest} files in the
+\c{upload/bindist/<distribution>/} directory, next to the generated binary
+distribution package files. The \c{bindist-result.json} file contains the
+structured JSON output of the \l{bpkg-pkg-bindist(1)} command. The
+\c{bindist-result.manifest} file contains the subset of the information from
+\c{bindist-result.json}. Specifically, it starts with the binary distribution
+package header manifest followed by a list of package file manifests. The
+manifest values are:
+
+\
+distribution:
+architecture:
+os-release-name-id:
+os-release-version-id:
+package-name:
+package-version:
+[package-system-version]:
+
+package-file-type:
+package-file-path:
+[package-file-system-name]:
+\
+
+The manifest values derive from the corresponding JSON object values and
+preserve their semantics. The only differences are that the
+\c{os-release-version-id} value may not be absent and the
+\c{package-file-path} values are relative to the
+\c{upload/bindist/<distribution>/} directory and are in the POSIX
+representation. See \l{bpkg-pkg-bindist(1)} for the JSON values semantics.
+
 \h#arch-controller|Controller Logic|
 
 A \c{bbot} controller that issues own build tasks maps available build
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 012a00a..75b61cb 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -99,6 +99,7 @@ config = "$config bpkg.create:config.install.root=\"'$~/usr/local'\""
 package_config = 'package-config:
 \
 +bpkg.bindist.fedora:
++bbot.bindist.upload:
 bpkg.create:config.bin.rpath=[null]
 \'
 #\