Add support for --stdout-format to pkg-status command

6 files changed, 498 insertions, 144 deletions

diff --git a/bpkg/common.cli b/bpkg/common.cli
index dd0417d..cccf1d3 100644
--- a/bpkg/common.cli
+++ b/bpkg/common.cli
@@ -80,6 +80,14 @@ namespace bpkg
        \li|Even more detailed information.||"
     }
 
+    bpkg::stdout_format --stdout-format = bpkg::stdout_format::lines
+    {
+      "<format>",
+      "Representation format to use for printing to \cb{stdout}. Valid values
+       for this option are \cb{lines} (default) and \cb{json}. See the JSON
+       OUTPUT section below for details on the \cb{json} format."
+    }
+
     size_t --jobs|-j
     {
       "<num>",
@@ -409,4 +417,72 @@ namespace bpkg
       "Don't load default options files."
     }
   };
+
+  {
+    "",
+    "
+    \h|JSON OUTPUT|
+
+    Commands that support the JSON output specify their formats as a
+    serialized representation of a C++ \cb{struct} or an array thereof. For
+    example:
+
+    \
+    struct package
+    {
+      string name;
+    };
+
+    struct configuration
+    {
+      uint64_t         id;
+      string           path;
+      optional<string> name;
+      bool             default;
+      vector<package>  packages;
+    };
+    \
+
+    An example of the serialized JSON representation of \cb{struct}
+    \cb{configuration}:
+
+    \
+    {
+      \"id\": 1,
+      \"path\": \"/tmp/hello-gcc\",
+      \"name\": \"gcc\",
+      \"default\": true,
+      \"packages\": [
+        {
+          \"name\": \"hello\"
+        }
+      ]
+    }
+    \
+
+    This sections provides details on the overall properties of such formats
+    and the semantics of the \cb{struct} serialization.
+
+    The order of members in a JSON object is fixed as specified in the
+    corresponding \cb{struct}. While new members may be added in the
+    future (and should be ignored by older consumers), the semantics of the
+    existing members (including whether the top-level entry is an object or
+    array) may not change.
+
+    An object member is required unless its type is \cb{optional<>},
+    \cb{bool}, or \cb{vector<>} (array). For \cb{bool} members absent means
+    \cb{false}. For \cb{vector<>} members absent means empty. An empty
+    top-level array is always present.
+
+    For example, the following JSON text is a possible serialization of
+    the above \cb{struct} \cb{configuration}:
+
+    \
+    {
+      \"id\": 1,
+      \"path\": \"/tmp/hello-gcc\"
+    }
+    \
+    "
+  }
 }
diff --git a/bpkg/options-types.hxx b/bpkg/options-types.hxx
index 741e93c..6576060 100644
--- a/bpkg/options-types.hxx
+++ b/bpkg/options-types.hxx
@@ -21,6 +21,12 @@ namespace bpkg
     all
   };
 
+  enum class stdout_format
+  {
+    lines,
+    json
+  };
+
   // Qualified options.
   //
   // An option that uses this type can have its values qualified using the
diff --git a/bpkg/pkg-status.cli b/bpkg/pkg-status.cli
index d2eb644..59319bf 100644
--- a/bpkg/pkg-status.cli
+++ b/bpkg/pkg-status.cli
@@ -32,12 +32,13 @@ namespace bpkg
      \c{\b{--recursive}|\b{-r}} options, respectively. Note that the status is
      written to \cb{stdout}, not \cb{stderr}.
 
-     The status output format is regular with components separated with
-     spaces. Each line starts with the package name followed by one of the
-     status words listed below. Some of them can be optionally followed by
-     '\cb{,}' (no spaces) and a sub-status word. Lines corresponding to
-     dependencies from linked configurations will additionally mention the
-     configuration directory in square brackets after the package name.
+     The default output format (see the \cb{--stdout-format} common option) is
+     regular with components separated with spaces. Each line starts with the
+     package name followed by one of the status words listed below. Some of
+     them can be optionally followed by '\cb{,}' (no spaces) and a sub-status
+     word. Lines corresponding to dependencies from linked configurations will
+     additionally mention the configuration directory in square brackets after
+     the package name.
 
      \dl|
 
@@ -83,11 +84,13 @@ namespace bpkg
      package may or may not be available from the system and that its version
      is unknown.
 
-     If the package version was specified, then the status word is always
-     followed by this version (or its revision).
+     The \cb{fetched}, \cb{unpacked}, \cb{configured}, and \cb{broken} status
+     words are followed by the version of the package. If the package version
+     was specified, then the \cb{unknown} status word is also followed by the
+     version.
 
      If the status is \cb{fetched}, \cb{unpacked}, \cb{configured}, or
-     \cb{broken} and newer versions are available, then this version is
+     \cb{broken} and newer versions are available, then the package version is
      followed by the \cb{available} status word and the list of newer
      versions. To instead see a list of all versions, including the older
      ones, specify the \c{\b{--old-available}|\b{-o}} option. In this case the
@@ -160,6 +163,80 @@ namespace bpkg
        libbar configured 2.0.0
      \
 
+     If the output format is \cb{json}, then the output is a JSON array of
+     objects which are the serialized representation of the following C++
+     \cb{struct} \cb{package_status}:
+
+     \
+     struct available_version
+     {
+       string version;
+       bool   system;
+       bool   dependency;
+     };
+
+     struct package_status
+     {
+       string                    name;
+       optional<string>          configuration;
+       optional<string>          constraint;
+       string                    status;
+       optional<string>          sub_status;
+       optional<string>          version;
+       bool                      hold_package;
+       bool                      hold_version;
+       vector<available_version> available_versions;
+       vector<package_status>    dependencies;
+     };
+     \
+
+     For example:
+
+     \
+     [
+       {
+         \"name\": \"hello\",
+         \"status\": \"configured\",
+         \"version\": \"1.0.0\",
+         \"hold_package\": true,
+         \"available_versions\": [
+           {
+             \"version\": \"1.0.1\"
+           },
+           {
+             \"version\": \"2.0.0\"
+           }
+         ],
+         \"dependencies\": [
+           {
+             \"name\": \"libhello\",
+             \"status\": \"configured\",
+             \"version\": \"1.0.2\",
+           }
+         ]
+       }
+     ]
+     \
+
+     See the JSON OUTPUT section in \l{bdep-common-options(1)} for details on
+     the overall properties of this format and the semantics of the
+     \cb{struct} serialization.
+
+     In \cb{package_status}, the \cb{configuration} member contains the
+     absolute directory of a linked configuration if this package resides in a
+     linked configuration. The \cb{constraint} member is present only if the
+     \cb{--constraint} option is specified. The \cb{version} member is absent
+     if the \cb{status} member is \cb{unknown} or \cb{available} and no
+     package version is specified on the command line. If the \cb{sub_status}
+     member is \cb{system}, then the \cb{version} member can be special
+     \cb{*}. The \cb{dependencies} member is present only if the
+     \cb{--immediate|-i} or \cb{--recursive|-r} options are specified.
+
+     In \cb{available_version}, if the \cb{system} member is \cb{true}, then
+     this version is available from the system, in which case the \cb{version}
+     member can be special \cb{?} or \cb{*}. If the \cb{dependency} member is
+     \cb{true}, then this version is only available as a dependency from
+     prerequisite repositories of other repositories.
      "
   }
 
diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx
index 2475fa1..93008a8 100644
--- a/bpkg/pkg-status.cxx
+++ b/bpkg/pkg-status.cxx
@@ -5,6 +5,8 @@
 
 #include <iostream>   // cout
 
+#include <libbutl/json/serializer.hxx>
+
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
 #include <bpkg/database.hxx>
@@ -27,102 +29,147 @@ namespace bpkg
   };
   using packages = vector<package>;
 
-  // If recursive or immediate is true, then print status for dependencies
-  // indented by two spaces.
-  //
-  static void
-  pkg_status (const pkg_status_options& o,
-              const packages& pkgs,
-              string& indent,
-              bool recursive,
-              bool immediate)
+  struct available_package_status
   {
-    tracer trace ("pkg_status");
+    shared_ptr<available_package> package;
 
-    for (const package& p: pkgs)
-    {
-      l4 ([&]{trace << "package " << p.name << "; version " << p.version;});
+    // Can only be built as a dependency.
+    //
+    // True if this package version doesn't belong to the repositories that
+    // were explicitly added to the configuration and their complements,
+    // recursively.
+    //
+    bool dependency;
+  };
+
+  class available_package_statuses: public vector<available_package_status>
+  {
+  public:
+    // Empty if the package is not available from the system. Can be `?`.
+    //
+    string system_package_version;
+
+    // Can only be built as a dependency.
+    //
+    // True if there are no package versions available from the repositories
+    // that were explicitly added to the configuration and their complements,
+    // recursively.
+    //
+    bool dependency = true;
+  };
 
-      database& pdb (p.pdb);
-      database& rdb (p.rdb);
+  static available_package_statuses
+  pkg_statuses (const pkg_status_options& o, const package& p)
+  {
+    database& rdb (p.rdb);
+    const shared_ptr<selected_package>& s (p.selected);
 
-      // Can't be both.
-      //
-      assert (p.version.empty () || !p.constraint);
+    available_package_statuses r;
 
-      const shared_ptr<selected_package>& s (p.selected);
+    bool known (false);
+
+    shared_ptr<repository_fragment> root (
+      rdb.load<repository_fragment> (""));
 
-      // Look for available packages.
+    using query = query<available_package>;
+
+    query q (query::id.name == p.name);
+    {
+      auto qr (rdb.query<available_package> (q));
+      known = !qr.empty ();
+      r.dependency = (filter_one (root, move (qr)).first == nullptr);
+    }
+
+    if (known)
+    {
+      // If the user specified the version, then only look for that
+      // specific version (we still do it since there might be other
+      // revisions).
       //
-      // Some of them are only available to upgrade/downgrade as dependencies.
+      if (!p.version.empty ())
+        q = q && compare_version_eq (query::id.version,
+                                     canonical_version (p.version),
+                                     p.version.revision.has_value (),
+                                     false /* iteration */);
+
+      // And if we found an existing package, then only look for versions
+      // greater than to what already exists unless we were asked to show
+      // old versions.
       //
-      struct apkg
-      {
-        shared_ptr<available_package> package;
-        bool build;
-      };
-      vector<apkg> apkgs;
-
-      // A package with this name is known in available packages potentially
-      // for build.
+      // Note that for a system wildcard version we will always show all
+      // available versions (since it is 0).
       //
-      bool known (false);
-      bool build (false);
+      if (s != nullptr && !o.old_available ())
+        q = q && query::id.version > canonical_version (s->version);
+
+      q += order_by_version_desc (query::id.version);
+
+      for (shared_ptr<available_package> ap:
+             pointer_result (rdb.query<available_package> (q)))
       {
-        shared_ptr<repository_fragment> root (
-          rdb.load<repository_fragment> (""));
+        bool dependency (filter (root, ap) == nullptr);
+        r.push_back (available_package_status {move (ap), dependency});
+      }
 
-        using query = query<available_package>;
+      // The idea is that in the future we will try to auto-discover a system
+      // version. For now we just say "maybe available from the system" even
+      // if the version was specified by the user. We will later compare it if
+      // the user did specify the version.
+      //
+      if (o.system ())
+        r.system_package_version = "?";
 
-        query q (query::id.name == p.name);
+      // Get rid of stubs.
+      //
+      for (auto i (r.begin ()); i != r.end (); ++i)
+      {
+        if (i->package->stub ())
         {
-          auto r (rdb.query<available_package> (q));
-          known = !r.empty ();
-          build = filter_one (root, move (r)).first != nullptr;
+          // All the rest are stubs so bail out.
+          //
+          r.erase (i, r.end ());
+          break;
         }
+      }
+    }
 
-        if (known)
-        {
-          // If the user specified the version, then only look for that
-          // specific version (we still do it since there might be other
-          // revisions).
-          //
-          if (!p.version.empty ())
-            q = q && compare_version_eq (query::id.version,
-                                         canonical_version (p.version),
-                                         p.version.revision.has_value (),
-                                         false /* iteration */);
+    return r;
+  }
 
-          // And if we found an existing package, then only look for versions
-          // greater than to what already exists unless we were asked to show
-          // old versions.
-          //
-          // Note that for a system wildcard version we will always show all
-          // available versions (since it is 0).
-          //
-          if (s != nullptr && !o.old_available ())
-            q = q && query::id.version > canonical_version (s->version);
+  static packages
+  pkg_prerequisites (const shared_ptr<selected_package>& s, database& rdb)
+  {
+    packages r;
+    for (const auto& pair: s->prerequisites)
+    {
+      shared_ptr<selected_package> d (pair.first.load ());
+      database& db (pair.first.database ());
+      const optional<version_constraint>& c (pair.second);
+      r.push_back (package {db, rdb, d->name, version (), move (d), c});
+    }
+    return r;
+  }
 
-          q += order_by_version_desc (query::id.version);
+  static void
+  pkg_status_lines (const pkg_status_options& o,
+                    const packages& pkgs,
+                    string& indent,
+                    bool recursive,
+                    bool immediate)
+  {
+    tracer trace ("pkg_status_lines");
 
-          // Packages that are in repositories that were explicitly added to
-          // the configuration and their complements, recursively, are also
-          // available to build.
-          //
-          for (shared_ptr<available_package> ap:
-                 pointer_result (
-                   rdb.query<available_package> (q)))
-          {
-            bool build (filter (root, ap));
-            apkgs.push_back (apkg {move (ap), build});
-          }
-        }
-      }
+    for (const package& p: pkgs)
+    {
+      l4 ([&]{trace << "package " << p.name << "; version " << p.version;});
+
+      available_package_statuses ps (pkg_statuses (o, p));
 
       cout << indent;
 
       // Selected.
       //
+      const shared_ptr<selected_package>& s (p.selected);
 
       // Hold package status.
       //
@@ -134,7 +181,7 @@ namespace bpkg
 
       // If the package name is selected, then print its exact spelling.
       //
-      cout << (s != nullptr ? s->name : p.name) << pdb;
+      cout << (s != nullptr ? s->name : p.name) << p.pdb;
 
       if (o.constraint () && p.constraint)
         cout << ' ' << *p.constraint;
@@ -158,77 +205,188 @@ namespace bpkg
 
       // Available.
       //
-      bool available (false);
-      if (known)
+      if (!ps.empty () || !ps.system_package_version.empty ())
       {
-        // Available from the system.
-        //
-        // The idea is that in the future we will try to auto-discover a
-        // system version and then print that. For now we just say "maybe
-        // available from the system" even if the version was specified by
-        // the user. We will later compare it if the user did specify the
-        // version.
-        //
-        string sys;
-        if (o.system ())
+        cout << (s != nullptr ? " " : "") << "available";
+
+        for (const available_package_status& a: ps)
         {
-          sys = "?";
-          available = true;
+          const version& v (a.package->version);
+
+          // Show the currently selected version in parenthesis.
+          //
+          bool cur (s != nullptr && v == s->version);
+
+          cout << ' '
+               << (cur ? "(" : a.dependency ? "[" : "")
+               << v
+               << (cur ? ")" : a.dependency ? "]" : "");
         }
 
-        // Get rid of stubs.
+        if (!ps.system_package_version.empty ())
+          cout << ' '
+               << (ps.dependency ? "[" : "")
+               << "sys:" << ps.system_package_version
+               << (ps.dependency ? "]" : "");
+      }
+      //
+      // Unknown.
+      //
+      else if (s == nullptr)
+      {
+        cout << "unknown";
+
+        // Print the user's version if specified.
         //
-        for (auto i (apkgs.begin ()); i != apkgs.end (); ++i)
+        if (!p.version.empty ())
+          cout << ' ' << p.version;
+      }
+
+      cout << endl;
+
+      if (recursive || immediate)
+      {
+        // Collect and recurse.
+        //
+        // Let's propagate the repository information source database from the
+        // dependent to its prerequisites.
+        //
+        if (s != nullptr)
         {
-          if (i->package->stub ())
+          packages dpkgs (pkg_prerequisites (s, p.rdb));
+
+          if (!dpkgs.empty ())
           {
-            // All the rest are stubs so bail out.
-            //
-            apkgs.erase (i, apkgs.end ());
-            break;
+            indent += "  ";
+            pkg_status_lines (o, dpkgs, indent, recursive, false /* immediate */);
+            indent.resize (indent.size () - 2);
           }
+        }
+      }
+    }
+  }
+
+  static void
+  pkg_status_json (const pkg_status_options& o,
+                   const packages& pkgs,
+                   json::stream_serializer& ss,
+                   bool recursive,
+                   bool immediate)
+  {
+    tracer trace ("pkg_status_json");
+
+    ss.begin_array ();
+
+    for (const package& p: pkgs)
+    {
+      l4 ([&]{trace << "package " << p.name << "; version " << p.version;});
+
+      available_package_statuses ps (pkg_statuses (o, p));
+
+      const shared_ptr<selected_package>& s (p.selected);
+
+      // Note that we won't check some values for being valid UTF-8 (package
+      // names, etc), since their characters belong to even stricter character
+      // sets.
+      //
+      ss.begin_object ();
+
+      // If the package name is selected, then print its exact spelling.
+      //
+      ss.member ("name",
+                 (s != nullptr ? s->name : p.name).string (),
+                 false /* check */);
+
+      if (!p.pdb.string.empty ())
+        ss.member ("configuration", p.pdb.string);
+
+      if (o.constraint () && p.constraint)
+        ss.member ("constraint", p.constraint->string (), false /* check */);
+
+      // Selected.
+      //
+      if (s != nullptr)
+      {
+        ss.member ("status", to_string (s->state), false /* check */);
+
+        if (s->substate != package_substate::none)
+          ss.member ("sub_status", to_string (s->substate), false /* check */);
+
+        ss.member ("version", s->version_string (), false /* check */);
+
+        if (s->hold_package)
+          ss.member ("hold_package", true);
+
+        if (s->hold_version)
+          ss.member ("hold_version", true);
+      }
+
+      // Available.
+      //
+      if (!ps.empty () || !ps.system_package_version.empty ())
+      {
+        if (s == nullptr)
+        {
+          ss.member ("status", "available", false /* check */);
 
-          available = true;
+          // Print the user's version if specified.
+          //
+          if (!p.version.empty ())
+            ss.member ("version", p.version.string (), false /* check */);
         }
 
-        if (available)
+        // Print the list of available versions, unless a specific available
+        // version is already printed.
+        //
+        if (s != nullptr || p.version.empty ())
         {
-          cout << (s != nullptr ? " " : "") << "available";
+          ss.member_name ("available_versions");
 
-          for (const apkg& a: apkgs)
+          // Serialize an available package version.
+          //
+          auto serialize = [&ss] (const string& v, bool s, bool d)
           {
-            const version& v (a.package->version);
+            ss.begin_object ();
 
-            // Show the currently selected version in parenthesis.
-            //
-            bool cur (s != nullptr &&  v == s->version);
+            ss.member ("version", v, false /* check */);
 
-            cout << ' '
-                 << (cur ? "(" : a.build ? "" : "[")
-                 << v
-                 << (cur ? ")" : a.build ? "" : "]");
-          }
+            if (s)
+              ss.member ("system", s);
+
+            if (d)
+              ss.member ("dependency", d);
+
+            ss.end_object ();
+          };
+
+          ss.begin_array ();
+
+          for (const available_package_status& a: ps)
+            serialize (a.package->version.string (),
+                       false /* system */,
+                       a.dependency);
+
+          if (!ps.system_package_version.empty ())
+            serialize (ps.system_package_version,
+                       true /* system */,
+                       ps.dependency);
 
-          if (!sys.empty ())
-            cout << ' '
-                 << (build ? "" : "[")
-                 << "sys:" << sys
-                 << (build ? "" : "]");
+          ss.end_array ();
         }
       }
-
-      if (s == nullptr && !available)
+      //
+      // Unknown.
+      //
+      else if (s == nullptr)
       {
-        cout << "unknown";
+        ss.member ("status", "unknown", false /* check */);
 
         // Print the user's version if specified.
         //
         if (!p.version.empty ())
-          cout << ' ' << p.version;
+          ss.member ("version", p.version.string (), false /* check */);
       }
 
-      cout << endl;
-
       if (recursive || immediate)
       {
         // Collect and recurse.
@@ -236,27 +394,22 @@ namespace bpkg
         // Let's propagate the repository information source database from the
         // dependent to its prerequisites.
         //
-        packages dpkgs;
         if (s != nullptr)
         {
-          for (const auto& pair: s->prerequisites)
+          packages dpkgs (pkg_prerequisites (s, p.rdb));
+
+          if (!dpkgs.empty ())
           {
-            shared_ptr<selected_package> d (pair.first.load ());
-            database& db (pair.first.database ());
-            const optional<version_constraint>& c (pair.second);
-            dpkgs.push_back (
-              package {db, rdb, d->name, version (), move (d), c});
+            ss.member_name ("dependencies");
+            pkg_status_json (o, dpkgs, ss, recursive, false /* immediate */);
           }
         }
-
-        if (!dpkgs.empty ())
-        {
-          indent += "  ";
-          pkg_status (o, dpkgs, indent, recursive, false /* immediate */);
-          indent.resize (indent.size () - 2);
-        }
       }
+
+      ss.end_object ();
     }
+
+    ss.end_array ();
   }
 
   int
@@ -377,8 +530,22 @@ namespace bpkg
       }
     }
 
-    string indent;
-    pkg_status (o, pkgs, indent, o.recursive (), o.immediate ());
+    switch (o.stdout_format ())
+    {
+    case stdout_format::lines:
+      {
+        string indent;
+        pkg_status_lines (o, pkgs, indent, o.recursive (), o.immediate ());
+        break;
+      }
+    case stdout_format::json:
+      {
+        json::stream_serializer s (cout);
+        pkg_status_json (o, pkgs, s, o.recursive (), o.immediate ());
+        cout << endl;
+        break;
+      }
+    }
 
     t.commit ();
     return 0;
diff --git a/bpkg/types-parsers.cxx b/bpkg/types-parsers.cxx
index 97ebafe..e27f050 100644
--- a/bpkg/types-parsers.cxx
+++ b/bpkg/types-parsers.cxx
@@ -141,6 +141,24 @@ namespace bpkg
         throw invalid_value (o, v);
     }
 
+    void parser<stdout_format>::
+    parse (stdout_format& x, bool& xs, scanner& s)
+    {
+      xs = true;
+      const char* o (s.next ());
+
+      if (!s.more ())
+        throw missing_value (o);
+
+      const string v (s.next ());
+      if (v == "lines")
+        x = stdout_format::lines;
+      else if (v == "json")
+        x = stdout_format::json;
+      else
+        throw invalid_value (o, v);
+    }
+
     void parser<repository_type>::
     parse (repository_type& x, bool& xs, scanner& s)
     {
diff --git a/bpkg/types-parsers.hxx b/bpkg/types-parsers.hxx
index 007d754..dba459a 100644
--- a/bpkg/types-parsers.hxx
+++ b/bpkg/types-parsers.hxx
@@ -84,6 +84,16 @@ namespace bpkg
     };
 
     template <>
+    struct parser<stdout_format>
+    {
+      static void
+      parse (stdout_format&, bool&, scanner&);
+
+      static void
+      merge (stdout_format& b, const stdout_format& a) {b = a;}
+    };
+
+    template <>
     struct parser<repository_type>
     {
       static void