Add json_map and json_set buildfile value types

These expose the std::map<json_value,json_value> and std::set<json_value>
types to buildfiles.

New functions:

  $size(<json-set>)
  $size(<json-map>)
  $keys(<json-map>)

Note that the $keys() function returns the list of map key as a json array.

For example:

m = [json_map] 2@([json] a@1 b@2) 1@([json] 1 2)
s = [json_set] ([json] x@1 y@2) ([json] a@1 b@2)

print ($m[2][b])              # 2
print ($s[([json] y@2 x@1)])  # true

6 files changed, 197 insertions, 95 deletions

diff --git a/libbuild2/functions-json.cxx b/libbuild2/functions-json.cxx
index 5715e13..e06d9a5 100644
--- a/libbuild2/functions-json.cxx
+++ b/libbuild2/functions-json.cxx
@@ -308,5 +308,28 @@ namespace build2
       }
     };
 #endif
+
+    // $size(<json-set>)
+    // $size(<json-map>)
+    //
+    // Return the number of elements in the sequence.
+    //
+    f["size"] += [] (set<json_value> v)             {return v.size ();};
+    f["size"] += [] (map<json_value, json_value> v) {return v.size ();};
+
+    // $keys(<json-map>)
+    //
+    // Return the list of keys in a json map as a json array.
+    //
+    // Note that the result is sorted in ascending order.
+    //
+    f["keys"] += [](map<json_value, json_value> v)
+    {
+      json_value r (json_type::array);
+      r.array.reserve (v.size ());
+      for (pair<const json_value, json_value>& p: v)
+        r.array.push_back (p.first); // @@ PERF: use C++17 map::extract() to steal.
+      return r;
+    };
   }
 }
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx
index bf806be..5572104 100644
--- a/libbuild2/parser.cxx
+++ b/libbuild2/parser.cxx
@@ -6066,7 +6066,12 @@ namespace build2
         {
           if (n[4] == '\0') return &value_traits<json_value>::value_type;
           if (n == "json_array") return &value_traits<json_array>::value_type;
-          if (n == "json_object") return &value_traits<json_object>::value_type;
+          if (n == "json_object")
+            return &value_traits<json_object>::value_type;
+          if (n == "json_set")
+            return &value_traits<set<json_value>>::value_type;
+          if (n == "json_map")
+            return &value_traits<map<json_value, json_value>>::value_type;
         }
         break;
       }
@@ -6266,6 +6271,8 @@ namespace build2
 
       if (n == "null")
       {
+        // @@ Looks like here we assume representationally empty?
+        //
         if (rhs && !rhs.empty ()) // Note: null means we had an expansion.
           fail (l) << "value with null attribute";
 
@@ -7808,9 +7815,9 @@ namespace build2
         // an empty sequence of names rather than a sequence of one empty
         // name.
         //
-        if (!d.empty ())
+        if (size_t n = d.size ())
         {
-          if (d.size () != 1)
+          if (n != 1)
           {
             assert (what_expansion != nullptr);
             concat_diag_multiple (loc, what_expansion);
@@ -9079,7 +9086,7 @@ namespace build2
           // Untyped concatenation. Note that if RHS is NULL/empty, we still
           // set the concat flag.
           //
-          else if (!result->null && !result->empty ())
+          else if (!result->null)
           {
             // This can only be an untyped value.
             //
@@ -9087,33 +9094,36 @@ namespace build2
             //
             const names& lv (cast<names> (*result));
 
-            // This should be a simple value or a simple directory.
-            //
-            if (lv.size () > 1)
-              concat_diag_multiple (loc, what);
+            if (size_t s = lv.size ())
+            {
+              // This should be a simple value or a simple directory.
+              //
+              if (s > 1)
+                concat_diag_multiple (loc, what);
 
-            const name& n (lv[0]);
+              const name& n (lv[0]);
 
-            if (n.qualified ())
-              fail (loc) << "concatenating " << what << " contains project "
-                         << "name";
+              if (n.qualified ())
+                fail (loc) << "concatenating " << what << " contains project "
+                           << "name";
 
-            if (n.typed ())
-              fail (loc) << "concatenating " << what << " contains target type";
+              if (n.typed ())
+                fail (loc) << "concatenating " << what << " contains target type";
 
-            if (!n.dir.empty ())
-            {
-              if (!n.value.empty ())
-                fail (loc) << "concatenating " << what << " contains "
-                           << "directory";
+              if (!n.dir.empty ())
+              {
+                if (!n.value.empty ())
+                  fail (loc) << "concatenating " << what << " contains "
+                             << "directory";
 
-              // Note that here we cannot assume what's in dir is really a
-              // path (think s/foo/bar/) so we have to reverse it exactly.
-              //
-              concat_data.value += n.dir.representation ();
+                // Note that here we cannot assume what's in dir is really a
+                // path (think s/foo/bar/) so we have to reverse it exactly.
+                //
+                concat_data.value += n.dir.representation ();
+              }
+              else
+                concat_data.value += n.value;
             }
-            else
-              concat_data.value += n.value;
           }
 
           // The same little hack as in the word case ($empty+foo).
@@ -9139,16 +9149,27 @@ namespace build2
 
           // Nothing else to do here if the result is NULL or empty.
           //
-          if (result->null || result->empty ())
-            continue;
-
-          // @@ Could move if nv is result_data; see untypify().
+          // Note that we cannot use value::empty() here since we are
+          // interested in representationally empty.
           //
-          names nv_storage;
-          names_view nv (reverse (*result, nv_storage, true /* reduce */));
+          if (!result->null)
+          {
+            // @@ Could move if nv is result_data; see untypify().
+            //
+            // Nuance: we should only be reducing empty simple value to empty
+            // list if we are not a second half of a pair.
+            //
+            bool pair (!ns.empty () && ns.back ().pair);
+
+            names nv_storage;
+            names_view nv (reverse (*result, nv_storage, !pair /* reduce */));
 
-          count = splice_names (
-            loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp);
+            if (!nv.empty ())
+            {
+              count = splice_names (
+                loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp);
+            }
+          }
         }
 
         continue;
diff --git a/libbuild2/variable.cxx b/libbuild2/variable.cxx
index 4a08b4d..ab65237 100644
--- a/libbuild2/variable.cxx
+++ b/libbuild2/variable.cxx
@@ -1635,6 +1635,17 @@ namespace build2
   }
 
   json_value value_traits<json_value>::
+  convert (name&& l, name* r)
+  {
+    // Here we expect either a simple value or a serialized representation.
+    //
+    if (r != nullptr)
+      throw invalid_argument ("pair in json element value");
+
+    return to_json_value (l, "json element");
+  }
+
+  json_value value_traits<json_value>::
   convert (names&& ns)
   {
     size_t n (ns.size ());
@@ -1781,56 +1792,36 @@ namespace build2
     }
   }
 
-  static names_view
-  json_reverse (const value& x, names& ns, bool)
+  name value_traits<json_value>::
+  reverse (const json_value& v)
   {
-    const json_value& v (x.as<json_value> ());
-
     switch (v.type)
     {
     case json_type::null:
       {
-        // @@ Hm, it would be nice if this somehow got mapped to [null]/empty
-        //    but still be round-trippable to JSON null. Perhaps via type
-        //    hint?
-        //
-        //    But won't `print ([json] null)` printing nothing be
-        //    surprising. Also, it's not clear that mapping JSON null to out
-        //    [null] is a good idea since our [null] means "no value" while
-        //    JSON null means "null value".
-        //
-        //    Maybe the current semantics is the best: we map our [null] and
-        //    empty names to JSON null (naturally) but we always reverse JSON
-        //    null to the JSON "null" literal. Or maybe we could reverse it to
-        //    null but type-hint it that it's a spelling or [null]/empty.
-        //    Quite fuzzy, admittedly. In our model null values decay to empty
-        //    so JSON null decaying to "null" literal is strange. Let's try
-        //    and see how it goes. See also json_subscript_impl() below.
+        // Return empty to be consistent with other places.
         //
 #if 0
-        ns.push_back (name ("null"));
+        return name ("null");
+#else
+        return name ();
 #endif
-        break;
       }
     case json_type::boolean:
       {
-        ns.push_back (name (v.boolean ? "true" : "false"));
-        break;
+        return name (v.boolean ? "true" : "false");
       }
     case json_type::signed_number:
       {
-        ns.push_back (value_traits<int64_t>::reverse (v.signed_number));
-        break;
+        return value_traits<int64_t>::reverse (v.signed_number);
       }
     case json_type::unsigned_number:
       {
-        ns.push_back (value_traits<uint64_t>::reverse (v.unsigned_number));
-        break;
+        return value_traits<uint64_t>::reverse (v.unsigned_number);
       }
     case json_type::hexadecimal_number:
       {
-        ns.push_back (name (to_string (v.unsigned_number, 16)));
-        break;
+        return name (to_string (v.unsigned_number, 16));
       }
     case json_type::string:
         //
@@ -1868,6 +1859,10 @@ namespace build2
         }
         catch (const invalid_json_output& e)
         {
+          // Note that while it feels like value_traits::reverse() should
+          // throw invalid_argument, we don't currently handle it anywhere so
+          // for now let's just fail.
+          //
           // Note: the same diagnostics as in $json.serialize().
           //
           diag_record dr;
@@ -1882,11 +1877,38 @@ namespace build2
 #else
         fail << "json serialization requested during bootstrap";
 #endif
-        ns.push_back (name (move (o)));
-        break;
+        return name (move (o));
       }
     }
 
+    assert (false);
+    return name ();
+  }
+
+  static names_view
+  json_reverse (const value& x, names& ns, bool reduce)
+  {
+    const json_value& v (x.as<json_value> ());
+
+    // @@ Hm, it would be nice if JSON null somehow got mapped to [null]/empty
+    //    but still be round-trippable to JSON null. Perhaps via type hint?
+    //
+    //    But won't `print ([json] null)` printing nothing be surprising.
+    //    Also, it's not clear that mapping JSON null to out [null] is a good
+    //    idea since our [null] means "no value" while JSON null means "null
+    //    value".
+    //
+    //    Maybe the current semantics is the best: we map our [null] and empty
+    //    names to JSON null (naturally) but we always reverse JSON null to
+    //    the JSON "null" literal. Or maybe we could reverse it to null but
+    //    type-hint it that it's a spelling or [null]/empty. Quite fuzzy,
+    //    admittedly. In our model null values decay to empty so JSON null
+    //    decaying to "null" literal is strange. Let's try and see how it
+    //    goes. See also json_subscript_impl() below.
+    //
+    if (v.type != json_type::null || !reduce)
+      ns.push_back (value_traits<json_value>::reverse (v));
+
     return ns;
   }
 
@@ -3294,11 +3316,15 @@ namespace build2
   value_traits<vector<pair<string, optional<bool>>>>;
 
   template struct LIBBUILD2_DEFEXPORT value_traits<set<string>>;
+  template struct LIBBUILD2_DEFEXPORT value_traits<set<json_value>>;
 
   template struct LIBBUILD2_DEFEXPORT
   value_traits<map<string, string>>;
 
   template struct LIBBUILD2_DEFEXPORT
+  value_traits<map<json_value, json_value>>;
+
+  template struct LIBBUILD2_DEFEXPORT
   value_traits<map<string, optional<string>>>;
 
   template struct LIBBUILD2_DEFEXPORT
diff --git a/libbuild2/variable.hxx b/libbuild2/variable.hxx
index d754edf..aed3350 100644
--- a/libbuild2/variable.hxx
+++ b/libbuild2/variable.hxx
@@ -100,6 +100,10 @@ namespace build2
 
     // If NULL, then the value is never empty.
     //
+    // Note that this is "semantically empty", not necessarily
+    // "representationally empty". For example, an empty JSON array is
+    // semantically empty but its representation (`[]`) is not.
+    //
     bool (*const empty) (const value&);
 
     // Custom subscript function. If NULL, then the generic implementation is
@@ -347,6 +351,10 @@ namespace build2
     // Check in a type-independent way if the value is empty. The value must
     // not be NULL.
     //
+    // Note that this is "semantically empty", not necessarily
+    // "representationally empty". For example, an empty JSON array is
+    // semantically empty but its representation (`[]`) is not.
+    //
     bool
     empty () const;
 
@@ -691,7 +699,7 @@ namespace build2
   // case (container) if invalid_argument is thrown, the names are not
   // guaranteed to be unchanged.
   //
-  //template <typename T> T convert (names&&); (declaration causes ambiguity)
+  template <typename T> T convert (names&&);
 
   // Convert value to T. If value is already of type T, then simply cast it.
   // Otherwise call convert(names) above. If value is NULL, then throw
@@ -1254,6 +1262,14 @@ namespace build2
     static void prepend (value&, json_value&&);
     static bool empty (const json_value&); // null or empty array/object
 
+    // These are provided to make it possible to use json_value as a container
+    // element.
+    //
+    static json_value convert (name&&, name*);
+    static name reverse (const json_value&);
+    static int compare (const json_value& x, const json_value& y) {
+      return x.compare (y);}
+
     static const json_value empty_instance; // null
     static const char* const type_name;
     static const build2::value_type value_type;
@@ -1348,11 +1364,15 @@ namespace build2
   value_traits<vector<pair<string, optional<bool>>>>;
 
   extern template struct LIBBUILD2_DECEXPORT value_traits<set<string>>;
+  extern template struct LIBBUILD2_DECEXPORT value_traits<set<json_value>>;
 
   extern template struct LIBBUILD2_DECEXPORT
   value_traits<map<string, string>>;
 
   extern template struct LIBBUILD2_DECEXPORT
+  value_traits<map<json_value, json_value>>;
+
+  extern template struct LIBBUILD2_DECEXPORT
   value_traits<map<string, optional<string>>>;
 
   extern template struct LIBBUILD2_DECEXPORT
diff --git a/libbuild2/variable.ixx b/libbuild2/variable.ixx
index a448cd8..ca84a33 100644
--- a/libbuild2/variable.ixx
+++ b/libbuild2/variable.ixx
@@ -362,13 +362,53 @@ namespace build2
 
   // This one will be SFINAE'd out unless T is a container.
   //
+  // If T is both (e.g., json_value), then make this version preferable.
+  //
   template <typename T>
   inline auto
-  convert (names&& ns) -> decltype (value_traits<T>::convert (move (ns)))
+  convert_impl (names&& ns, int)
+    -> decltype (value_traits<T>::convert (move (ns)))
   {
     return value_traits<T>::convert (move (ns));
   }
 
+  // This one will be SFINAE'd out unless T is a simple value.
+  //
+  // If T is both (e.g., json_value), then make this version less preferable.
+  //
+  template <typename T>
+  auto // NOTE: not inline!
+  convert_impl (names&& ns, ...) ->
+    decltype (value_traits<T>::convert (move (ns[0]), nullptr))
+  {
+    size_t n (ns.size ());
+
+    if (n == 0)
+    {
+      if (value_traits<T>::empty_value)
+        return T ();
+    }
+    else if (n == 1)
+    {
+      return convert<T> (move (ns[0]));
+    }
+    else if (n == 2 && ns[0].pair != '\0')
+    {
+      return convert<T> (move (ns[0]), move (ns[1]));
+    }
+
+    throw invalid_argument (
+      string ("invalid ") + value_traits<T>::type_name +
+      (n == 0 ? " value: empty" : " value: multiple names"));
+  }
+
+  template <typename T>
+  inline T
+  convert (names&& ns)
+  {
+    return convert_impl<T> (move (ns), 0);
+  }
+
   // bool value
   //
   inline void value_traits<bool>::
diff --git a/libbuild2/variable.txx b/libbuild2/variable.txx
index 9d39ed7..12a2667 100644
--- a/libbuild2/variable.txx
+++ b/libbuild2/variable.txx
@@ -27,34 +27,6 @@ namespace build2
     return false;
   }
 
-  // This one will be SFINAE'd out unless T is a simple value.
-  //
-  template <typename T>
-  auto
-  convert (names&& ns) ->
-    decltype (value_traits<T>::convert (move (ns[0]), nullptr))
-  {
-    size_t n (ns.size ());
-
-    if (n == 0)
-    {
-      if (value_traits<T>::empty_value)
-        return T ();
-    }
-    else if (n == 1)
-    {
-      return convert<T> (move (ns[0]));
-    }
-    else if (n == 2 && ns[0].pair != '\0')
-    {
-      return convert<T> (move (ns[0]), move (ns[1]));
-    }
-
-    throw invalid_argument (
-      string ("invalid ") + value_traits<T>::type_name +
-      (n == 0 ? " value: empty" : " value: multiple names"));
-  }
-
   [[noreturn]] LIBBUILD2_SYMEXPORT void
   convert_throw (const value_type* from, const value_type& to);