Add $process.run() and $process.run_regex() functions

$process.run(<prog>[ <args>...])

Return trimmed stdout.

$process.run_regex(<prog>[ <args>...], <pat> [, <fmt>])

Return stdout lines matched and optionally processed with regex.

Each line of stdout (including the customary trailing blank) is matched (as a
whole) against <pat> and, if successful, returned, optionally processed with
<fmt>, as an element of a list.

10 files changed, 350 insertions, 12 deletions

diff --git a/build2/function.cxx b/build2/function.cxx
index 43dd531..4367185 100644
--- a/build2/function.cxx
+++ b/build2/function.cxx
@@ -329,6 +329,7 @@ namespace build2
   void builtin_functions ();        // functions-builtin.cxx
   void filesystem_functions ();     // functions-filesystem.cxx
   void path_functions ();           // functions-path.cxx
+  void process_functions ();        // functions-process.cxx
   void process_path_functions ();   // functions-process-path.cxx
   void regex_functions ();          // functions-regex.cxx
   void string_functions ();         // functions-string.cxx
@@ -341,6 +342,7 @@ namespace build2
       builtin_functions ();
       filesystem_functions ();
       path_functions ();
+      process_functions ();
       process_path_functions ();
       regex_functions ();
       string_functions ();
diff --git a/build2/function.hxx b/build2/function.hxx
index c2686f2..62666cc 100644
--- a/build2/function.hxx
+++ b/build2/function.hxx
@@ -45,10 +45,16 @@ namespace build2
   // expected to issue diagnostics and throw failed. Note that the arguments
   // are conceptually "moved" and can be reused by the implementation.
   //
+  // @@ Maybe it makes sense to implicitly convert types like string to names
+  //    -- providing all the overload combinations really gets tedious.
+  //
   // A function can also optionally receive the current scope by having the
   // first argument of the const scope* type. It may be NULL is the function
   // is called out of any scope (e.g., command line).
   //
+  // Note also that we don't pass the location to the function instead
+  // printing the info message pointing to the call site.
+  //
   // Normally functions come in families that share a common qualification
   // (e.g., string. or path.). The function_family class is a "registrar"
   // that simplifies handling of function families. For example:
diff --git a/build2/functions-builtin.cxx b/build2/functions-builtin.cxx
index 45ad7df..8db38c2 100644
--- a/build2/functions-builtin.cxx
+++ b/build2/functions-builtin.cxx
@@ -11,7 +11,7 @@ namespace build2
   // otherwise.
   //
   static inline value
-  getvar (const string& name)
+  getenvvar (const string& name)
   {
     optional<string> v (getenv (name));
 
@@ -19,7 +19,7 @@ namespace build2
       return value ();
 
     names r;
-    r.emplace_back (to_name (*v));
+    r.emplace_back (to_name (move (*v)));
     return value (move (r));
   }
 
@@ -45,12 +45,12 @@ namespace build2
     //
     f["getenv"] = [](string name)
     {
-      return getvar (name);
+      return getenvvar (name);
     };
 
     f["getenv"] = [](names name)
     {
-      return getvar (convert<string> (move (name)));
+      return getenvvar (convert<string> (move (name)));
     };
   }
 }
diff --git a/build2/functions-process.cxx b/build2/functions-process.cxx
new file mode 100644
index 0000000..ef36828
--- /dev/null
+++ b/build2/functions-process.cxx
@@ -0,0 +1,253 @@
+// file      : build2/functions-process.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbutl/regex.mxx>
+
+#include <build2/function.hxx>
+#include <build2/variable.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace build2
+{
+  // Ideas for potential further improvements:
+  //
+  // - Use scope to query environment.
+  // - Mode to ignore error/suppress diagnostics and return NULL?
+  // - Similar regex flags to regex.* functions (icase, etc)?
+
+  // Process arguments.
+  //
+  static pair<process_path, strings>
+  process_args (names&& args, const char* fn)
+  {
+    if (args.empty () || args[0].empty ())
+      fail << "executable name expected in process." << fn << "()";
+
+    process_path pp;
+    try
+    {
+      size_t erase;
+
+      // This can be a process_path (pair) or just a path.
+      //
+      if (args[0].pair)
+      {
+        pp = convert<process_path> (move (args[0]), move (args[1]));
+        erase = 2;
+      }
+      else
+      {
+        pp = run_search (convert<path> (move (args[0])));
+        erase = 1;
+      }
+
+      args.erase (args.begin (), args.begin () + erase);
+    }
+    catch (const invalid_argument& e)
+    {
+      fail << "invalid process." << fn << "() executable path: " << e.what ();
+    }
+
+    strings sargs;
+    try
+    {
+      sargs = convert<strings> (move (args));
+    }
+    catch (const invalid_argument& e)
+    {
+      fail << "invalid process." << fn << "() argument: " << e.what ();
+    }
+
+    return pair<process_path, strings> (move (pp), move (sargs));
+  }
+
+  static process
+  start (const scope*,
+         const process_path& pp,
+         const strings& args,
+         cstrings& cargs)
+  {
+    cargs.reserve (args.size () + 2);
+    cargs.push_back (pp.recall_string ());
+    transform (args.begin (),
+               args.end (),
+               back_inserter (cargs),
+               [] (const string& s) {return s.c_str ();});
+    cargs.push_back (nullptr);
+
+    return run_start (3               /* verbosity */,
+                      pp,
+                      cargs.data (),
+                      0               /* stdin  */,
+                      -1              /* stdout */);
+  }
+
+  static void
+  finish (cstrings& args, process& pr, bool io)
+  {
+    run_finish (args, pr);
+
+    if (io)
+      fail << "error reading " << args[0] << " output";
+  }
+
+  static value
+  run (const scope* s, const process_path& pp, const strings& args)
+  {
+    cstrings cargs;
+    process pr (start (s, pp, args, cargs));
+
+    string v;
+    bool io (false);
+    try
+    {
+      ifdstream is (move (pr.in_ofd));
+
+      // Note that getline() will fail if there is no output.
+      //
+      if (is.peek () != ifdstream::traits_type::eof ())
+        getline (is, v, '\0');
+
+      is.close (); // Detect errors.
+    }
+    catch (const io_error&)
+    {
+      // Presumably the child process failed and issued diagnostics so let
+      // finish() try to deal with that first.
+      //
+      io = true;
+    }
+
+    finish (cargs, pr, io);
+
+    names r;
+    r.push_back (to_name (move (trim (v))));
+    return value (move (r));
+  }
+
+  regex
+  parse_regex (const string&, regex::flag_type); // functions-regex.cxx
+
+  static value
+  run_regex (const scope* s,
+             const process_path& pp,
+             const strings& args,
+             const string& pat,
+             const optional<string>& fmt)
+  {
+    regex re (parse_regex (pat, regex::ECMAScript));
+
+    cstrings cargs;
+    process pr (start (s, pp, args, cargs));
+
+    names r;
+    bool io (false);
+    try
+    {
+      ifdstream is (move (pr.in_ofd), ifdstream::badbit);
+
+      for (string l; !eof (getline (is, l)); )
+      {
+        if (fmt)
+        {
+          pair<string, bool> p (regex_replace_match (l, re, *fmt));
+
+          if (p.second)
+            r.push_back (to_name (move (p.first)));
+        }
+        else
+        {
+          if (regex_match (l, re))
+            r.push_back (to_name (move (l)));
+        }
+      }
+
+      is.close (); // Detect errors.
+    }
+    catch (const io_error&)
+    {
+      // Presumably the child process failed and issued diagnostics so let
+      // finish() try to deal with that first.
+      //
+      io = true;
+    }
+
+    finish (cargs, pr, io);
+
+    return value (move (r));
+  }
+
+  static inline value
+  run_regex (const scope* s,
+             names&& args,
+             const string& pat,
+             const optional<string>& fmt)
+  {
+    pair<process_path, strings> pa (process_args (move (args), "run_regex"));
+    return run_regex (s, pa.first, pa.second, pat, fmt);
+  }
+
+  void
+  process_functions ()
+  {
+    function_family f ("process");
+
+    // $process.run(<prog>[ <args>...])
+    //
+    // Return trimmed stdout.
+    //
+    f[".run"] = [](const scope* s, names args)
+    {
+      pair<process_path, strings> pa (process_args (move (args), "run"));
+      return run (s, pa.first, pa.second);
+    };
+
+    f["run"] = [](const scope* s, process_path pp)
+    {
+      return run (s, pp, strings ());
+    };
+
+    // $process.run_regex(<prog>[ <args>...], <pat> [, <fmt>])
+    //
+    // Return stdout lines matched and optionally processed with regex.
+    //
+    // Each line of stdout (including the customary trailing blank) is matched
+    // (as a whole) against <pat> and, if successful, returned, optionally
+    // processed with <fmt>, as an element of a list.
+    //
+    f[".run_regex"] = [](const scope* s, names a, string p, optional<string> f)
+    {
+      return run_regex (s, move (a), p, f);
+    };
+
+    f[".run_regex"] = [] (const scope* s, names a, names p, optional<names> f)
+    {
+      return run_regex (s,
+                        move (a),
+                        convert<string> (move (p)),
+                        f ? convert<string> (move (*f)) : nullopt_string);
+    };
+
+    f["run_regex"] = [](const scope* s,
+                        process_path pp,
+                        string p,
+                        optional<string> f)
+    {
+      return run_regex (s, pp, strings (), p, f);
+    };
+
+    f["run_regex"] = [](const scope* s,
+                        process_path pp,
+                        names p,
+                        optional<names> f)
+    {
+      return run_regex (s,
+                        pp, strings (),
+                        convert<string> (move (p)),
+                        f ? convert<string> (move (*f)) : nullopt_string);
+    };
+  }
+}
diff --git a/build2/functions-regex.cxx b/build2/functions-regex.cxx
index 76fcce9..9c428fd 100644
--- a/build2/functions-regex.cxx
+++ b/build2/functions-regex.cxx
@@ -29,7 +29,9 @@ namespace build2
 
   // Parse a regular expression. Throw invalid_argument if it is not valid.
   //
-  static regex
+  // Note: also used in functions-process.cxx (thus not static).
+  //
+  regex
   parse_regex (const string& s, regex::flag_type f)
   {
     try
diff --git a/build2/utility.cxx b/build2/utility.cxx
index cbae507..07cf88e 100644
--- a/build2/utility.cxx
+++ b/build2/utility.cxx
@@ -254,10 +254,14 @@ namespace build2
     fail (loc) << "unable to execute " << args[0] << ": " << e << endf;
   }
 
-  const string empty_string;
-  const path empty_path;
+  const string   empty_string;
+  const path     empty_path;
   const dir_path empty_dir_path;
 
+  const optional<string>   nullopt_string;
+  const optional<path>     nullopt_path;
+  const optional<dir_path> nullopt_dir_path;
+
   void
   append_options (cstrings& args, const lookup& l, const char* e)
   {
diff --git a/build2/utility.hxx b/build2/utility.hxx
index 81a5d87..6e76f05 100644
--- a/build2/utility.hxx
+++ b/build2/utility.hxx
@@ -154,7 +154,7 @@ namespace build2
 
   process_path
   run_search (const path&,
-              bool init,
+              bool init = false,
               const dir_path& fallback = dir_path (),
               const location& = location ());
 
@@ -388,12 +388,16 @@ namespace build2
       verbosity, pp, args, forward<F> (f), error, ignore_exit, checksum);
   }
 
-  // Empty string and path.
+  // Empty/nullopt string and path.
   //
-  extern const string empty_string;
-  extern const path empty_path;
+  extern const string   empty_string;
+  extern const path     empty_path;
   extern const dir_path empty_dir_path;
 
+  extern const optional<string>   nullopt_string;
+  extern const optional<path>     nullopt_path;
+  extern const optional<dir_path> nullopt_dir_path;
+
   // Hash a path potentially without the specific directory prefix.
   //
   // If prefix is not empty and is a super-path of the path to hash, then only
diff --git a/build2/variable.cxx b/build2/variable.cxx
index 8000913..0c07db6 100644
--- a/build2/variable.cxx
+++ b/build2/variable.cxx
@@ -600,7 +600,20 @@ namespace build2
         catch (invalid_path& e)
         {
           n.value = move (e.path); // Restore the name object for diagnostics.
+          // Fall through.
+        }
+      }
 
+      // Reassemble split dir/value.
+      //
+      if (n.untyped () && n.unqualified ())
+      {
+        try
+        {
+          return n.dir / n.value;
+        }
+        catch (const invalid_path&)
+        {
           // Fall through.
         }
       }
@@ -649,7 +662,26 @@ namespace build2
         {
           return dir_path (move (n.value));
         }
-        catch (const invalid_path&) {} // Fall through.
+        catch (invalid_path& e)
+        {
+          n.value = move (e.path); // Restore the name object for diagnostics.
+          // Fall through.
+        }
+      }
+
+      // Reassemble split dir/value.
+      //
+      if (n.untyped () && n.unqualified ())
+      {
+        try
+        {
+          n.dir /= n.value;
+          return move (n.dir);
+        }
+        catch (const invalid_path&)
+        {
+          // Fall through.
+        }
       }
 
       // Fall through.
diff --git a/tests/function/process/buildfile b/tests/function/process/buildfile
new file mode 100644
index 0000000..4ea4742
--- /dev/null
+++ b/tests/function/process/buildfile
@@ -0,0 +1,5 @@
+# file      : tests/function/process/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: testscript $b
diff --git a/tests/function/process/testscript b/tests/function/process/testscript
new file mode 100644
index 0000000..f01af6f
--- /dev/null
+++ b/tests/function/process/testscript
@@ -0,0 +1,30 @@
+# file      : tests/function/process/testscript
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+.include ../../common.test
+
+: run
+:
+$* <<EOI >>~/EOO/
+print $process.run($build.path --version)
+EOI
+/build2 .+/
+/.+/*
+EOO
+
+: run-regex-match
+:
+$* <<EOI >>~/EOO/
+print $process.run_regex($build.path --version, 'build2 .+')
+EOI
+/build2 .+/
+EOO
+
+: run-regex-replace
+:
+$* <<EOI >>~/EOO/
+print $process.run_regex($build.path --version, 'build2 ([0-9.]+).*', '\1')
+EOI
+/[0-9]+.[0-9]+.[0-9]+/d
+EOO