Add command running API

5 files changed, 771 insertions, 0 deletions

diff --git a/libbutl/command.cxx b/libbutl/command.cxx
new file mode 100644
index 0000000..89d7c68
--- /dev/null
+++ b/libbutl/command.cxx
@@ -0,0 +1,269 @@
+// file      : libbutl/command.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef __cpp_modules
+#include <libbutl/command.mxx>
+#endif
+
+#include <cassert>
+
+#ifndef __cpp_lib_modules
+#include <map>
+#include <string>
+#include <cstddef>
+#include <functional>
+
+#include <ios>       // ios::failure
+#include <vector>
+#include <utility>   // move()
+#include <stdexcept> // invalid_argument
+#endif
+
+// Other includes.
+
+#ifdef __cpp_modules
+module butl.command;
+
+// Only imports additional to interface.
+#ifdef __clang__
+#ifdef __cpp_lib_modules
+import std.core;
+import std.io;
+#endif
+import butl.process;
+import butl.optional;
+#endif
+
+import butl.fdstream;
+import butl.string_parser;
+#else
+#include <libbutl/fdstream.mxx>
+#include <libbutl/string-parser.mxx>
+#endif
+
+using namespace std;
+
+namespace butl
+{
+  process_exit
+  command_run (const string& cmd_str,
+               const optional<process_env>& env,
+               const optional<command_substitution_map>& substitutions,
+               char subst,
+               const function<command_callback>& callback)
+  {
+    // Split the command line into the program path, arguments, and redirects,
+    // removing one level of quoting.
+    //
+    // Note: may throw invalid_argument.
+    //
+    vector<string> cmd (
+      string_parser::parse_quoted (cmd_str, true /* unquote */));
+
+    auto bad_arg = [] (const string& d) {throw invalid_argument (d);};
+
+    if (cmd.empty ())
+      bad_arg ("no program path specified");
+
+    // Perform substitutions in a string. Throw invalid_argument for a
+    // malformed substitution or an unknown variable.
+    //
+    auto substitute = [&substitutions, subst, &bad_arg] (string&& s)
+    {
+      if (!substitutions)
+        return move (s);
+
+      string r;
+      size_t p (0); // Current parsing position.
+
+      for (size_t sp; (sp = s.find (subst, p)) != string::npos; ++p)
+      {
+        // Append the source string fraction preceding this substitution.
+        //
+        r.append (s, p, sp - p);
+
+        ++sp;                   // Start of the variable name.
+        p = s.find (subst, sp); // End of the variable name.
+
+        // Unescape the substitution character (adjacent substitution
+        // characters).
+        //
+        if (p == sp)
+        {
+          r += subst;
+          continue;
+        }
+
+        // Verify that the variable name is properly terminated and doesn't
+        // contain whitespaces.
+        //
+        if (p == string::npos)
+          bad_arg (string ("unmatched substitution character '") + subst +
+                   "' in '" + s + "'");
+
+        string vn (s, sp, p - sp);
+
+        assert (!vn.empty ()); // Otherwise it would be an escape sequence.
+
+        if (vn.find_first_of (" \t") != string::npos)
+          bad_arg ("whitespace in variable name '" + vn + "'");
+
+        // Find the variable and append its value or fail if it's unknown.
+        //
+        auto i (substitutions->find (vn));
+
+        if (i == substitutions->end ())
+          bad_arg ("unknown variable '" + vn + "'");
+
+        r += i->second;
+      }
+
+      // Append the source string tail, following the last substitution, and
+      // optimizing for the no-substitutions case.
+      //
+      if (p == 0)
+        return move (s);
+
+      r.append (s.begin () + p, s.end ());
+      return r;
+    };
+
+    // Perform substitutions in the program path.
+    //
+    string prog (substitute (move (cmd.front ())));
+
+    // Sort the remaining command line elements into the arguments and
+    // redirects, performing the substitutions. Complete relative redirects
+    // against CWD and use the rightmost redirect.
+    //
+    vector<string> args;
+
+    optional<dir_path> redir;
+    bool redir_append (false);
+
+    const dir_path& cwd (env && env->cwd != nullptr ? *env->cwd : dir_path ());
+
+    for (auto i (cmd.begin () + 1), e (cmd.end ()); i != e; ++i)
+    {
+      string a (move (*i));
+
+      if (a[0] == '>') // Redirect.
+      {
+        redir_append = a[1] == '>';
+
+        size_t n (redir_append ? 2 : 1);
+
+        if (a.size () != n) // Strip the >/>> prefix.
+        {
+          a.erase (0, n);
+        }
+        else // Take the space-separated file path from the next element.
+        {
+          if (++i == e)
+            bad_arg ("no stdout redirect file specified");
+
+          a = move (*i);
+        }
+
+        try
+        {
+          redir = dir_path (substitute (move (a)));
+        }
+        catch (const invalid_path& e)
+        {
+          bad_arg ("invalid stdout redirect file path '" + e.path + "'");
+        }
+
+        if (redir->empty ())
+          bad_arg ("empty stdout redirect file path");
+
+        if (redir->relative () && !cwd.empty ())
+          redir = cwd / *redir;
+      }
+      else             // Argument.
+      {
+        args.push_back (substitute (move (a)));
+      }
+    }
+
+    // Prepare the process environment.
+    //
+    // Note: cwd passed to process_env() may not be a temporary object.
+    //
+    process_env pe (prog, cwd, env ? env->vars : nullptr);
+
+    // Open the redirect file descriptor, if specified.
+    //
+    // Intercept the exception to make the error description more informative.
+    //
+    auto_fd rd;
+
+    if (redir)
+    try
+    {
+      fdopen_mode m (fdopen_mode::out | fdopen_mode::create);
+      m |= redir_append ? fdopen_mode::at_end : fdopen_mode::truncate;
+
+      rd = fdopen (*redir, m);
+    }
+    catch (const ios::failure& e)
+    {
+      // @@ For libstdc++ the resulting exception description will be
+      //    something like:
+      //
+      //    unable to open stdout redirect file ...: No such file or directory
+      //
+      //    Maybe we should improve our operator<<(ostream,exception) to
+      //    lowercase the first word that follows ": " (code description) for
+      //    exceptions derived from system_error.
+      //
+      string msg ("unable to open stdout redirect file '" + redir->string () +
+                  "'");
+
+      // For old versions of g++ (as of 4.9) ios_base::failure is not derived
+      // from system_error and so we cannot recover the errno value. Lets use
+      // EIO in this case. This is a temporary code after all.
+      //
+      const system_error* se (dynamic_cast<const system_error*> (&e));
+
+      throw_generic_ios_failure (se != nullptr ? se->code ().value () : EIO,
+                                 msg.c_str ());
+    }
+
+    // Finally, run the process.
+    //
+    // If the callback is specified, then intercept its call, appending the
+    // stdout redirect to the arguments list, if present.
+    //
+    return process_run_callback (
+      [&callback, &redir, redir_append] (const char* const args[], size_t n)
+      {
+        if (callback)
+        {
+          if (redir)
+          {
+            vector<const char*> elems (args, args + n);
+            string rd ((redir_append ? ">>" : ">") + redir->string ());
+
+            // Inject the redirect prior to the trailing NULL.
+            //
+            assert (n > 0);
+
+            elems.insert (elems.begin () + n - 1, rd.c_str ());
+
+            callback (elems.data (), elems.size ());
+          }
+          else
+          {
+            callback (args, n);
+          }
+        }
+      },
+      0                     /* stdin */,
+      redir ? rd.get () : 1 /* stdout */,
+      2                     /* stderr */,
+      pe,
+      args);
+  }
+}
diff --git a/libbutl/command.mxx b/libbutl/command.mxx
new file mode 100644
index 0000000..e62a032
--- /dev/null
+++ b/libbutl/command.mxx
@@ -0,0 +1,84 @@
+// file      : libbutl/command.mxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef __cpp_modules
+#pragma once
+#endif
+
+#ifndef __cpp_lib_modules
+#include <map>
+#include <string>
+#include <cstddef>    // size_t
+#include <functional>
+#endif
+
+// Other includes.
+
+#ifdef __cpp_modules
+export module butl.command;
+#ifdef __cpp_lib_modules
+import std.core;
+#endif
+import butl.process;
+import butl.optional;
+#else
+#include <libbutl/process.mxx>
+#include <libbutl/optional.mxx>
+#endif
+
+#include <libbutl/export.hxx>
+
+LIBBUTL_MODEXPORT namespace butl
+{
+  // Run a process, interpreting the command line as whitespace-separated,
+  // potentially quoted program path, arguments, and redirects. Throw
+  // std::invalid_argument on the parsing error, ios::failure on the
+  // underlying OS error, and process_error on the process running error.
+  //
+  // The process environment path is unused and must point to the empty
+  // process path.
+  //
+  // Currently only the following stdout redirects are supported:
+  //
+  // >file   # Overwrite file.
+  // >>file  # Append to file.
+  //
+  // In particular, the file descriptor cannot be specified. The file path can
+  // optionally be separated from '>' by whitespaces. Note that redirects are
+  // distinguished from arguments by the presence of leading '>' and prior to
+  // possible substitutions (so the redirect character cannot be the result of
+  // a substitution; see below).
+  //
+  // The relative redirect file paths are completed against the command
+  // current working directory. Note that if it is altered via the process
+  // environment, then the new value is used.
+  //
+  // The command line elements (program, arguments, etc) may optionally
+  // contain substitutions - variable names enclosed with the substitution
+  // symbol ('@' by default) - which are replaced with the corresponding
+  // variable values to produce the actual command. Variable names must not
+  // contain whitespaces and an attempt to substitute an unknown or a
+  // malformed variable is an error. Double substitution character ('@@' by
+  // default) is an escape sequence.
+  //
+  // If the variable map is absent, then '@' has no special meaning and is
+  // treated as a regular character.
+  //
+  // The callback function, if specified, is called prior to running the
+  // command process with the substituted command elements and including
+  // redirects which will be in the "canonical" form (single argument without
+  // space after '>'). The callback can be used, for example, for tracing the
+  // resulting command line, etc.
+  //
+  using command_substitution_map = std::map<std::string, std::string>;
+  using command_callback = void (const char* const args[], std::size_t n);
+
+  LIBBUTL_SYMEXPORT process_exit
+  command_run (const std::string& command,
+               const optional<process_env>& = nullopt,
+               const optional<command_substitution_map>& = nullopt,
+               char subst = '@',
+               const std::function<command_callback>& = {});
+
+}
diff --git a/tests/command/buildfile b/tests/command/buildfile
new file mode 100644
index 0000000..cb0f272
--- /dev/null
+++ b/tests/command/buildfile
@@ -0,0 +1,8 @@
+# file      : tests/command/buildfile
+# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+import libs = libbutl%lib{butl}
+libs += $stdmod_lib
+
+exe{driver}: {hxx cxx}{*} $libs testscript
diff --git a/tests/command/driver.cxx b/tests/command/driver.cxx
new file mode 100644
index 0000000..a1baf68
--- /dev/null
+++ b/tests/command/driver.cxx
@@ -0,0 +1,255 @@
+// file      : tests/command/driver.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <cassert>
+
+#ifndef __cpp_lib_modules
+#include <ios>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <stdexcept> // invalid_argument
+#endif
+
+// Other includes.
+
+#ifdef __cpp_modules
+#ifdef __cpp_lib_modules
+import std.core;
+import std.io;
+#endif
+import butl.path;
+import butl.path_io;
+import butl.process; // process::print()
+import butl.command;
+import butl.utility;
+import butl.optional;
+#else
+#include <libbutl/path.mxx>
+#include <libbutl/path-io.mxx>
+#include <libbutl/process.mxx>
+#include <libbutl/command.mxx>
+#include <libbutl/utility.mxx>
+#include <libbutl/optional.mxx>
+#endif
+
+using namespace std;
+using namespace butl;
+
+// Usages:
+//
+// argv[0] [-d <dir>] [-v <name>[=<value>]] [-s <name>=<value>] [-c <char>]
+//         [-p] <command>
+//
+// argv[0] -C [-A] [-D] [-V <name>] [-S <status>] <arguments>
+//
+// In the first form run the specified command, changing the current
+// directory, (re)setting the environment variables, performing substitutions,
+// and printing the "expanded" command line, if requested.
+//
+// In the second form optionally print the program arguments, CWD, the
+// environment variable values and exit with the status specified. This mode
+// is normally used for the command being tested to dump the environment
+// obtained from the caller.
+//
+// -d <dir>
+//    Change the CWD for the command process.
+//
+// -v <name>[=<value>]
+//    (Un)set the environment variable for the command process. Can be
+//    specified multiple times.
+//
+// -s <name>=<value>
+//    Perform command line substitutions using the specified variable. Can be
+//    specified multiple times.
+//
+// -c <char>
+//    Substitution symbol. The default is '@'.
+//
+// -p
+//    Print the "expanded" command line.
+//
+// -C
+//    Print the program arguments (-A), CWD (-D), environment variables (-V)
+//    to stdout and exit with the status specifies (-S).
+//
+// -A
+//    Print the program arguments to stdout.
+//
+// -D
+//    Print the process CWD to stdout.
+//
+// -V <name>
+//    Print the environment variable value (or <unset>) to stdout. Can be
+//    specified multiple times.
+//
+// -S <status>
+//    Exit with the specified status code.
+//
+int
+main (int argc, const char* argv[])
+{
+  using butl::optional;
+  using butl::getenv;
+
+  // Parse and validate the arguments.
+  //
+  dir_path cwd;
+  vector<const char*> vars;
+  optional<command_substitution_map> substitutions;
+  char subst ('@');
+  optional<string> command;
+  bool print (false);
+
+  for (int i (1); i != argc; ++i)
+  {
+    string o (argv[i]);
+
+    if (o == "-d")
+    {
+      assert (++i != argc);
+      cwd = dir_path (argv[i]);
+    }
+    else if (o == "-v")
+    {
+      assert (++i != argc);
+      vars.push_back (argv[i]);
+    }
+    else if (o == "-s")
+    {
+      assert (++i != argc);
+
+      if (!substitutions)
+        substitutions = command_substitution_map ();
+
+      string v (argv[i]);
+      size_t p (v.find ('='));
+
+      assert (p != string::npos && p != 0);
+
+      (*substitutions)[string (v, 0, p)] = string (v, p + 1);
+    }
+    else if (o == "-c")
+    {
+      assert (++i != argc);
+
+      string v (argv[i]);
+
+      assert (v.size () == 1);
+      subst = v[0];
+    }
+    else if (o == "-p")
+    {
+      print = true;
+    }
+    else if (o == "-C")
+    {
+      assert (i == 1); // Must go first.
+
+      int ec (0);
+      bool print_cwd (false);
+
+      // Include the program path into the arguments list.
+      //
+      vector<const char*> args ({argv[0]});
+
+      for (++i; i != argc; ++i)
+      {
+        o = argv[i];
+
+        if (o == "-A")
+        {
+          print = true;
+        }
+        else if (o == "-D")
+        {
+          print_cwd = true;
+        }
+        else if (o == "-V")
+        {
+          assert (++i != argc);
+          vars.push_back (argv[i]);
+        }
+        else if (o == "-S")
+        {
+          assert (++i != argc);
+          ec = stoi (argv[i]);
+        }
+        else
+          args.push_back (argv[i]);
+      }
+
+      args.push_back (nullptr);
+
+      if (print)
+      {
+        process::print (cout, args.data ());
+        cout << endl;
+      }
+
+      if (print_cwd)
+        cout << dir_path::current_directory () << endl;
+
+      for (const auto& v: vars)
+      {
+        optional<string> vv (getenv (v));
+        cout << (vv ? *vv : "<unset>") << endl;
+      }
+
+      return ec;
+    }
+    else
+    {
+      assert (!command);
+      command = argv[i];
+    }
+  }
+
+  assert (command);
+
+  // Run the command.
+  //
+  try
+  {
+    optional<process_env> pe;
+
+    if (!cwd.empty () || !vars.empty ())
+      pe = process_env (process_path (), cwd, vars);
+
+    process_exit e (command_run (*command,
+                                 pe,
+                                 substitutions,
+                                 subst,
+                                 [print] (const char* const args[], size_t n)
+                                 {
+                                   if (print)
+                                   {
+                                     process::print (cout, args, n);
+                                     cout << endl;
+                                   }
+                                 }));
+
+    if (!e)
+      cerr << "process " << argv[0] << " " << e << endl;
+
+    return e.normal () ? e.code () : 1;
+  }
+  catch (const invalid_argument& e)
+  {
+    cerr << e << endl;
+    return 1;
+  }
+  catch (const ios::failure& e)
+  {
+    cerr << e << endl;
+    return 1;
+  }
+  catch (const process_error& e)
+  {
+    cerr << e << endl;
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/tests/command/testscript b/tests/command/testscript
new file mode 100644
index 0000000..4d45f4f
--- /dev/null
+++ b/tests/command/testscript
@@ -0,0 +1,155 @@
+# file      : tests/command/testscript
+# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+cmd="'$0' -C -A" # Command that prints its arguments to stdout.
+
+# Note that when cross-testing the driver may not be able to run the command
+# due to the meaningless program path.
+#
++if ($test.target != $build.host)
+  exit
+end
+
+: quioting
+:
+{
+  $* "$cmd 'abc def'" >~'%.+ "abc def"%'
+}
+
+: substitution
+:
+{
+  test.options += -s v1=abc -s v2=def
+
+  : program
+  :
+  {
+    $* -s "program=$0" '@program@ -C -A abc' >~'%.+driver.* abc%'
+  }
+
+  : args
+  :
+  {
+    $* "$cmd abc"         >~'%.+ abc%'       : none
+    $* "$cmd x@v1@"       >~'%.+ xabc%'      : single
+    $* "$cmd x@v1@y@v2@z" >~'%.+ xabcydefz%' : multiple
+    $* "$cmd @v1@@v2@"    >~'%.+ abcdef%'    : adjacent
+  }
+
+  : redirect
+  :
+  {
+    $* -s v=f "$cmd abc >@v@" &f;
+    cat f >~'%.+ abc%'
+  }
+}
+
+: redirect
+:
+{
+  : overwrite
+  :
+  {
+    $* -p "$cmd abc >f" >~'%.+driver.* -C -A abc >f%' &f;
+    cat f >~'%.+ abc%'
+  }
+
+  : append
+  :
+  {
+    echo 'xyz' >=f;
+    $* -p "$cmd abc >>f" >~'%.+driver.* -C -A abc >>f%';
+
+    cat f >>~%EOO%
+      xyz
+      %.+ abc%
+      EOO
+  }
+
+  : space-separated
+  :
+  {
+    : overwrite
+    :
+    {
+      $* "$cmd abc > f" &f;
+      cat f >~'%.+ abc%'
+    }
+
+    : append
+    :
+    {
+      echo 'xyz' >=f;
+      $* "$cmd abc >> f";
+
+      cat f >>~%EOO%
+        xyz
+        %.+ abc%
+        EOO
+    }
+  }
+
+  : not-redirect
+  :
+  $* -s v='>f' "$cmd abc @v@" >~'%.+ abc >f%'
+
+  : errors
+  :
+  {
+    $* "$cmd >d/f" 2>~"%unable to open stdout redirect file 'd/f'.*%" != 0 : io-failure
+    $* "$cmd > ''" 2>  'empty stdout redirect file path'              != 0 : empty-path
+  }
+}
+
+: invalid-argument
+:
+{
+  $* ""           2>'no program path specified'         != 0 : no-prog
+  $* "p 'abc def" 2>'unterminated quoted string'        != 0 : unterminated
+  $* "p >"        2>'no stdout redirect file specified' != 0 : no-redirect-file
+  $* "p >>"       2>'no stdout redirect file specified' != 0 : no-append-file
+
+  : substitution
+  :
+  {
+    test.options += -s v=a
+
+    $* 'p @a b@'   2>"unmatched substitution character '@' in '@a'" != 0 : unterm-var
+    $* "p '@a b@'" 2>"whitespace in variable name 'a b'"            != 0 : ws-var
+    $* 'p @x@'     2>"unknown variable 'x'"                         != 0 : unknown-var
+  }
+}
+
+: process
+:
+{
+  : cwd
+  :
+  {
+    mkdir abc;
+    $* -d abc "$cmd -D" >>/~%EOO%
+      %.+/driver.*%
+      %.+/abc%
+      EOO
+  }
+
+  : env-var
+  :
+  {
+    $* -v test=abc "$cmd -V test" >>/~%EOO%
+      %.+/driver.*%
+      abc
+      EOO
+  }
+
+  : error
+  :
+  {
+    $* "''" 2>'no such file or directory' != 0 : empty-prog
+  }
+
+  : non-zero-status
+  :
+  $* "$0 -C -S 10" 2>/~'%.+ exited with code 10%' == 10
+}