Implement run buildfile directive

Now we can do:

run echo 'foo = bar'
print $foo

8 files changed, 300 insertions, 102 deletions

diff --git a/build2/diagnostics.cxx b/build2/diagnostics.cxx
index f04751c..850c83f 100644
--- a/build2/diagnostics.cxx
+++ b/build2/diagnostics.cxx
@@ -65,21 +65,24 @@ namespace build2
   {
     stream_verb (r.os, sverb_);
 
-    r << *loc_.file << ':';
-
-    if (!ops.no_line ())
+    if (!loc_.empty ())
     {
-      if (loc_.line != 0)
-        r << loc_.line << ':';
+      r << *loc_.file << ':';
 
-      if (!ops.no_column ())
+      if (!ops.no_line ())
       {
-        if (loc_.column != 0)
-          r << loc_.column << ':';
+        if (loc_.line != 0)
+          r << loc_.line << ':';
+
+        if (!ops.no_column ())
+        {
+          if (loc_.column != 0)
+            r << loc_.column << ':';
+        }
       }
-    }
 
-    r << ' ';
+      r << ' ';
+    }
 
     if (type_ != nullptr)
       r << type_ << ": ";
diff --git a/build2/diagnostics.hxx b/build2/diagnostics.hxx
index 5ec2ef0..173b059 100644
--- a/build2/diagnostics.hxx
+++ b/build2/diagnostics.hxx
@@ -279,24 +279,6 @@ namespace build2
     const stream_verbosity sverb_;
   };
 
-  class location
-  {
-  public:
-    // Note that location maintains a shallow reference to path. Zero lines
-    // or columns are not printed.
-    //
-    location (): file (nullptr), line (0), column (0) {}
-    location (const path* f, uint64_t l = 0, uint64_t c = 0)
-        : file (f), line (l), column (c) {}
-
-    bool
-    empty () const {return file == nullptr;}
-
-    const path* file;
-    uint64_t line;
-    uint64_t column;
-  };
-
   struct location_prologue_base
   {
     location_prologue_base (const char* type,
diff --git a/build2/parser.cxx b/build2/parser.cxx
index 0c360a5..15235a2 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -388,6 +388,10 @@ namespace build2
         {
           f = &parser::parse_include;
         }
+        else if (n == "run")
+        {
+          f = &parser::parse_run;
+        }
         else if (n == "import")
         {
           f = &parser::parse_import;
@@ -929,10 +933,56 @@ namespace build2
   }
 
   void parser::
-  parse_source (token& t, type& tt)
+  source (istream& is,
+          const path& p,
+          const location& loc,
+          bool enter,
+          bool deft)
   {
-    tracer trace ("parser::parse_source", &path_);
+    tracer trace ("parser::source", &path_);
+
+    l5 ([&]{trace (loc) << "entering " << p;});
+
+    if (enter)
+      enter_buildfile (p);
+
+    const path* op (path_);
+    path_ = &p;
+
+    lexer l (is, *path_);
+    lexer* ol (lexer_);
+    lexer_ = &l;
+
+    target* odt;
+    if (deft)
+    {
+      odt = default_target_;
+      default_target_ = nullptr;
+    }
+
+    token t;
+    type tt;
+    next (t, tt);
+    parse_clause (t, tt);
+
+    if (tt != type::eos)
+      fail (t) << "unexpected " << t;
+
+    if (deft)
+    {
+      process_default_target (t);
+      default_target_ = odt;
+    }
+
+    lexer_ = ol;
+    path_ = op;
+
+    l5 ([&]{trace (loc) << "leaving " << p;});
+  }
 
+  void parser::
+  parse_source (token& t, type& tt)
+  {
     // The rest should be a list of buildfiles. Parse them as names in the
     // value mode to get variable expansion and directory prefixes.
     //
@@ -968,30 +1018,11 @@ namespace build2
       try
       {
         ifdstream ifs (p);
-
-        l5 ([&]{trace (t) << "entering " << p;});
-
-        enter_buildfile (p);
-
-        const path* op (path_);
-        path_ = &p;
-
-        lexer l (ifs, *path_);
-        lexer* ol (lexer_);
-        lexer_ = &l;
-
-        token t;
-        type tt;
-        next (t, tt);
-        parse_clause (t, tt);
-
-        if (tt != type::eos)
-          fail (t) << "unexpected " << t;
-
-        l5 ([&]{trace (t) << "leaving " << p;});
-
-        lexer_ = ol;
-        path_ = op;
+        source (ifs,
+                p,
+                get_location (t),
+                true  /* enter */,
+                false /* default_target */);
       }
       catch (const io_error& e)
       {
@@ -1104,46 +1135,132 @@ namespace build2
       try
       {
         ifdstream ifs (p);
+        source (ifs,
+                p,
+                get_location (t),
+                true  /* enter */,
+                true  /* default_target */);
+      }
+      catch (const io_error& e)
+      {
+        fail (l) << "unable to read buildfile " << p << ": " << e;
+      }
 
-        l5 ([&]{trace (t) << "entering " << p;});
+      pbase_ = opb;
+      scope_ = ocs;
+      root_ = ors;
+    }
 
-        enter_buildfile (p);
+    if (tt == type::newline)
+      next (t, tt);
+    else if (tt != type::eos)
+      fail (t) << "expected newline instead of " << t;
+  }
 
-        const path* op (path_);
-        path_ = &p;
+  void parser::
+  parse_run (token& t, type& tt)
+  {
+    // run <name> [<arg>...]
+    //
 
-        lexer l (ifs, *path_);
-        lexer* ol (lexer_);
-        lexer_ = &l;
+    // Parse the command line as names in the value mode to get variable
+    // expansion, etc.
+    //
+    mode (lexer_mode::value);
+    next (t, tt);
+    const location l (get_location (t));
 
-        target* odt (default_target_);
-        default_target_ = nullptr;
+    strings args;
+    try
+    {
+      args = convert<strings> (tt != type::newline && tt != type::eos
+                               ? parse_names (t, tt,
+                                              pattern_mode::ignore,
+                                              false,
+                                              "argument",
+                                              nullptr)
+                               : names ());
+    }
+    catch (const invalid_argument& e)
+    {
+      fail (l) << "invalid run argument: " << e.what ();
+    }
 
-        token t;
-        type tt;
-        next (t, tt);
-        parse_clause (t, tt);
+    if (args.empty () || args[0].empty ())
+      fail (l) << "executable name expected after run";
+
+    cstrings cargs;
+    cargs.reserve (args.size () + 1);
+    transform (args.begin (),
+               args.end (),
+               back_inserter (cargs),
+               [] (const string& s) {return s.c_str ();});
+    cargs.push_back (nullptr);
+
+    process pr (run_start (3               /* verbosity */,
+                           cargs,
+                           0               /* stdin  */,
+                           -1              /* stdout */,
+                           true            /* error  */,
+                           empty_dir_path  /* cwd    */,
+                           l));
+    bool bad (false);
+    try
+    {
+      // While a failing process could write garbage to stdout, for simplicity
+      // let's assume it is well behaved.
+      //
+      ifdstream is (move (pr.in_ofd), fdstream_mode::skip);
 
-        if (tt != type::eos)
-          fail (t) << "unexpected " << t;
+      // Come up with a "path" that contains both the original buildfile
+      // location as well as the process name. The resulting diagnostics will
+      // look like this:
+      //
+      // buildfile:10:1 (foo stdout): unterminated single quote
+      //
+      path p;
+      {
+        string s (l.file->string ());
+        s += ':';
 
-        process_default_target (t);
+        if (!ops.no_line ())
+        {
+          s += to_string (l.line);
+          s += ':';
 
-        l5 ([&]{trace (t) << "leaving " << p;});
+          if (!ops.no_column ())
+          {
+            s += to_string (l.column);
+            s += ':';
+          }
+        }
 
-        default_target_ = odt;
-        lexer_ = ol;
-        path_ = op;
-      }
-      catch (const io_error& e)
-      {
-        fail (l) << "unable to read buildfile " << p << ": " << e;
+        s += " (";
+        s += args[0];
+        s += " stdout)";
+        p = path (move (s));
       }
 
-      pbase_ = opb;
-      scope_ = ocs;
-      root_ = ors;
+      source (is,
+              p,
+              l,
+              false /* enter */,
+              false /* default_target */);
+
+      is.close (); // Detect errors.
     }
+    catch (const io_error&)
+    {
+      // Presumably the child process failed and issued diagnostics so let
+      // run_finish() try to deal with that first.
+      //
+      bad = true;
+    }
+
+    run_finish (cargs, pr, l);
+
+    if (bad)
+      fail (l) << "error reading " << args[0] << " output";
 
     if (tt == type::newline)
       next (t, tt);
diff --git a/build2/parser.hxx b/build2/parser.hxx
index 3c51801..1b39491 100644
--- a/build2/parser.hxx
+++ b/build2/parser.hxx
@@ -94,6 +94,9 @@ namespace build2
     parse_include (token&, token_type&);
 
     void
+    parse_run (token&, token_type&);
+
+    void
     parse_import (token&, token_type&);
 
     void
@@ -197,6 +200,16 @@ namespace build2
     attributes&
     attributes_top () {return attributes_.top ();}
 
+    // Source a stream optionnaly entering it as a buildfile and performing
+    // the default target processing.
+    //
+    void
+    source (istream&,
+            const path&,
+            const location&,
+            bool enter,
+            bool default_target);
+
     // If chunk is true, then parse the smallest but complete, name-wise,
     // chunk of input. Note that in this case you may still end up with
     // multiple names, for example, {foo bar} or $foo. In the pre-parse mode
diff --git a/build2/types.hxx b/build2/types.hxx
index 86a53f2..310e689 100644
--- a/build2/types.hxx
+++ b/build2/types.hxx
@@ -273,6 +273,26 @@ namespace build2
   using butl::standard_version;
   using butl::standard_version_constraint;
 
+  // Diagnostics location.
+  //
+  class location
+  {
+  public:
+    // Note that location maintains a shallow reference to path. Zero lines
+    // or columns are not printed.
+    //
+    explicit
+    location (const path* f = nullptr, uint64_t l = 0, uint64_t c = 0)
+        : file (f), line (l), column (c) {}
+
+    bool
+    empty () const {return file == nullptr;}
+
+    const path* file;
+    uint64_t    line;
+    uint64_t    column;
+  };
+
   // See context.
   //
   enum class run_phase {load, match, execute};
diff --git a/build2/utility.cxx b/build2/utility.cxx
index 8d450df..f1def09 100644
--- a/build2/utility.cxx
+++ b/build2/utility.cxx
@@ -186,25 +186,28 @@ namespace build2
   }
 
   process_path
-  run_search (const char*& args0)
+  run_search (const char*& args0, const location& l)
   try
   {
     return process::path_search (args0);
   }
   catch (const process_error& e)
   {
-    fail << "unable to execute " << args0 << ": " << e << endf;
+    fail (l) << "unable to execute " << args0 << ": " << e << endf;
   }
 
   process_path
-  run_search (const path& f, bool init, const dir_path& fallback)
+  run_search (const path& f,
+              bool init,
+              const dir_path& fallback,
+              const location& l)
   try
   {
     return process::path_search (f, init, fallback);
   }
   catch (const process_error& e)
   {
-    fail << "unable to execute " << f << ": " << e << endf;
+    fail (l) << "unable to execute " << f << ": " << e << endf;
   }
 
   process
@@ -214,7 +217,8 @@ namespace build2
              int in,
              int out,
              bool err,
-             const dir_path& cwd)
+             const dir_path& cwd,
+             const location& l)
   try
   {
     assert (args[0] == pp.recall_string ());
@@ -239,11 +243,15 @@ namespace build2
       exit (1);
     }
     else
-      fail << "unable to execute " << args[0] << ": " << e << endf;
+      fail (l) << "unable to execute " << args[0] << ": " << e << endf;
   }
 
   bool
-  run_finish (const char* args[], process& pr, bool err, const string& l)
+  run_finish (const char* args[],
+              process& pr,
+              bool err,
+              const string& l,
+              const location& loc)
   try
   {
     tracer trace ("run_finish");
@@ -254,7 +262,7 @@ namespace build2
     const process_exit& e (*pr.exit);
 
     if (!e.normal ())
-      fail << "process " << args[0] << " " << e;
+      fail (loc) << "process " << args[0] << " " << e;
 
     // Normall but non-zero exit status.
     //
@@ -275,13 +283,13 @@ namespace build2
     // result in a single error line printed by run_start() above.
     //
     if (l.compare (0, 18, "unable to execute ") == 0)
-      fail << l;
+      fail (loc) << l;
 
     return false;
   }
   catch (const process_error& e)
   {
-    fail << "unable to execute " << args[0] << ": " << e << endf;
+    fail (loc) << "unable to execute " << args[0] << ": " << e << endf;
   }
 
   const string empty_string;
diff --git a/build2/utility.hxx b/build2/utility.hxx
index e5c21e7..d5809bc 100644
--- a/build2/utility.hxx
+++ b/build2/utility.hxx
@@ -178,10 +178,13 @@ namespace build2
   // case of an error.
   //
   process_path
-  run_search (const char*& args0);
+  run_search (const char*& args0, const location& = location ());
 
   process_path
-  run_search (const path&, bool init, const dir_path& fallback = dir_path ());
+  run_search (const path&,
+              bool init,
+              const dir_path& fallback = dir_path (),
+              const location& = location ());
 
   // Wait for process termination. Issue diagnostics and throw failed in case
   // of abnormal termination. If the process has terminated normally but with
@@ -194,12 +197,13 @@ namespace build2
   run_finish (const char* args[],
               process&,
               bool error = true,
-              const string& = string ());
+              const string& = string (),
+              const location& = location ());
 
   inline void
-  run_finish (cstrings& args, process& pr)
+  run_finish (cstrings& args, process& pr, const location& l = location ())
   {
-    run_finish (args.data (), pr);
+    run_finish (args.data (), pr, true, string (), l);
   }
 
   // Start a process with the specified arguments. If in is -1, then redirect
@@ -216,7 +220,8 @@ namespace build2
              int in,
              int out,
              bool error = true,
-             const dir_path& cwd = dir_path ());
+             const dir_path& cwd = dir_path (),
+             const location& = location ());
 
   inline process
   run_start (const process_path& pp,
@@ -224,9 +229,10 @@ namespace build2
              int in,
              int out,
              bool error = true,
-             const dir_path& cwd = dir_path ())
+             const dir_path& cwd = dir_path (),
+             const location& l = location ())
   {
-    return run_start (verb_never, pp, args, in, out, error, cwd);
+    return run_start (verb_never, pp, args, in, out, error, cwd, l);
   }
 
   inline void
@@ -255,10 +261,23 @@ namespace build2
              int in,
              int out,
              bool error = true,
-             const dir_path& cwd = dir_path ())
+             const dir_path& cwd = dir_path (),
+             const location& l = location ())
   {
-    process_path pp (run_search (args[0]));
-    return run_start (verbosity, pp, args, in, out, error, cwd);
+    process_path pp (run_search (args[0], l));
+    return run_start (verbosity, pp, args, in, out, error, cwd, l);
+  }
+
+  inline process
+  run_start (uint16_t verbosity,
+             cstrings& args,
+             int in,
+             int out,
+             bool error = true,
+             const dir_path& cwd = dir_path (),
+             const location& l = location ())
+  {
+    return run_start (verbosity, args.data (), in, out, error, cwd, l);
   }
 
   inline void
diff --git a/tests/directive/run.test b/tests/directive/run.test
new file mode 100644
index 0000000..a6573f2
--- /dev/null
+++ b/tests/directive/run.test
@@ -0,0 +1,36 @@
+# file      : tests/directive/run.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# We are going to run the build system driver so no cross-testing.
+#
+crosstest = false
+
+.include ../common.test
+
+: no-output
+:
+cat <'assert true' >=buildfile;
+$* <"run $0 noop"
+
+: output
+:
+cat <'print foo=bar' >=buildfile;
+$* <<"EOI" >'bar'
+run $0 noop
+print \$foo
+EOI
+
+: bad-exit
+:
+cat <'assert false' >=buildfile;
+$* <"run $0 noop" 2>>EOE != 0
+buildfile:1:1: error: assertion failed
+EOE
+
+: bad-output
+:
+cat <'print run' >=buildfile;
+$* <"run $0 noop" 2>>~%EOE% != 0
+%-:1:5: \(.+ stdout\):1:4: error: executable name expected after run%
+EOE