Add cat, false and true builtins

18 files changed, 421 insertions, 60 deletions

diff --git a/bootstrap.sh b/bootstrap.sh
index 15410cb..3496a1f 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -131,4 +131,4 @@ src="$src build2/pkgconfig/*.cxx"
 src="$src $libbutl/butl/*.cxx"
 
 set -x
-"$cxx" "-I$libbutl" -I. '-DBUILD2_HOST_TRIPLET="'"$host"'"' -std=c++1y "$@" -o build2/b-boot $src
+"$cxx" "-I$libbutl" -I. '-DBUILD2_HOST_TRIPLET="'"$host"'"' -std=c++1y "$@" -o build2/b-boot $src -lpthread
diff --git a/build2/buildfile b/build2/buildfile
index 86b67a9..cd04ce7 100644
--- a/build2/buildfile
+++ b/build2/buildfile
@@ -88,6 +88,9 @@ test/script/{hxx         cxx}{ token            } \
 #
 obj{b context}: cxx.poptions += -DBUILD2_HOST_TRIPLET=\"$cxx.target\"
 
+if ($cxx.target.class != "windows")
+  cxx.libs += -lpthread
+
 # Generated options parser.
 #
 {hxx ixx cxx}{b-options}: cli{b}
diff --git a/build2/test/script/builtin b/build2/test/script/builtin
index e3c16b3..bd5fe50 100644
--- a/build2/test/script/builtin
+++ b/build2/test/script/builtin
@@ -5,11 +5,11 @@
 #ifndef BUILD2_TEST_SCRIPT_BUILTIN
 #define BUILD2_TEST_SCRIPT_BUILTIN
 
+#include <map>
+
 #include <build2/types>
 #include <build2/utility>
 
-#include <map>
-
 namespace build2
 {
   namespace test
@@ -18,21 +18,26 @@ namespace build2
     {
       class scope;
 
+      // Start builtin command. Throw system_error on failure.
+      //
       // Note that unlike argc/argv, our args don't include the program name.
       //
-      using builtin = int (*) (scope&,
-                               const strings& args,
-                               auto_fd in, auto_fd out, auto_fd err);
+      // Also note that the future object being returned doesn't block in dtor
+      // until the builtin command terminates.
+      //
+      using builtin = future<uint8_t> (scope&,
+                                       const strings& args,
+                                       auto_fd in, auto_fd out, auto_fd err);
 
-      class builtin_map: public std::map<string, builtin>
+      class builtin_map: public std::map<string, builtin*>
       {
       public:
-        using base = std::map<string, builtin>;
+        using base = std::map<string, builtin*>;
         using base::base;
 
         // Return NULL if not a builtin.
         //
-        builtin
+        builtin*
         find (const string& n) const
         {
           auto i (base::find (n));
diff --git a/build2/test/script/builtin.cxx b/build2/test/script/builtin.cxx
index 9241447..2dcfde6 100644
--- a/build2/test/script/builtin.cxx
+++ b/build2/test/script/builtin.cxx
@@ -10,12 +10,22 @@
 #  include <sys/utime.h>
 #endif
 
+#include <thread>
+
 #include <butl/path-io>    // use default operator<< implementation
-#include <butl/fdstream>   // fdopen_mode
+#include <butl/fdstream>   // fdopen_mode, fdstream_mode
 #include <butl/filesystem> // mkdir_status
 
 #include <build2/test/script/script>
 
+// Strictly speaking a builtin which reads/writes from/to standard streams
+// must be asynchronous so that the caller can communicate with it through
+// pipes without being blocked on I/O operations. However, as an optimization,
+// we allow builtins that only print diagnostics to STDERR to be synchronous
+// assuming that their output will always fit the pipe buffer. Synchronous
+// builtins must not read from STDIN and write to STDOUT. Later we may relax
+// this rule to allow a "short" output for such builtins.
+//
 using namespace std;
 using namespace butl;
 
@@ -25,6 +35,18 @@ namespace build2
   {
     namespace script
     {
+      using builtin_impl = uint8_t (scope&,
+                                    const strings& args,
+                                    auto_fd in, auto_fd out, auto_fd err);
+      static future<uint8_t>
+      to_future (uint8_t status)
+      {
+        promise<uint8_t> p;
+        future<uint8_t> f (p.get_future ());
+        p.set_value (status);
+        return f;
+      }
+
       // Operation failed, diagnostics has already been issued.
       //
       struct failed {};
@@ -48,13 +70,141 @@ namespace build2
         return p;
       }
 
+      // Builtin commands functions.
+      //
+
+      // cat <file>...
+      //
+      // Read files in sequence and write their contents to STDOUT in the same
+      // sequence. Read from STDIN if no argumements provided or '-' is
+      // specified as a file path. STDIN, STDOUT and file streams are set to
+      // binary mode prior to I/O operations.
+      //
+      // Note that POSIX doesn't specify if after I/O operation failure the
+      // command should proceed with the rest of the arguments. The current
+      // implementation exits immediatelly in such a case.
+      //
+      // @@ Shouldn't we check that we don't print a nonempty regular file to
+      //    itself, as that would merely exhaust the output device? POSIX
+      //    allows (but not requires) such a check and some implementations do
+      //    this. That would require to fstat() file descriptors and complicate
+      //    the code a bit. Was able to reproduce on a big file (should be
+      //    bigger than the stream buffer size) with the test
+      //    'cat file >>>&file'.
+      //
+      // Note: must be executed asynchronously.
+      //
+      static uint8_t
+      cat (scope& sp,
+           const strings& args,
+           auto_fd in, auto_fd out, auto_fd err) noexcept
+      try
+      {
+        uint8_t r (1);
+        ofdstream cerr (move (err));
+
+        try
+        {
+          ifdstream cin  (move (in),  fdstream_mode::binary);
+          ofdstream cout (move (out), fdstream_mode::binary);
+
+          // Copy input stream to STDOUT.
+          //
+          auto copy = [&cout] (istream& is)
+          {
+            if (is.peek () != ifdstream::traits_type::eof ())
+              cout << is.rdbuf ();
+
+            is.clear (istream::eofbit); // Sets eofbit.
+          };
+
+          // Path of a file being printed to STDOUT. An empty path represents
+          // STDIN. Used in diagnostics.
+          //
+          path p;
+
+          try
+          {
+            // Print STDIN.
+            //
+            if (args.empty ())
+              copy (cin);
+
+            // Print files.
+            //
+            for (auto i (args.begin ()); i != args.end (); ++i)
+            {
+              if (*i == "-")
+              {
+                if (!cin.eof ())
+                {
+                  p.clear ();
+                  copy (cin);
+                }
+
+                continue;
+              }
+
+              p = parse_path (*i, sp.wd_path);
+
+              ifdstream is (p, ifdstream::binary);
+              copy (is);
+              is.close ();
+            }
+          }
+          catch (const io_error& e)
+          {
+            cerr << "cat: unable to print ";
+
+            if (p.empty ())
+              cerr << "stdin";
+            else
+              cerr << "'" << p << "'";
+
+            cerr << ": " << e.what () << endl;
+            throw failed ();
+          }
+
+          cin.close ();
+          cout.close ();
+          r = 0;
+        }
+        catch (const invalid_path& e)
+        {
+          cerr << "cat: invalid path '" << e.path << "'" << endl;
+        }
+        // Can be thrown while closing cin, cout or writing to cerr (that's
+        // why need to check its state before writing).
+        //
+        catch (const io_error& e)
+        {
+          if (cerr.good ())
+            cerr << "cat: " << e.what () << endl;
+        }
+        catch (const failed&)
+        {
+          // Diagnostics has already been issued.
+        }
+
+        cerr.close ();
+        return r;
+      }
+      catch (const std::exception&)
+      {
+        return 1;
+      }
+
       // echo <string>...
       //
-      static int
-      echo (scope&, const strings& args, auto_fd in, auto_fd out, auto_fd err)
+      // Note: must be executed asynchronously.
+      //
+      static uint8_t
+      echo (scope&,
+            const strings& args,
+            auto_fd in, auto_fd out, auto_fd err) noexcept
       try
       {
-        int r (1);
+        uint8_t r (1);
         ofdstream cerr (move (err));
 
         try
@@ -83,6 +233,26 @@ namespace build2
         return 1;
       }
 
+      // false
+      //
+      // Return 1. Failure to close the file descriptors is silently ignored.
+      //
+      static future<uint8_t>
+      false_ (scope&, const strings&, auto_fd, auto_fd, auto_fd)
+      {
+        return to_future (1);
+      }
+
+      // true
+      //
+      // Return 0. Failure to close the file descriptors is silently ignored.
+      //
+      static future<uint8_t>
+      true_ (scope&, const strings&, auto_fd, auto_fd, auto_fd)
+      {
+        return to_future (0);
+      }
+
       // Create a directory if not exist and its parent directories if
       // necessary. Throw system_error on failure. Register created
       // directories for cleanup. The directory path must be absolute.
@@ -106,20 +276,19 @@ namespace build2
       //    Create any missing intermediate pathname components. Each argument
       //    that names an existing directory must be ignored without error.
       //
-      static int
+      // Note that POSIX doesn't specify if after a directory creation failure
+      // the command should proceed with the rest of the arguments. The current
+      // implementation exits immediatelly in such a case.
+      //
+      // Note: can be executed synchronously.
+      //
+      static uint8_t
       mkdir (scope& sp,
              const strings& args,
-             auto_fd in, auto_fd out, auto_fd err)
+             auto_fd in, auto_fd out, auto_fd err) noexcept
       try
       {
-        // @@ Should we set a proper verbosity so paths get printed as
-        //    relative? Can be inconvenient for end-user when build2 runs from
-        //    a testscript.
-        //
-        //    No, don't think so. If this were an external program, there
-        //    won't be such functionality.
-        //
-        int r (1);
+        uint8_t r (1);
         ofdstream cerr (move (err));
 
         try
@@ -170,7 +339,6 @@ namespace build2
             {
               cerr << "mkdir: unable to create directory '" << p << "': "
                    << e.what () << endl;
-
               throw failed ();
             }
           }
@@ -181,6 +349,14 @@ namespace build2
         {
           cerr << "mkdir: invalid path '" << e.path << "'" << endl;
         }
+        // Can be thrown while closing in, out or writing to cerr (that's why
+        // need to check its state before writing).
+        //
+        catch (const io_error& e)
+        {
+          if (cerr.good ())
+            cerr << "mkdir: " << e.what () << endl;
+        }
         catch (const failed&)
         {
           // Diagnostics has already been issued.
@@ -194,7 +370,7 @@ namespace build2
         return 1;
       }
 
-      // touch <path>...
+      // touch <file>...
       //
       // Change file access and modification times to the current time. Create
       // a file if doesn't exist. Fail if a file system entry other than file
@@ -203,13 +379,19 @@ namespace build2
       // Note that POSIX doesn't specify the behavior for touching an entry
       // other than file.
       //
-      static int
+      // Also note that POSIX doesn't specify if after a file touch failure the
+      // command should proceed with the rest of the arguments. The current
+      // implementation exits immediatelly in such a case.
+      //
+      // Note: can be executed synchronously.
+      //
+      static uint8_t
       touch (scope& sp,
              const strings& args,
-             auto_fd in, auto_fd out, auto_fd err)
+             auto_fd in, auto_fd out, auto_fd err) noexcept
       try
       {
-        int r (1);
+        uint8_t r (1);
         ofdstream cerr (move (err));
 
         try
@@ -283,6 +465,14 @@ namespace build2
         {
           cerr << "touch: invalid path '" << e.path << "'" << endl;
         }
+        // Can be thrown while closing in, out or writing to cerr (that's why
+        // need to check its state before writing).
+        //
+        catch (const io_error& e)
+        {
+          if (cerr.good ())
+            cerr << "touch: " << e.what () << endl;
+        }
         catch (const failed&)
         {
           // Diagnostics has already been issued.
@@ -296,11 +486,70 @@ namespace build2
         return 1;
       }
 
+      static void
+      thread_thunk (builtin_impl* fn,
+                    scope& sp,
+                    const strings& args,
+                    auto_fd in, auto_fd out, auto_fd err,
+                    promise<uint8_t> p)
+      {
+        // The use of set_value_at_thread_exit() would be more appropriate but
+        // the function is not supported by old versions of g++ (e.g., not in
+        // 4.9). There could also be overhead associated with it.
+        //
+        p.set_value (fn (sp, args, move (in), move (out), move (err)));
+      }
+
+      // Run builtin implementation asynchronously.
+      //
+      static future<uint8_t>
+      async_impl (builtin_impl fn,
+                  scope& sp,
+                  const strings& args,
+                  auto_fd in, auto_fd out, auto_fd err)
+      {
+        promise<uint8_t> p;
+        future<uint8_t> f (p.get_future ());
+
+        thread t (thread_thunk,
+                  fn,
+                  ref (sp),
+                  cref (args),
+                  move (in), move (out), move (err),
+                  move (p));
+
+        t.detach ();
+        return f;
+      }
+
+      template <builtin_impl fn>
+      static future<uint8_t>
+      async_impl (scope& sp,
+                  const strings& args,
+                  auto_fd in, auto_fd out, auto_fd err)
+      {
+        return async_impl (fn, sp, args, move (in), move (out), move (err));
+      }
+
+      // Run builtin implementation synchronously.
+      //
+      template <builtin_impl fn>
+      static future<uint8_t>
+      sync_impl (scope& sp,
+                 const strings& args,
+                 auto_fd in, auto_fd out, auto_fd err)
+      {
+        return to_future (fn (sp, args, move (in), move (out), move (err)));
+      }
+
       const builtin_map builtins
       {
-        {"echo",  &echo},
-        {"mkdir", &mkdir},
-        {"touch", &touch}
+        {"cat",   &async_impl<&cat>},
+        {"echo",  &async_impl<&echo>},
+        {"false", &false_},
+        {"mkdir", &sync_impl<&mkdir>},
+        {"touch", &sync_impl<&touch>},
+        {"true",  &true_}
       };
     }
   }
diff --git a/build2/test/script/parser b/build2/test/script/parser
index a81ddf0..fdfbe11 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -158,6 +158,12 @@ namespace build2
         const string&
         insert_id (string, location);
 
+        // Set lexer pointers for both the current and the base classes.
+        //
+      protected:
+        void
+        set_lexer (lexer* l);
+
       protected:
         using base_parser = build2::parser;
 
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index 10103f3..438e1f2 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -60,8 +60,7 @@ namespace build2
         pre_parse_ = true;
 
         lexer l (is, *path_, lexer_mode::script_line);
-        lexer_ = &l;
-        base_parser::lexer_ = &l;
+        set_lexer (&l);
 
         id_prefix_.clear ();
 
@@ -948,8 +947,7 @@ namespace build2
               path_ = &p;
 
               lexer* ol (lexer_);
-              lexer_ = &l;
-              base_parser::lexer_ = &l;
+              set_lexer (&l);
 
               string oip (id_prefix_);
               id_prefix_ += to_string (dl.line);
@@ -963,8 +961,7 @@ namespace build2
                 fail (t) << "stray " << t;
 
               id_prefix_ = oip;
-              base_parser::lexer_ = ol;
-              lexer_ = ol;
+              set_lexer (ol);
               path_ = op;
             }
             catch (const io_error& e)
@@ -2219,8 +2216,7 @@ namespace build2
 
         pre_parse_ = false;
 
-        lexer_ = nullptr;
-        base_parser::lexer_ = nullptr;
+        set_lexer (nullptr);
 
         script_ = &s;
         runner_ = &r;
@@ -2694,6 +2690,13 @@ namespace build2
 
         return p.first->first;
       }
+
+      void parser::
+      set_lexer (lexer* l)
+      {
+        lexer_ = l;
+        base_parser::lexer_ = l;
+      }
     }
   }
 }
diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx
index 1d6920a..aa35612 100644
--- a/build2/test/script/runner.cxx
+++ b/build2/test/script/runner.cxx
@@ -406,7 +406,8 @@ namespace build2
               ifd.reset (fdnull ()); // @@ Eventually will be throwing.
 
               if (ifd.get () == -1) // @@ TMP
-                throw io_error ("", error_code (errno, system_category ()));
+                throw io_error (
+                  error_code (errno, system_category ()).message ());
 
               in = -2;
             }
@@ -512,7 +513,8 @@ namespace build2
                 fd.reset (fdnull ()); // @@ Eventully will be throwing.
 
                 if (fd.get () == -1) // @@ TMP
-                  throw io_error ("", error_code (errno, system_category ()));
+                  throw io_error (
+                    error_code (errno, system_category ()).message ());
               }
               catch (const io_error& e)
               {
@@ -590,13 +592,24 @@ namespace build2
         }
 
         optional<process::status_type> status;
-        builtin b (builtins.find (c.program.string ()));
+        builtin* b (builtins.find (c.program.string ()));
 
         if (b != nullptr)
         {
           // Execute the builtin.
           //
-          status = (*b) (sp, c.arguments, move (ifd), move (ofd), move (efd));
+          try
+          {
+            future<uint8_t> f (
+              (*b) (sp, c.arguments, move (ifd), move (ofd), move (efd)));
+
+            status = f.get ();
+          }
+          catch (const system_error& e)
+          {
+            fail (ll) << "unable to execute " << c.program << " builtin: "
+                      << e.what ();
+          }
         }
         else
         {
diff --git a/build2/types b/build2/types
index 9b3dbc7..783aff0 100644
--- a/build2/types
+++ b/build2/types
@@ -8,6 +8,7 @@
 #include <tuple>
 #include <vector>
 #include <string>
+#include <future>
 #include <memory>           // unique_ptr, shared_ptr
 #include <utility>          // pair, move()
 #include <cstddef>          // size_t, nullptr_t
@@ -63,6 +64,8 @@ namespace build2
   using std::istream;
   using std::ostream;
 
+  using std::future;
+
   // Exceptions. While <exception> is included, there is no using for
   // std::exception -- use qualified.
   //
diff --git a/build2/utility b/build2/utility
index e08ae69..deefc43 100644
--- a/build2/utility
+++ b/build2/utility
@@ -5,12 +5,13 @@
 #ifndef BUILD2_UTILITY
 #define BUILD2_UTILITY
 
-#include <tuple>    // make_tuple()
-#include <memory>   // make_shared()
-#include <string>   // to_string()
-#include <utility>  // move(), forward(), declval(), make_pair()
-#include <cassert>  // assert()
-#include <iterator> // make_move_iterator()
+#include <tuple>      // make_tuple()
+#include <memory>     // make_shared()
+#include <string>     // to_string()
+#include <utility>    // move(), forward(), declval(), make_pair()
+#include <cassert>    // assert()
+#include <iterator>   // make_move_iterator()
+#include <functional> // ref(), cref()
 
 #include <butl/utility> // combine_hash(), reverse_iterate(), casecmp(),
                         // lcase()
@@ -27,6 +28,9 @@ namespace build2
   using std::forward;
   using std::declval;
 
+  using std::ref;
+  using std::cref;
+
   using std::make_pair;
   using std::make_tuple;
   using std::make_shared;
diff --git a/tests/test/script/buildfile b/tests/test/script/buildfile
index e613013..82af4dd 100644
--- a/tests/test/script/buildfile
+++ b/tests/test/script/buildfile
@@ -2,6 +2,6 @@
 # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = runner/ integration/
+d = builtin/ runner/ integration/
 ./: $d
 include $d
diff --git a/tests/test/script/builtin/buildfile b/tests/test/script/builtin/buildfile
new file mode 100644
index 0000000..baa4996
--- /dev/null
+++ b/tests/test/script/builtin/buildfile
@@ -0,0 +1,5 @@
+# file      : tests/test/script/builtin/buildfile
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: test{cat echo mkdir touch}
diff --git a/tests/test/script/builtin/cat.test b/tests/test/script/builtin/cat.test
new file mode 100644
index 0000000..7797906
--- /dev/null
+++ b/tests/test/script/builtin/cat.test
@@ -0,0 +1,57 @@
+# file      : tests/test/script/runner/cat.test
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+: in
+:
+cat <<EOI >>EOO
+foo
+bar
+EOI
+foo
+bar
+EOO
+
+: dash
+:
+cat - <<EOI >>EOO
+foo
+bar
+EOI
+foo
+bar
+EOO
+
+: file
+:
+cat <<EOI >>>out;
+foo
+bar
+EOI
+cat out >>EOO
+foo
+bar
+EOO
+
+: in-repeat
+:
+cat - <<EOI >>EOO
+foo
+bar
+EOI
+foo
+bar
+EOO
+
+: non-existent
+:
+cat in 2>- != 0 # @@ REGEX
+
+: empty-path
+:
+: Cat an empty path.
+:
+cat '' 2>"cat: invalid path ''" == 1
+
+# @@ When piping is ready test cat on a big file to test it is asynchronous.
+#
diff --git a/tests/test/script/builtin/echo.test b/tests/test/script/builtin/echo.test
new file mode 100644
index 0000000..7f43aac
--- /dev/null
+++ b/tests/test/script/builtin/echo.test
@@ -0,0 +1,11 @@
+# file      : tests/test/script/runner/echo.test
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+: string
+:
+echo foo >foo
+
+: strings
+:
+echo foo bar >"foo bar"
diff --git a/tests/test/script/runner/mkdir.test b/tests/test/script/builtin/mkdir.test
index 6b7b5c9..6b7b5c9 100644
--- a/tests/test/script/runner/mkdir.test
+++ b/tests/test/script/builtin/mkdir.test
diff --git a/tests/test/script/runner/touch.test b/tests/test/script/builtin/touch.test
index e9d9f68..4d2ff57 100644
--- a/tests/test/script/runner/touch.test
+++ b/tests/test/script/builtin/touch.test
@@ -15,12 +15,13 @@ rm a
 
 : file-update
 :
-: Test that existing file touch doesn't register cleanup. If it did then it
-: would be left dangling after 'rm' call and so test would fail.
+: Test that existing file touch doesn't fail.
 :
-$* -f a;
-touch a;
-rm a
+cat <"" >>>a;
+touch a
+
+# @@ How we can test that touch of an existing file doesn't register a cleanup?
+#
 
 : no-args
 :
diff --git a/tests/test/script/runner/buildfile b/tests/test/script/runner/buildfile
index b9c0e69..e5f2761 100644
--- a/tests/test/script/runner/buildfile
+++ b/tests/test/script/runner/buildfile
@@ -4,9 +4,6 @@
 
 import libs = libbutl%lib{butl}
 
-exe{driver}: cxx{driver} $libs test{cleanup mkdir redirect status touch}
-
-if ($cxx.target.class == "windows") # @@ TMP
-  test{*}: ext = ".exe"
+exe{driver}: cxx{driver} $libs test{cleanup redirect status}
 
 include ../../../../../build2/
diff --git a/tests/test/script/runner/redirect.test b/tests/test/script/runner/redirect.test
index 68cc9aa..16c17d8 100644
--- a/tests/test/script/runner/redirect.test
+++ b/tests/test/script/runner/redirect.test
@@ -166,6 +166,6 @@ EOO
   echo <foo 1>-                     : in-str
   echo "foo" >foo                   : out-str
   echo "foo" 2>foo 1>&2             : err-str
-  cat <foo >foo                     : inout-str # @@ cat is not a builtin yet.
+  cat <foo >foo                     : inout-str
   cat <foo 2>foo 1>&2               : inerr-str
 }
diff --git a/tests/test/script/runner/status.test b/tests/test/script/runner/status.test
index f1ad5bf..00d7257 100644
--- a/tests/test/script/runner/status.test
+++ b/tests/test/script/runner/status.test
@@ -15,6 +15,10 @@ b = $build.driver -q --no-column --buildfile - <"./: test{testscript}" \
 c = cat >>>testscript
 test = \'$test\'
 
++if ($cxx.target.class == "windows")
+  ext = ".exe"
+end
+
 # Successfull tests.
 #
 : eq-true