Add support for no-newline redirects in testscript

The no-newline operators are '<:', '>:', '<<:', and '>>:'.

10 files changed, 212 insertions, 103 deletions

diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index 2192f35..1b038c3 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -229,14 +229,30 @@ namespace build2
             {
               xchar p (peek ());
 
-              if (p == '!' || p == '<')
+              if (p == '!' || p == ':' || p == '<')
               {
                 get ();
-                return make_token (
-                  p == '!' ? type::in_null : type::in_document);
+
+                switch (p)
+                {
+                case '!': return make_token (type::in_null);
+                case ':': return make_token (type::in_str_nn);
+                case '<':
+                  {
+                    p = peek ();
+
+                    if (p == ':')
+                    {
+                      get ();
+                      return make_token (type::in_doc_nn);
+                    }
+                    else
+                      return make_token (type::in_doc);
+                  }
+                }
               }
               else
-                return make_token (type::in_string);
+                return make_token (type::in_str);
 
             }
             // >
@@ -245,14 +261,30 @@ namespace build2
             {
               xchar p (peek ());
 
-              if (p == '!' || p == '>')
+              if (p == '!' || p == ':' || p == '>')
               {
                 get ();
-                return make_token (
-                  p == '!' ? type::out_null : type::out_document);
+
+                switch (p)
+                {
+                case '!': return make_token (type::out_null);
+                case ':': return make_token (type::out_str_nn);
+                case '>':
+                  {
+                    p = peek ();
+
+                    if (p == ':')
+                    {
+                      get ();
+                      return make_token (type::out_doc_nn);
+                    }
+                    else
+                      return make_token (type::out_doc);
+                  }
+                }
               }
               else
-                return make_token (type::out_string);
+                return make_token (type::out_str);
             }
           }
         }
diff --git a/build2/test/script/parser b/build2/test/script/parser
index 39ec4f8..ef65c1a 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -70,7 +70,7 @@ namespace build2
         parse_command_exit (token&, token_type&);
 
         string
-        parse_here_document (token&, token_type&, const string&);
+        parse_here_document (token&, token_type&, const string&, bool);
 
         // Customization hooks.
         //
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index 510b8cc..84368b4 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -381,6 +381,7 @@ namespace build2
           err_document
         };
         pending p (pending::program);
+        bool nn (false); // True if pending here-{str,doc} is "no-newline".
 
         // Ordered sequence of here-document redirects that we can expect to
         // see after the command line.
@@ -389,17 +390,25 @@ namespace build2
         {
           redirect* redir;
           string end;
+          bool no_newline;
         };
         vector<here_doc> hd;
 
         // Add the next word to either one of the pending positions or
         // to program arguments by default.
         //
-        auto add_word = [&c, &p, &hd, this] (string&& w, const location& l)
+        auto add_word =
+          [&c, &p, &nn, &hd, this] (string&& w, const location& l)
         {
-          auto add_here_end = [&hd] (redirect& r, string&& w)
+          auto add_here_str = [&hd, &nn] (redirect& r, string&& w)
           {
-            hd.push_back (here_doc {&r, move (w)});
+            if (!nn) w += '\n';
+            r.value = move (w);
+          };
+
+          auto add_here_end = [&hd, &nn] (redirect& r, string&& w)
+          {
+            hd.push_back (here_doc {&r, move (w), nn});
           };
 
           switch (p)
@@ -425,12 +434,13 @@ namespace build2
           case pending::out_document: add_here_end (c.out, move (w)); break;
           case pending::err_document: add_here_end (c.err, move (w)); break;
 
-          case pending::in_string:  c.in.value = move (w);  break;
-          case pending::out_string: c.out.value = move (w); break;
-          case pending::err_string: c.err.value = move (w); break;
+          case pending::in_string:  add_here_str (c.in,  move (w)); break;
+          case pending::out_string: add_here_str (c.out, move (w)); break;
+          case pending::err_string: add_here_str (c.err, move (w)); break;
           }
 
           p = pending::none;
+          nn = false;
         };
 
         // Make sure we don't have any pending positions to fill.
@@ -458,11 +468,11 @@ namespace build2
         // Parse the redirect operator.
         //
         auto parse_redirect =
-          [&c, &p, this] (const token& t, const location& l)
+          [&c, &p, &nn, this] (const token& t, const location& l)
         {
           // Our semantics is the last redirect seen takes effect.
           //
-          assert (p == pending::none);
+          assert (p == pending::none && !nn);
 
           // See if we have the file descriptor.
           //
@@ -497,8 +507,10 @@ namespace build2
           switch (tt)
           {
           case type::in_null:
-          case type::in_string:
-          case type::in_document:
+          case type::in_str:
+          case type::in_str_nn:
+          case type::in_doc:
+          case type::in_doc_nn:
             {
               if ((fd = fd == 3 ? 0 : fd) != 0)
                 fail (l) << "invalid in redirect file descriptor " << fd;
@@ -506,8 +518,10 @@ namespace build2
               break;
             }
           case type::out_null:
-          case type::out_string:
-          case type::out_document:
+          case type::out_str:
+          case type::out_str_nn:
+          case type::out_doc:
+          case type::out_doc_nn:
             {
               if ((fd = fd == 3 ? 1 : fd) == 0)
                 fail (l) << "invalid out redirect file descriptor " << fd;
@@ -520,11 +534,17 @@ namespace build2
           switch (tt)
           {
           case type::in_null:
-          case type::out_null:     rt = redirect_type::null;          break;
-          case type::in_string:
-          case type::out_string:   rt = redirect_type::here_string;   break;
-          case type::in_document:
-          case type::out_document: rt = redirect_type::here_document; break;
+          case type::out_null:   rt = redirect_type::null;          break;
+
+          case type::in_str_nn:
+          case type::out_str_nn: nn = true; // Fall through.
+          case type::in_str:
+          case type::out_str:    rt = redirect_type::here_string;   break;
+
+          case type::in_doc_nn:
+          case type::out_doc_nn: nn = true; // Fall through.
+          case type::in_doc:
+          case type::out_doc:    rt = redirect_type::here_document; break;
           }
 
           redirect& r (fd == 0 ? c.in : fd == 1 ? c.out : c.err);
@@ -574,12 +594,19 @@ namespace build2
               done = true;
               break;
             }
+
           case type::in_null:
-          case type::in_string:
-          case type::in_document:
           case type::out_null:
-          case type::out_string:
-          case type::out_document:
+
+          case type::in_str:
+          case type::in_doc:
+          case type::out_str:
+          case type::out_doc:
+
+          case type::in_str_nn:
+          case type::in_doc_nn:
+          case type::out_str_nn:
+          case type::out_doc_nn:
             {
               if (pre_parse_)
               {
@@ -587,10 +614,16 @@ namespace build2
                 // end markers since we need to know how many of the to pre-
                 // parse after the command.
                 //
+                nn = false;
+
                 switch (tt)
                 {
-                case type::in_document:
-                case type::out_document:
+                case type::in_doc_nn:
+                case type::out_doc_nn:
+                  nn = true;
+                  // Fall through.
+                case type::in_doc:
+                case type::out_doc:
                   // We require the end marker to be a literal, unquoted word.
                   // In particularm, we don't allow quoted because of cases
                   // like foo"$bar" (where we will see word 'foo').
@@ -600,7 +633,7 @@ namespace build2
                   if (tt != type::word || t.quoted)
                     fail (l) << "here-document end marker expected";
 
-                  hd.push_back (here_doc {nullptr, move (t.value)});
+                  hd.push_back (here_doc {nullptr, move (t.value), nn});
                   break;
                 }
 
@@ -618,11 +651,18 @@ namespace build2
               switch (tt)
               {
               case type::in_null:
-              case type::in_string:
-              case type::in_document:
               case type::out_null:
-              case type::out_string:
-              case type::out_document:
+
+              case type::in_str:
+              case type::in_doc:
+              case type::out_str:
+              case type::out_doc:
+
+              case type::in_str_nn:
+              case type::in_doc_nn:
+              case type::out_str_nn:
+              case type::out_doc_nn:
+
                 parse_redirect (t, l);
                 next (t, tt);
                 break;
@@ -750,15 +790,27 @@ namespace build2
                     switch (tt)
                     {
                     case type::in_null:
-                    case type::in_string:
                     case type::out_null:
-                    case type::out_string:
-                      parse_redirect (t, l);
-                      break;
-                    case type::in_document:
-                    case type::out_document:
-                      fail (l) << "here-document redirect in expansion";
-                      break;
+
+                    case type::in_str:
+                    case type::out_str:
+
+                    case type::in_str_nn:
+                    case type::out_str_nn:
+                      {
+                        parse_redirect (t, l);
+                        break;
+                      }
+
+                    case type::in_doc:
+                    case type::out_doc:
+
+                    case type::in_doc_nn:
+                    case type::out_doc_nn:
+                      {
+                        fail (l) << "here-document redirect in expansion";
+                        break;
+                      }
                     }
                   }
 
@@ -806,7 +858,7 @@ namespace build2
           mode (lexer_mode::here_line);
           next (t, tt);
 
-          string v (parse_here_document (t, tt, h.end));
+          string v (parse_here_document (t, tt, h.end, h.no_newline));
 
           if (!pre_parse_)
           {
@@ -857,7 +909,7 @@ namespace build2
       }
 
       string parser::
-      parse_here_document (token& t, type& tt, const string& em)
+      parse_here_document (token& t, type& tt, const string& em, bool nn)
       {
         string r;
 
@@ -880,6 +932,9 @@ namespace build2
 
           if (!pre_parse_)
           {
+            if (!r.empty ()) // Add newline after previous line.
+              r += '\n';
+
             // What shall we do if the expansion results in multiple names?
             // For, example if the line contains just the variable expansion
             // and it is of type strings. Adding all the elements space-
@@ -903,8 +958,6 @@ namespace build2
 
               r += s;
             }
-
-            r += '\n'; // Here-document line always includes a newline.
           }
 
           // We should expand the whole line at once so this would normally be
@@ -919,6 +972,11 @@ namespace build2
         if (tt == type::eos)
           fail (t) << "missing here-document end marker '" << em << "'";
 
+        // Add final newline if requested.
+        //
+        if (!pre_parse_ && !nn)
+          r += '\n';
+
         return r;
       }
 
diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx
index 510b61d..7377528 100644
--- a/build2/test/script/runner.cxx
+++ b/build2/test/script/runner.cxx
@@ -90,14 +90,7 @@ namespace build2
           {
             ofdstream os (orp);
             cln.add (orp);
-
             os << rd.value;
-
-            // Here-document is always newline-terminated.
-            //
-            if (rd.type == redirect_type::here_string)
-              os << endl;
-
             os.close ();
           }
           catch (const io_error& e)
@@ -313,12 +306,6 @@ namespace build2
             {
               ofdstream os (pr.out_fd);
               os << c.in.value;
-
-              // Here-document is always newline-terminated.
-              //
-              if (c.in.type == redirect_type::here_string)
-                os << endl;
-
               os.close ();
             }
 
diff --git a/build2/test/script/script b/build2/test/script/script
index a7bdd7b..44ec7c5 100644
--- a/build2/test/script/script
+++ b/build2/test/script/script
@@ -49,7 +49,7 @@ namespace build2
       struct redirect
       {
         redirect_type type = redirect_type::none;
-        string value;
+        string value;    // Note: includes trailing newline, if required.
         string here_end; // Only for here-documents.
       };
 
diff --git a/build2/test/script/token b/build2/test/script/token
index da468af..c1e05fd 100644
--- a/build2/test/script/token
+++ b/build2/test/script/token
@@ -32,12 +32,16 @@ namespace build2
           log_or,                       // ||
 
           in_null,                      // <!
-          in_string,                    // <
-          in_document,                  // <<
+          in_str,                       // <
+          in_str_nn,                    // <:
+          in_doc,                       // <<
+          in_doc_nn,                    // <<:
 
           out_null,                     // >!
-          out_string,                   // >
-          out_document                  // >>
+          out_str,                      // >
+          out_str_nn,                   // >:
+          out_doc,                      // >>
+          out_doc_nn                    // >>:
         };
 
         token_type () = default;
diff --git a/build2/test/script/token.cxx b/build2/test/script/token.cxx
index 6ed0443..31bfcd5 100644
--- a/build2/test/script/token.cxx
+++ b/build2/test/script/token.cxx
@@ -21,20 +21,24 @@ namespace build2
 
         switch (t.type)
         {
-        case token_type::semi:         os << q << ';' << q; break;
-
-        case token_type::pipe:         os << q << '|' << q; break;
-        case token_type::clean:        os << q << '&' << q; break;
-        case token_type::log_and:      os << q << "&&" << q; break;
-        case token_type::log_or:       os << q << "||" << q; break;
-
-        case token_type::in_null:      os << q << "<!" << q; break;
-        case token_type::in_string:    os << q << '<' << q; break;
-        case token_type::in_document:  os << q << "<<" << q; break;
-
-        case token_type::out_null:     os << q << ">!" << q; break;
-        case token_type::out_string:   os << q << '>' << q; break;
-        case token_type::out_document: os << q << ">>" << q; break;
+        case token_type::semi:         os << q << ';'   << q; break;
+
+        case token_type::pipe:         os << q << '|'   << q; break;
+        case token_type::clean:        os << q << '&'   << q; break;
+        case token_type::log_and:      os << q << "&&"  << q; break;
+        case token_type::log_or:       os << q << "||"  << q; break;
+
+        case token_type::in_null:      os << q << "<!"  << q; break;
+        case token_type::in_str:       os << q << '<'   << q; break;
+        case token_type::in_str_nn:    os << q << "<:"  << q; break;
+        case token_type::in_doc:       os << q << "<<"  << q; break;
+        case token_type::in_doc_nn:    os << q << "<<:" << q; break;
+
+        case token_type::out_null:     os << q << ">!"  << q; break;
+        case token_type::out_str:      os << q << '>'   << q; break;
+        case token_type::out_str_nn:   os << q << ">:"  << q; break;
+        case token_type::out_doc:      os << q << ">>"  << q; break;
+        case token_type::out_doc_nn:   os << q << ">>:" << q; break;
 
         default: build2::token_printer (os, t, d);
         }
diff --git a/doc/testscript.cli b/doc/testscript.cli
index b12ca98..825fde0 100644
--- a/doc/testscript.cli
+++ b/doc/testscript.cli
@@ -719,20 +719,22 @@ test-line:
 command-line:
   command
 
-command: <path>(' '+(<arg>|stdin|stdout|stderr))* command-exit?
+command: <path>(' '+(<arg>|redirect))* command-exit?
   *here-document
 
-stdin:  '0'?('<!'|\
-             '<' <text>|\
-             '<<' <here-end>)
+redirect: stdin|stdout|stderr
 
-stdout: '1'?('>!'|\
-             '>' <text>|\
-             '>>' <here-end>)
+stdin:  '0'?(in-redirect)
+stdout: '1'?(out-redirect)
+stderr: '2'(out-redirect)
 
-stderr:  '2'('>!'|\
-             '>' <text>|\
-             '>>' <here-end>)
+in-redirect:  '<!'|\
+              ('<'|'<:') <text>|\
+              ('<<'|'<<:') <here-end>
+
+out-redirect: '>!'|\
+              ('>'|'>:') <text>|\
+              ('>>'|'>>:') <here-end>
 
 command-exit: ('=='|'!=') <exit-status>
 
diff --git a/tests/test/script/testscript b/tests/test/script/testscript
index b1cf0a5..9b43467 100644
--- a/tests/test/script/testscript
+++ b/tests/test/script/testscript
@@ -40,3 +40,29 @@ EOI
 foo
 bar
 EOE
+
+# No-newline tests.
+#
+# @@ TMP Need does not compare test.
+#
+$* -i 1 <:"foo" >:"foo"            # no-newline-str
+#$* -i 1 <:"foo" >!"foo"            # no-newline-str-fail1
+#$* -i 1 <"foo"  >:!"foo"           # no-newline-str-fail2
+
+$* -i 1 <<:EOI >>:EOO              # no-newline-doc
+foo
+EOI
+foo
+EOO
+
+#$* -i 1 <<:EOI >>!EOO              # no-newline-doc-fail1
+#foo
+#EOI
+#foo
+#EOO
+
+#$* -i 1 <<EOI >>:!EOO              # no-newline-doc-fail2
+#foo
+#EOI
+#foo
+#EOO
diff --git a/unit-tests/test/script/parser/scope.test b/unit-tests/test/script/parser/scope.test
index 7517022..7749b3d 100644
--- a/unit-tests/test/script/parser/scope.test
+++ b/unit-tests/test/script/parser/scope.test
@@ -89,16 +89,12 @@ EOI
 testscript:2:1: error: expected another line after semicolon
 EOE
 
-# @@ Need newline-less support.
-#
-#$* <<EOI 2>>EOE != 0                 # expected-newline-cmd
-#cmd ;\
-#EOI
-#testscript:2:1: error: expected newline instead of <end of file>
-#EOE
+$* <<:EOI 2>>EOE != 0                # expected-newline-cmd
+cmd;
+EOI
+testscript:1:5: error: expected newline instead of <end of file>
+EOE
 
-#$* <<EOI 2>>EOE != 0                 # expected-newline-var
-#x =abc;\
-#EOI
-#testscript:2:1: error: expected newline instead of <end of file>
-#EOE
+$* <:"x = abc;" 2>>EOE != 0          # expected-newline-var
+testscript:1:9: error: expected newline instead of <end of file>
+EOE