Fix testscript here-document quoting issue

4 files changed, 35 insertions, 3 deletions

diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index f75ad4a..d2657c6 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -20,6 +20,7 @@ namespace build2
         const char* s1 (nullptr);
         const char* s2 (nullptr);
         bool s (true);
+        bool q (true);
 
         switch (m)
         {
@@ -58,11 +59,12 @@ namespace build2
         case lexer_mode::here_line:
           {
             // This one is like a double-quoted string except it treats
-            // newlines as a separator.
+            // newlines as a separator. We also treat quotes as literals.
             //
             s1 = "$(\n";
             s2 = "   ";
             s = false;
+            q = false;
             break;
           }
         default:
@@ -74,7 +76,7 @@ namespace build2
           }
         }
 
-        state_.push (state {m, '\0', s, s1, s2});
+        state_.push (state {m, '\0', s, q, s1, s2});
       }
 
       token lexer::
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index a9a179b..f2f9b81 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -743,8 +743,9 @@ namespace build2
                 r += ' ';
 
               r += s;
-              r += '\n'; // Here-document line always includes a newline.
             }
+
+            r += '\n'; // Here-document line always includes a newline.
           }
 
           // We should expand the whole line at once so this would normally be
diff --git a/unit-tests/test/script/parser/command-re-parse.test b/unit-tests/test/script/parser/command-re-parse.test
new file mode 100644
index 0000000..956052d
--- /dev/null
+++ b/unit-tests/test/script/parser/command-re-parse.test
@@ -0,0 +1,9 @@
+
+# double-quote
+#
+$* <<EOI >>EOO
+x = cmd \\">!\\" "'<!'"
+$x
+EOI
+cmd ">!" "<!"
+EOO
diff --git a/unit-tests/test/script/parser/here-document.test b/unit-tests/test/script/parser/here-document.test
new file mode 100644
index 0000000..7cfa8d4
--- /dev/null
+++ b/unit-tests/test/script/parser/here-document.test
@@ -0,0 +1,20 @@
+
+# quote
+#
+# Note: they are still recognized in eval contexts.
+#
+$* <<EOI >>EOO
+cmd <<EOF
+'single'
+"double"
+b'o't"h"
+('single' "double")
+EOF
+EOI
+cmd <<EOF
+'single'
+"double"
+b'o't"h"
+'single' "double"
+EOF
+EOO