Add support for 'for' loop second (... | for x) and third (for x <...) forms in script

11 files changed, 1128 insertions, 99 deletions

diff --git a/libbuild2/test/script/lexer+for-loop.test.testscript b/libbuild2/test/script/lexer+for-loop.test.testscript
new file mode 100644
index 0000000..fcd12f7
--- /dev/null
+++ b/libbuild2/test/script/lexer+for-loop.test.testscript
@@ -0,0 +1,231 @@
+# file      : libbuild2/test/script/lexer+for-loop.test.testscript
+# license   : MIT; see accompanying LICENSE file
+
+test.arguments = for-loop
+
+: semi
+{
+  : immediate
+  :
+  $* <"cmd;" >>EOO
+  'cmd'
+  ;
+  <newline>
+  EOO
+
+  : separated
+  :
+  $* <"cmd ;" >>EOO
+  'cmd'
+  ;
+  <newline>
+  EOO
+
+  : only
+  :
+  $* <";" >>EOO
+  ;
+  <newline>
+  EOO
+}
+
+: colon
+:
+{
+  : immediate
+  :
+  $* <"cmd: dsc" >>EOO
+  'cmd'
+  :
+  'dsc'
+  <newline>
+  EOO
+
+  : separated
+  :
+  $* <"cmd :dsc" >>EOO
+  'cmd'
+  :
+  'dsc'
+  <newline>
+  EOO
+
+  : only
+  :
+  $* <":" >>EOO
+  :
+  <newline>
+  EOO
+}
+
+: redirect
+:
+{
+  : pass
+  :
+  $* <"cmd <| 1>|" >>EOO
+  'cmd'
+  <|
+  '1'
+  >|
+  <newline>
+  EOO
+
+  : null
+  :
+  $* <"cmd <- 1>-" >>EOO
+  'cmd'
+  <-
+  '1'
+  >-
+  <newline>
+  EOO
+
+  : trace
+  :
+  $* <"cmd 1>!" >>EOO
+  'cmd'
+  '1'
+  >!
+  <newline>
+  EOO
+
+  : merge
+  :
+  $* <"cmd 1>&2" >>EOO
+  'cmd'
+  '1'
+  >&
+  '2'
+  <newline>
+  EOO
+
+  : str
+  :
+  $* <"cmd <a 1>b" >>EOO
+  'cmd'
+  <
+  'a'
+  '1'
+  >
+  'b'
+  <newline>
+  EOO
+
+  : str-nn
+  :
+  $* <"cmd <:a 1>:b" >>EOO
+  'cmd'
+  <:
+  'a'
+  '1'
+  >:
+  'b'
+  <newline>
+  EOO
+
+  : doc
+  :
+  $* <"cmd <<EOI 1>>EOO" >>EOO
+  'cmd'
+  <<
+  'EOI'
+  '1'
+  >>
+  'EOO'
+  <newline>
+  EOO
+
+  : doc-nn
+  :
+  $* <"cmd <<:EOI 1>>:EOO" >>EOO
+  'cmd'
+  <<:
+  'EOI'
+  '1'
+  >>:
+  'EOO'
+  <newline>
+  EOO
+
+  : file-cmp
+  :
+  $* <"cmd <<<in >>>out 2>>>err" >>EOO
+  'cmd'
+  <<<
+  'in'
+  >>>
+  'out'
+  '2'
+  >>>
+  'err'
+  <newline>
+  EOO
+
+  : file-write
+  :
+  $* <"cmd >=out 2>+err" >>EOO
+  'cmd'
+  >=
+  'out'
+  '2'
+  >+
+  'err'
+  <newline>
+  EOO
+}
+
+: cleanup
+:
+{
+  : always
+  :
+  $* <"cmd &file" >>EOO
+  'cmd'
+  &
+  'file'
+  <newline>
+  EOO
+
+  : maybe
+  :
+  $* <"cmd &?file" >>EOO
+  'cmd'
+  &?
+  'file'
+  <newline>
+  EOO
+
+  : never
+  :
+  $* <"cmd &!file" >>EOO
+  'cmd'
+  &!
+  'file'
+  <newline>
+  EOO
+}
+
+: for
+:
+{
+  : form-1
+  :
+  $* <"for x: a" >>EOO
+  'for'
+  'x'
+  :
+  'a'
+  <newline>
+  EOO
+
+  : form-3
+  :
+  $* <"for <<<a x" >>EOO
+  'for'
+  <<<
+  'a'
+  'x'
+  <newline>
+  EOO
+}
diff --git a/libbuild2/test/script/lexer.cxx b/libbuild2/test/script/lexer.cxx
index f9c8ac6..9475ad4 100644
--- a/libbuild2/test/script/lexer.cxx
+++ b/libbuild2/test/script/lexer.cxx
@@ -41,6 +41,12 @@ namespace build2
 
         switch (m)
         {
+        case lexer_mode::for_loop:
+          {
+            // Leading tokens of the for-loop. Like command_line but also
+            // recognizes lsbrace like value.
+          }
+          // Fall through.
         case lexer_mode::command_line:
           {
             s1 = ":;=!|&<> $(#\t\n";
@@ -122,6 +128,7 @@ namespace build2
         case lexer_mode::first_token:
         case lexer_mode::second_token:
         case lexer_mode::variable_line:
+        case lexer_mode::for_loop:
           r = next_line ();
           break;
         case lexer_mode::description_line:
@@ -157,7 +164,8 @@ namespace build2
         //
         if (st.lsbrace)
         {
-          assert (m == lexer_mode::variable_line);
+          assert (m == lexer_mode::variable_line ||
+                  m == lexer_mode::for_loop);
 
           state_.top ().lsbrace = false; // Note: st is a copy.
 
@@ -197,10 +205,11 @@ namespace build2
 
         // Line separators.
         //
-        if (m == lexer_mode::command_line ||
-            m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token ||
-            m == lexer_mode::variable_line)
+        if (m == lexer_mode::command_line  ||
+            m == lexer_mode::first_token   ||
+            m == lexer_mode::second_token  ||
+            m == lexer_mode::variable_line ||
+            m == lexer_mode::for_loop)
         {
           switch (c)
           {
@@ -210,7 +219,8 @@ namespace build2
 
         if (m == lexer_mode::command_line ||
             m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token)
+            m == lexer_mode::second_token ||
+            m == lexer_mode::for_loop)
         {
           switch (c)
           {
@@ -222,7 +232,8 @@ namespace build2
         //
         if (m == lexer_mode::command_line ||
             m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token)
+            m == lexer_mode::second_token ||
+            m == lexer_mode::for_loop)
         {
           switch (c)
           {
@@ -244,7 +255,8 @@ namespace build2
         //
         if (m == lexer_mode::command_line ||
             m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token)
+            m == lexer_mode::second_token ||
+            m == lexer_mode::for_loop)
         {
           if (optional<token> t = next_cmd_op (c, sep))
             return move (*t);
diff --git a/libbuild2/test/script/lexer.hxx b/libbuild2/test/script/lexer.hxx
index 452e794..def269b 100644
--- a/libbuild2/test/script/lexer.hxx
+++ b/libbuild2/test/script/lexer.hxx
@@ -24,10 +24,11 @@ namespace build2
         enum
         {
           command_line = base_type::value_next,
-          first_token,     // Expires at the end of the token.
-          second_token,    // Expires at the end of the token.
-          variable_line,   // Expires at the end of the line.
-          description_line // Expires at the end of the line.
+          first_token,      // Expires at the end of the token.
+          second_token,     // Expires at the end of the token.
+          variable_line,    // Expires at the end of the line.
+          description_line, // Expires at the end of the line.
+          for_loop          // Used for sensing the for-loop leading tokens.
         };
 
         lexer_mode () = default;
diff --git a/libbuild2/test/script/lexer.test.cxx b/libbuild2/test/script/lexer.test.cxx
index 76f102d..ef3ce4d 100644
--- a/libbuild2/test/script/lexer.test.cxx
+++ b/libbuild2/test/script/lexer.test.cxx
@@ -36,6 +36,7 @@ namespace build2
           else if (s == "variable-line")     m = lexer_mode::variable_line;
           else if (s == "description-line")  m = lexer_mode::description_line;
           else if (s == "variable")          m = lexer_mode::variable;
+          else if (s == "for-loop")          m = lexer_mode::for_loop;
           else                               assert (false);
         }
 
diff --git a/libbuild2/test/script/parser+for.test.testscript b/libbuild2/test/script/parser+for.test.testscript
index 70c1c89..426a39b 100644
--- a/libbuild2/test/script/parser+for.test.testscript
+++ b/libbuild2/test/script/parser+for.test.testscript
@@ -16,7 +16,7 @@
       cmd
     end
     EOI
-    testscript:1:4: error: expected variable name instead of <newline>
+    testscript:1:1: error: for: missing variable name
     EOE
 
     : untyped
@@ -311,3 +311,703 @@
   testscript:4:1: error: both leading and trailing descriptions
   EOE
 }
+
+: form-2
+:
+: ... | for x
+:
+{
+  : for
+  :
+  {
+    : status
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x != 0
+      cmd
+    end
+    EOI
+    testscript:1:20: error: for-loop exit code cannot be checked
+    EOE
+
+    : not-last
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x | echo x
+      cmd
+    end
+    EOI
+    testscript:1:20: error: for-loop must be last command in a pipe
+    EOE
+
+    : not-last-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x|echo x
+      cmd
+    end
+    EOI
+    testscript:1:19: error: for-loop must be last command in a pipe
+    EOE
+
+    : expression-after
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x && echo x
+      cmd
+    end
+    EOI
+    testscript:1:20: error: command expression involving for-loop
+    EOE
+
+    : expression-after-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x&&echo x
+      cmd
+    end
+    EOI
+    testscript:1:19: error: command expression involving for-loop
+    EOE
+
+    : expression-before
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && echo x | for x
+      cmd
+    end
+    EOI
+    testscript:1:24: error: command expression involving for-loop
+    EOE
+
+    : expression-before-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && echo x|for x
+      cmd
+    end
+    EOI
+    testscript:1:22: error: command expression involving for-loop
+    EOE
+
+    : cleanup
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x &f
+      cmd
+    end
+    EOI
+    testscript:1:20: error: cleanup in for-loop
+    EOE
+
+    : cleanup-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x&f
+      cmd
+    end
+    EOI
+    testscript:1:19: error: cleanup in for-loop
+    EOE
+
+    : stdout-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x >a
+      cmd
+    end
+    EOI
+    testscript:1:20: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x>a
+      cmd
+    end
+    EOI
+    testscript:1:19: error: output redirect in for-loop
+    EOE
+
+    : stdin-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x <a
+      cmd
+    end
+    EOI
+    testscript:1:20: error: stdin is both piped and redirected
+    EOE
+
+    : no-var
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for
+      cmd
+    end
+    EOI
+    testscript:1:1: error: for: missing variable name
+    EOE
+
+    : untyped
+    :
+    $* <<EOI >>EOO
+    echo 'a b' | for -w x
+      cmd $x
+    end
+    EOI
+    echo 'a b' | for -w x
+    EOO
+
+    : expansion
+    :
+    $* <<EOI >>EOO
+    vs = a b
+    echo $vs | for x
+      cmd $x
+    end
+    EOI
+    echo a b | for x
+    EOO
+
+    : typed-var
+    :
+    $* <<EOI >>EOO
+    echo 'a b' | for -w [dir_path] x
+      cmd $x
+    end
+    EOI
+    echo 'a b' | for -w [dir_path] x
+    EOO
+  }
+
+  : after-semi
+  :
+  $* -s <<EOI >>EOO
+  cmd1;
+  echo 'a b' | for x
+    cmd2 $x
+  end
+  EOI
+  {
+    {
+      cmd1
+      echo 'a b' | for x
+    }
+  }
+  EOO
+
+  : setup
+  :
+  $* -s <<EOI >>EOO
+  +echo 'a b' | for x
+    cmd $x
+  end
+  EOI
+  {
+    +echo 'a b' | for x
+  }
+  EOO
+
+  : tdown
+  :
+  $* -s <<EOI >>EOO
+  -echo 'a b' | for x
+    cmd $x
+  end
+  EOI
+  {
+    -echo 'a b' | for x
+  }
+  EOO
+
+  : end
+  :
+  {
+    : without-end
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      cmd
+    EOI
+    testscript:3:1: error: expected closing 'end'
+    EOE
+  }
+
+  : elif
+  :
+  {
+    : without-if
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      elif true
+        cmd
+      end
+    end
+    EOI
+    testscript:2:3: error: 'elif' without preceding 'if'
+    EOE
+  }
+
+  : nested
+  :
+  {
+    $* -l -r <<EOI >>EOO
+    echo 'a b' | for x      # 1
+      cmd1 $x               # 2
+      if ($x == "a")        # 3
+        cmd2                # 4
+        echo x y | for y    # 5
+          cmd3              # 6
+        end
+      else
+        cmd4                # 7
+      end
+      cmd5                  # 8
+    end;
+    cmd6                    # 9
+    EOI
+    echo 'a b' | for x # 1
+    cmd6 # 9
+    EOO
+  }
+
+  : contained
+  :
+  {
+    : semi
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      cmd;
+      cmd
+    end
+    EOI
+    testscript:2:3: error: ';' inside 'for'
+    EOE
+
+    : colon-leading
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      : foo
+      cmd
+    end
+    EOI
+    testscript:2:3: error: description inside 'for'
+    EOE
+
+    : colon-trailing
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      cmd : foo
+    end
+    EOI
+    testscript:2:3: error: description inside 'for'
+    EOE
+
+    : eos
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+    EOI
+    testscript:2:1: error: expected closing 'end'
+    EOE
+
+    : scope
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      cmd
+      {
+      }
+    end
+    EOI
+    testscript:3:3: error: expected closing 'end'
+    EOE
+
+    : setup
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      +cmd
+    end
+    EOI
+    testscript:2:3: error: setup command inside 'for'
+    EOE
+
+    : tdown
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      -cmd
+    end
+    EOI
+    testscript:2:3: error: teardown command inside 'for'
+    EOE
+  }
+
+  : leading-and-trailing-description
+  :
+  $* <<EOI 2>>EOE != 0
+  : foo
+  echo 'a b' | for x
+    cmd
+  end : bar
+  EOI
+  testscript:4:1: error: both leading and trailing descriptions
+  EOE
+}
+
+: form-3
+:
+: for x <...
+:
+{
+  : for
+  :
+  {
+    : status
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a != 0
+      cmd
+    end
+    EOI
+    testscript:1:10: error: for-loop exit code cannot be checked
+    EOE
+
+    : not-last
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a | echo x
+      cmd
+    end
+    EOI
+    testscript:1:10: error: for-loop must be last command in a pipe
+    EOE
+
+    : not-last-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x|echo x
+      cmd
+    end
+    EOI
+    testscript:1:9: error: for-loop must be last command in a pipe
+    EOE
+
+    : expression-after
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a && echo x
+      cmd
+    end
+    EOI
+    testscript:1:10: error: command expression involving for-loop
+    EOE
+
+    : expression-after-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x&&echo x
+      cmd
+    end
+    EOI
+    testscript:1:9: error: command expression involving for-loop
+    EOE
+
+    : expression-before
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && for x <a
+      cmd
+    end
+    EOI
+    testscript:1:15: error: command expression involving for-loop
+    EOE
+
+    : cleanup
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a &f
+      cmd
+    end
+    EOI
+    testscript:1:10: error: cleanup in for-loop
+    EOE
+
+    : cleanup-before-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for &f x <a
+      cmd
+    end
+    EOI
+    testscript:1:5: error: cleanup in for-loop
+    EOE
+
+    : cleanup-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x&f
+      cmd
+    end
+    EOI
+    testscript:1:9: error: cleanup in for-loop
+    EOE
+
+    : stdout-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    for x >a
+      cmd
+    end
+    EOI
+    testscript:1:7: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-before-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for >a x
+      cmd
+    end
+    EOI
+    testscript:1:5: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for x>a
+      cmd
+    end
+    EOI
+    testscript:1:6: error: output redirect in for-loop
+    EOE
+
+    : no-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a
+      cmd
+    end
+    EOI
+    testscript:1:1: error: for: missing variable name
+    EOE
+
+    : untyped
+    :
+    $* <<EOI >>EOO
+    for -w x <'a b'
+      cmd $x
+    end
+    EOI
+    for -w x <'a b'
+    EOO
+
+    : expansion
+    :
+    $* <<EOI >>EOO
+    vs = a b
+    for x <$vs
+      cmd $x
+    end
+    EOI
+    for x b <a
+    EOO
+
+    : typed-var
+    :
+    $* <<EOI >>EOO
+    for -w [dir_path] x <'a b'
+      cmd $x
+    end
+    EOI
+    for -w [dir_path] x <'a b'
+    EOO
+  }
+
+  : after-semi
+  :
+  $* -s <<EOI >>EOO
+  cmd1;
+  for x <'a b'
+    cmd2 $x
+  end
+  EOI
+  {
+    {
+      cmd1
+      for x <'a b'
+    }
+  }
+  EOO
+
+  : setup
+  :
+  $* -s <<EOI >>EOO
+  +for x <'a b'
+    cmd $x
+  end
+  EOI
+  {
+    +for x <'a b'
+  }
+  EOO
+
+  : tdown
+  :
+  $* -s <<EOI >>EOO
+  -for x <'a b'
+    cmd $x
+  end
+  EOI
+  {
+    -for x <'a b'
+  }
+  EOO
+
+  : end
+  :
+  {
+    : without-end
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      cmd
+    EOI
+    testscript:3:1: error: expected closing 'end'
+    EOE
+  }
+
+  : elif
+  :
+  {
+    : without-if
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      elif true
+        cmd
+      end
+    end
+    EOI
+    testscript:2:3: error: 'elif' without preceding 'if'
+    EOE
+  }
+
+  : nested
+  :
+  {
+    $* -l -r <<EOI >>EOO
+    for -w x <'a b'         # 1
+      cmd1 $x               # 2
+      if ($x == "a")        # 3
+        cmd2                # 4
+        for -w y <'x y'     # 5
+          cmd3              # 6
+        end
+      else
+        cmd4                # 7
+      end
+      cmd5                  # 8
+    end;
+    cmd6                    # 9
+    EOI
+    for -w x <'a b' # 1
+    cmd6 # 9
+    EOO
+  }
+
+  : contained
+  :
+  {
+    : semi
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      cmd;
+      cmd
+    end
+    EOI
+    testscript:2:3: error: ';' inside 'for'
+    EOE
+
+    : colon-leading
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      : foo
+      cmd
+    end
+    EOI
+    testscript:2:3: error: description inside 'for'
+    EOE
+
+    : colon-trailing
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      cmd : foo
+    end
+    EOI
+    testscript:2:3: error: description inside 'for'
+    EOE
+
+    : eos
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+    EOI
+    testscript:2:1: error: expected closing 'end'
+    EOE
+
+    : scope
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      cmd
+      {
+      }
+    end
+    EOI
+    testscript:3:3: error: expected closing 'end'
+    EOE
+
+    : setup
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      +cmd
+    end
+    EOI
+    testscript:2:3: error: setup command inside 'for'
+    EOE
+
+    : tdown
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      -cmd
+    end
+    EOI
+    testscript:2:3: error: teardown command inside 'for'
+    EOE
+  }
+
+  : leading-and-trailing-description
+  :
+  $* <<EOI 2>>EOE != 0
+  : foo
+  for x <'a b'
+    cmd
+  end : bar
+  EOI
+  testscript:4:1: error: both leading and trailing descriptions
+  EOE
+}
diff --git a/libbuild2/test/script/parser.cxx b/libbuild2/test/script/parser.cxx
index f302aee..f85b185 100644
--- a/libbuild2/test/script/parser.cxx
+++ b/libbuild2/test/script/parser.cxx
@@ -311,10 +311,11 @@ namespace build2
         // enter: next token is peeked at (type in tt)
         // leave: newline
 
-        assert (!fct                         ||
-                *fct == line_type::cmd_if    ||
-                *fct == line_type::cmd_while ||
-                *fct == line_type::cmd_for);
+        assert (!fct                              ||
+                *fct == line_type::cmd_if         ||
+                *fct == line_type::cmd_while      ||
+                *fct == line_type::cmd_for_stream ||
+                *fct == line_type::cmd_for_args);
 
         // Note: token is only peeked at.
         //
@@ -324,6 +325,52 @@ namespace build2
         //
         line_type lt;
         type st (type::eos); // Later, can only be set to plus or minus.
+        bool semi (false);
+
+        // Parse the command line tail, starting from the newline or the
+        // potential colon/semicolon token.
+        //
+        // Note that colon and semicolon are only valid in test command lines
+        // and after 'end' in flow control constructs. Note that we always
+        // recognize them lexically, even when they are not valid tokens per
+        // the grammar.
+        //
+        auto parse_command_tail = [&t, &tt, &st, &lt, &d, &semi, &ll, this] ()
+        {
+          if (tt != type::newline)
+          {
+            if (lt != line_type::cmd && lt != line_type::cmd_end)
+              fail (t) << "expected newline instead of " << t;
+
+            switch (st)
+            {
+            case type::plus:  fail (t) << t << " after setup command" << endf;
+            case type::minus: fail (t) << t << " after teardown command" << endf;
+            }
+          }
+
+          switch (tt)
+          {
+          case type::colon:
+            {
+              if (d)
+                fail (ll) << "both leading and trailing descriptions";
+
+              d = parse_trailing_description (t, tt);
+              break;
+            }
+          case type::semi:
+            {
+              semi = true;
+              replay_pop (); // See above for the reasoning.
+              next (t, tt);  // Get newline.
+              break;
+            }
+          }
+
+          if (tt != type::newline)
+            fail (t) << "expected newline instead of " << t;
+        };
 
         switch (tt)
         {
@@ -371,10 +418,12 @@ namespace build2
             {
               const string& n (t.value);
 
+              // Handle the for-loop consistently with pre_parse_line_start().
+              //
               if      (n == "if")    lt = line_type::cmd_if;
               else if (n == "if!")   lt = line_type::cmd_ifn;
               else if (n == "while") lt = line_type::cmd_while;
-              else if (n == "for")   lt = line_type::cmd_for;
+              else if (n == "for")   lt = line_type::cmd_for_stream;
             }
 
             break;
@@ -388,8 +437,6 @@ namespace build2
 
         // Pre-parse the line keeping track of whether it ends with a semi.
         //
-        bool semi (false);
-
         line ln;
         switch (lt)
         {
@@ -436,47 +483,80 @@ namespace build2
 
             break;
           }
-        case line_type::cmd_for:
+          //
+          // See pre_parse_line_start() for details.
+          //
+        case line_type::cmd_for_args: assert (false); break;
+        case line_type::cmd_for_stream:
           {
-            // First take care of the variable name. There is no reason not to
-            // support variable attributes.
+            // First we need to sense the next few tokens and detect which
+            // form of the for-loop that actually is (see
+            // libbuild2/build/script/parser.cxx for details).
             //
-            mode (lexer_mode::normal);
+            token pt (t);
+            assert (pt.type == type::word && pt.value == "for");
 
+            mode (lexer_mode::for_loop);
             next_with_attributes (t, tt);
-            attributes_push (t, tt);
-
-            if (tt != type::word || t.qtype != quote_type::unquoted)
-              fail (t) << "expected variable name instead of " << t;
 
             string& n (t.value);
 
-            if (special_variable (n))
-              fail (t) << "attempt to set '" << n << "' variable directly";
+            if (tt == type::lsbrace ||                       // Attributes.
+                (tt == type::word                &&          // Variable name.
+                 t.qtype == quote_type::unquoted &&
+                 (n[0] == '_'  ||
+                  alpha (n[0]) ||
+                  n == "*"     ||
+                  n == "~"     ||
+                  n == "@")))
+            {
+              attributes_push (t, tt);
 
-            ln.var = &script_->var_pool.insert (move (n));
+              if (tt != type::word || t.qtype != quote_type::unquoted)
+                fail (t) << "expected variable name instead of " << t;
 
-            next (t, tt);
+              if (special_variable (n))
+                fail (t) << "attempt to set '" << n << "' variable directly";
 
-            if (tt != type::colon)
+              if (lexer_->peek_char ().first == ':')
+                lt = line_type::cmd_for_args;
+            }
+
+            if (lt == line_type::cmd_for_stream) // for x <...
             {
-              // @@ TMP We will need to fallback to parsing the 'for x <...'
-              //        form instead.
-              //
-              fail (t) << "expected ':' instead of " << t
-                       << " after variable name";
+              ln.var = nullptr;
+
+              expire_mode ();
+
+              parse_command_expr_result r (
+                parse_command_expr (t, tt,
+                                    lexer::redirect_aliases,
+                                    move (pt)));
+
+              assert (r.for_loop);
+
+              parse_command_tail ();
+              parse_here_documents (t, tt, r);
             }
+            else                                 // for x: ...
+            {
+              ln.var = &script_->var_pool.insert (move (n));
 
-            expire_mode (); // Expire the normal lexer mode.
+              next (t, tt);
 
-            // Parse the value similar to the var line type (see above),
-            // except for the fact that we don't expect a trailing semicolon.
-            //
-            mode (lexer_mode::variable_line);
-            parse_variable_line (t, tt);
+              assert (tt == type::colon);
 
-            if (tt != type::newline)
-              fail (t) << "expected newline instead of " << t << " after for";
+              expire_mode ();
+
+              // Parse the value similar to the var line type (see above),
+              // except for the fact that we don't expect a trailing semicolon.
+              //
+              mode (lexer_mode::variable_line);
+              parse_variable_line (t, tt);
+
+              if (tt != type::newline)
+                fail (t) << "expected newline instead of " << t << " after for";
+            }
 
             break;
           }
@@ -501,51 +581,20 @@ namespace build2
           // Fall through.
         case line_type::cmd:
           {
-            pair<command_expr, here_docs> p;
+            parse_command_expr_result r;
 
             if (lt != line_type::cmd_else && lt != line_type::cmd_end)
-              p = parse_command_expr (t, tt, lexer::redirect_aliases);
+              r = parse_command_expr (t, tt, lexer::redirect_aliases);
 
-            // Colon and semicolon are only valid in test command lines and
-            // after 'end' in a flow control construct. Note that we still
-            // recognize them lexically, they are just not valid tokens per
-            // the grammar.
-            //
-            if (tt != type::newline)
+            if (r.for_loop)
             {
-              if (lt != line_type::cmd && lt != line_type::cmd_end)
-                fail (t) << "expected newline instead of " << t;
-
-              switch (st)
-              {
-              case type::plus:  fail (t) << t << " after setup command" << endf;
-              case type::minus: fail (t) << t << " after teardown command" << endf;
-              }
-            }
-
-            switch (tt)
-            {
-            case type::colon:
-              {
-                if (d)
-                  fail (ll) << "both leading and trailing descriptions";
-
-                d = parse_trailing_description (t, tt);
-                break;
-              }
-            case type::semi:
-              {
-                semi = true;
-                replay_pop (); // See above for the reasoning.
-                next (t, tt);  // Get newline.
-                break;
-              }
+              lt     = line_type::cmd_for_stream;
+              ln.var = nullptr;
             }
 
-            if (tt != type::newline)
-              fail (t) << "expected newline instead of " << t;
+            parse_command_tail ();
+            parse_here_documents (t, tt, r);
 
-            parse_here_documents (t, tt, p);
             break;
           }
         }
@@ -579,7 +628,8 @@ namespace build2
             break;
           }
         case line_type::cmd_while:
-        case line_type::cmd_for:
+        case line_type::cmd_for_stream:
+        case line_type::cmd_for_args:
           {
             semi = pre_parse_loop (t, tt, lt, d, *ls);
             break;
@@ -608,7 +658,8 @@ namespace build2
           case line_type::cmd_if:
           case line_type::cmd_ifn:
           case line_type::cmd_while:
-          case line_type::cmd_for:
+          case line_type::cmd_for_stream:
+          case line_type::cmd_for_args:
             {
               // See if this is a variable-only flow control construct.
               //
@@ -951,7 +1002,7 @@ namespace build2
         //
         size_t i (ls.size ());
 
-        line_type fct; // Flow control type the block type relates to.
+        line_type fct; // Flow control construct type the block type relates to.
 
         switch (bt)
         {
@@ -965,7 +1016,8 @@ namespace build2
             break;
           }
         case line_type::cmd_while:
-        case line_type::cmd_for:
+        case line_type::cmd_for_stream:
+        case line_type::cmd_for_args:
           {
             fct = bt;
             break;
@@ -1068,7 +1120,9 @@ namespace build2
         // enter: <newline> (previous line)
         // leave: <newline>
 
-        assert (lt == line_type::cmd_while || lt == line_type::cmd_for);
+        assert (lt == line_type::cmd_while      ||
+                lt == line_type::cmd_for_stream ||
+                lt == line_type::cmd_for_args);
 
         tt = peek (lexer_mode::first_token);
 
@@ -1428,7 +1482,7 @@ namespace build2
 
         // Note: this one is only used during execution.
 
-        pair<command_expr, here_docs> p (
+        parse_command_expr_result pr (
           parse_command_expr (t, tt, lexer::redirect_aliases));
 
         if (tt == type::colon)
@@ -1436,10 +1490,10 @@ namespace build2
 
         assert (tt == type::newline);
 
-        parse_here_documents (t, tt, p);
+        parse_here_documents (t, tt, pr);
         assert (tt == type::newline);
 
-        command_expr r (move (p.first));
+        command_expr r (move (pr.expr));
 
         // If the test program runner is specified, then adjust the
         // expressions to run test programs via this runner.
@@ -1582,6 +1636,7 @@ namespace build2
         auto exec_cmd = [&ct, this] (token& t, build2::script::token_type& tt,
                                      const iteration_index* ii, size_t li,
                                      bool single,
+                                     const function<command_function>& cf,
                                      const location& ll)
         {
           // We use the 0 index to signal that this is the only command.
@@ -1593,7 +1648,7 @@ namespace build2
           command_expr ce (
             parse_command_line (t, static_cast<token_type&> (tt)));
 
-          runner_->run (*scope_, ce, ct, ii, li, ll);
+          runner_->run (*scope_, ce, ct, ii, li, cf, ll);
         };
 
         auto exec_cond = [this] (token& t, build2::script::token_type& tt,
@@ -1827,7 +1882,8 @@ namespace build2
       // The rest.
       //
 
-      // When add a special variable don't forget to update lexer::word().
+      // When add a special variable don't forget to update lexer::word() and
+      // for-loop parsing in pre_parse_line().
       //
       bool parser::
       special_variable (const string& n) noexcept
diff --git a/libbuild2/test/script/parser.test.cxx b/libbuild2/test/script/parser.test.cxx
index ab0aee9..7339346 100644
--- a/libbuild2/test/script/parser.test.cxx
+++ b/libbuild2/test/script/parser.test.cxx
@@ -100,11 +100,32 @@ namespace build2
         }
 
         virtual void
-        run (scope&,
+        run (scope& env,
              const command_expr& e, command_type t,
              const iteration_index* ii, size_t i,
-             const location&) override
+             const function<command_function>& cf,
+             const location& ll) override
         {
+          // If the functions is specified, then just execute it with an empty
+          // stdin so it can perform the housekeeping (stop replaying tokens,
+          // increment line index, etc).
+          //
+          if (cf != nullptr)
+          {
+            assert (e.size () == 1 && !e[0].pipe.empty ());
+
+            const command& c (e[0].pipe.back ());
+
+            // Must be enforced by the caller.
+            //
+            assert (!c.out && !c.err && !c.exit);
+
+            cf (env, c.arguments,
+                fdopen_null (), false /* pipe */,
+                nullopt /* deadline */, c,
+                ll);
+          }
+
           const char* s (nullptr);
 
           switch (t)
diff --git a/libbuild2/test/script/runner.cxx b/libbuild2/test/script/runner.cxx
index 42eef04..340ada4 100644
--- a/libbuild2/test/script/runner.cxx
+++ b/libbuild2/test/script/runner.cxx
@@ -143,6 +143,7 @@ namespace build2
       run (scope& sp,
            const command_expr& expr, command_type ct,
            const iteration_index* ii, size_t li,
+           const function<command_function>& cf,
            const location& ll)
       {
         // Noop for teardown commands if keeping tests output is requested.
@@ -176,7 +177,7 @@ namespace build2
             dr << info << "test id: " << sp.id_path.posix_string ();
           });
 
-        build2::script::run (sp, expr, ii, li, ll);
+        build2::script::run (sp, expr, ii, li, ll, cf);
       }
 
       bool default_runner::
diff --git a/libbuild2/test/script/runner.hxx b/libbuild2/test/script/runner.hxx
index 0309a35..687d991 100644
--- a/libbuild2/test/script/runner.hxx
+++ b/libbuild2/test/script/runner.hxx
@@ -48,10 +48,14 @@ namespace build2
         // Location is the start position of this command line in the
         // testscript. It can be used in diagnostics.
         //
+        // Optionally, execute the specified function instead of the last
+        // pipe command.
+        //
         virtual void
         run (scope&,
              const command_expr&, command_type,
              const iteration_index*, size_t index,
+             const function<command_function>&,
              const location&) = 0;
 
         virtual bool
@@ -88,6 +92,7 @@ namespace build2
         run (scope&,
              const command_expr&, command_type,
              const iteration_index*, size_t,
+             const function<command_function>&,
              const location&) override;
 
         virtual bool
diff --git a/libbuild2/test/script/script.cxx b/libbuild2/test/script/script.cxx
index e10afec..bbe5326 100644
--- a/libbuild2/test/script/script.cxx
+++ b/libbuild2/test/script/script.cxx
@@ -115,7 +115,7 @@ namespace build2
       }
 
       void scope::
-      set_variable (string&& nm,
+      set_variable (string nm,
                     names&& val,
                     const string& attrs,
                     const location& ll)
diff --git a/libbuild2/test/script/script.hxx b/libbuild2/test/script/script.hxx
index b75f68e..319a9e2 100644
--- a/libbuild2/test/script/script.hxx
+++ b/libbuild2/test/script/script.hxx
@@ -32,6 +32,7 @@ namespace build2
       using build2::script::environment_vars;
       using build2::script::deadline;
       using build2::script::timeout;
+      using build2::script::command_function;
 
       class parser; // Required by VC for 'friend class parser' declaration.
 
@@ -105,7 +106,7 @@ namespace build2
         small_vector<const path*, 1> test_programs;
 
         void
-        set_variable (string&& name,
+        set_variable (string name,
                       names&&,
                       const string& attrs,
                       const location&) override;