Add support for 'for' loop second (... | for x) and third (for x <...) forms in script

11 files changed, 868 insertions, 69 deletions

diff --git a/libbuild2/build/script/lexer+for-loop.test.testscript b/libbuild2/build/script/lexer+for-loop.test.testscript
new file mode 100644
index 0000000..3f8e6b5
--- /dev/null
+++ b/libbuild2/build/script/lexer+for-loop.test.testscript
@@ -0,0 +1,188 @@
+# file      : libbuild2/build/script/lexer+for-loop.test.testscript
+# license   : MIT; see accompanying LICENSE file
+
+test.arguments = for-loop
+
+: redirect
+:
+{
+  : pass
+  :
+  $* <"cmd <| 1>|" >>EOO
+  'cmd'
+  <|
+  '1'
+  >|
+  <newline>
+  EOO
+
+  : null
+  :
+  $* <"cmd <- 1>-" >>EOO
+  'cmd'
+  <-
+  '1'
+  >-
+  <newline>
+  EOO
+
+  : trace
+  :
+  $* <"cmd 1>!" >>EOO
+  'cmd'
+  '1'
+  >!
+  <newline>
+  EOO
+
+  : merge
+  :
+  $* <"cmd 1>&2" >>EOO
+  'cmd'
+  '1'
+  >&
+  '2'
+  <newline>
+  EOO
+
+  : str
+  :
+  $* <"cmd <<<=a 1>>>?b" >>EOO
+  'cmd'
+  <<<=
+  'a'
+  '1'
+  >>>?
+  'b'
+  <newline>
+  EOO
+
+  : str-nn
+  :
+  $* <"cmd <<<=:a 1>>>?:b" >>EOO
+  'cmd'
+  <<<=:
+  'a'
+  '1'
+  >>>?:
+  'b'
+  <newline>
+  EOO
+
+  : str-nn-alias
+  :
+  $* <"cmd <<<:a 1>>>?:b" >>EOO
+  'cmd'
+  <<<:
+  'a'
+  '1'
+  >>>?:
+  'b'
+  <newline>
+  EOO
+
+  : doc
+  :
+  $* <"cmd <<EOI 1>>EOO" >>EOO
+  'cmd'
+  <<
+  'EOI'
+  '1'
+  >>
+  'EOO'
+  <newline>
+  EOO
+
+  : doc-nn
+  :
+  $* <"cmd <<:EOI 1>>?:EOO" >>EOO
+  'cmd'
+  <<:
+  'EOI'
+  '1'
+  >>?:
+  'EOO'
+  <newline>
+  EOO
+
+  : file-cmp
+  :
+  $* <"cmd <=in >?out 2>?err" >>EOO
+  'cmd'
+  <=
+  'in'
+  >?
+  'out'
+  '2'
+  >?
+  'err'
+  <newline>
+  EOO
+
+  : file-write
+  :
+  $* <"cmd >=out 2>+err" >>EOO
+  'cmd'
+  >=
+  'out'
+  '2'
+  >+
+  'err'
+  <newline>
+  EOO
+}
+
+: cleanup
+:
+{
+  : always
+  :
+  $* <"cmd &file" >>EOO
+  'cmd'
+  &
+  'file'
+  <newline>
+  EOO
+
+  : maybe
+  :
+  $* <"cmd &?file" >>EOO
+  'cmd'
+  &?
+  'file'
+  <newline>
+  EOO
+
+  : never
+  :
+  $* <"cmd &!file" >>EOO
+  'cmd'
+  &!
+  'file'
+  <newline>
+  EOO
+}
+
+: for
+:
+{
+  : form-1
+  :
+  $* <"for x: a" >>EOO
+  'for'
+  'x'
+  :
+  'a'
+  <newline>
+  EOO
+
+  : form-3
+  :
+  $* <"for <<<a x" >>EOO
+  'for'
+  <<<
+  'a'
+  'x'
+  <newline>
+  EOO
+}
diff --git a/libbuild2/build/script/lexer.cxx b/libbuild2/build/script/lexer.cxx
index d849ac9..5c13239 100644
--- a/libbuild2/build/script/lexer.cxx
+++ b/libbuild2/build/script/lexer.cxx
@@ -78,6 +78,19 @@ namespace build2
             s2 = "      ";
             break;
           }
+        case lexer_mode::for_loop:
+          {
+            // Leading tokens of the for-loop. Like command_line but
+            // recognizes colon as a separator and lsbrace like value.
+            //
+            // Note that while sensing the form of the for-loop (`for x:...`
+            // vs `for x <...`) we need to make sure that the pre-parsed token
+            // types are valid for the execution phase.
+            //
+            s1 = ":=!|&<> $(#\t\n";
+            s2 = " ==          ";
+            break;
+          }
         default:
           {
             // Recognize special variable names ($>, $<, $~).
@@ -109,6 +122,7 @@ namespace build2
         case lexer_mode::first_token:
         case lexer_mode::second_token:
         case lexer_mode::variable_line:
+        case lexer_mode::for_loop:
           r = next_line ();
           break;
         default: return base_lexer::next ();
@@ -141,7 +155,8 @@ namespace build2
         //
         if (st.lsbrace)
         {
-          assert (m == lexer_mode::variable_line);
+          assert (m == lexer_mode::variable_line ||
+                  m == lexer_mode::for_loop);
 
           state_.top ().lsbrace = false; // Note: st is a copy.
 
@@ -179,11 +194,20 @@ namespace build2
         case '(': return make_token (type::lparen);
         }
 
+        if (m == lexer_mode::for_loop)
+        {
+          switch (c)
+          {
+          case ':': return make_token (type::colon);
+          }
+        }
+
         // Command line operator/separators.
         //
         if (m == lexer_mode::command_line ||
             m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token)
+            m == lexer_mode::second_token ||
+            m == lexer_mode::for_loop)
         {
           switch (c)
           {
@@ -205,7 +229,8 @@ namespace build2
         //
         if (m == lexer_mode::command_line ||
             m == lexer_mode::first_token  ||
-            m == lexer_mode::second_token)
+            m == lexer_mode::second_token ||
+            m == lexer_mode::for_loop)
         {
           if (optional<token> t = next_cmd_op (c, sep))
             return move (*t);
diff --git a/libbuild2/build/script/lexer.hxx b/libbuild2/build/script/lexer.hxx
index 646d3b9..313d80a 100644
--- a/libbuild2/build/script/lexer.hxx
+++ b/libbuild2/build/script/lexer.hxx
@@ -24,9 +24,10 @@ namespace build2
         enum
         {
           command_line = base_type::value_next,
-          first_token,  // Expires at the end of the token.
-          second_token, // Expires at the end of the token.
-          variable_line // Expires at the end of the line.
+          first_token,   // Expires at the end of the token.
+          second_token,  // Expires at the end of the token.
+          variable_line, // Expires at the end of the line.
+          for_loop       // Used for sensing the for-loop leading tokens.
         };
 
         lexer_mode () = default;
diff --git a/libbuild2/build/script/lexer.test.cxx b/libbuild2/build/script/lexer.test.cxx
index e496f94..d8733ba 100644
--- a/libbuild2/build/script/lexer.test.cxx
+++ b/libbuild2/build/script/lexer.test.cxx
@@ -35,6 +35,7 @@ namespace build2
           else if (s == "second-token")      m = lexer_mode::second_token;
           else if (s == "variable-line")     m = lexer_mode::variable_line;
           else if (s == "variable")          m = lexer_mode::variable;
+          else if (s == "for-loop")          m = lexer_mode::for_loop;
           else                               assert (false);
         }
 
diff --git a/libbuild2/build/script/parser+for.test.testscript b/libbuild2/build/script/parser+for.test.testscript
index 877f958..c5f6587 100644
--- a/libbuild2/build/script/parser+for.test.testscript
+++ b/libbuild2/build/script/parser+for.test.testscript
@@ -16,7 +16,7 @@
       cmd
     end
     EOI
-    buildfile:11:4: error: expected variable name instead of <newline>
+    buildfile:11:1: error: for: missing variable name
     EOE
 
     : untyped
@@ -180,3 +180,461 @@
     EOE
   }
 }
+
+: form-2
+:
+: ... | for x
+:
+{
+  : for
+  :
+  {
+    : status
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x != 0
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: for-loop exit code cannot be checked
+    EOE
+
+    : not-last
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x | echo x
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: for-loop must be last command in a pipe
+    EOE
+
+    : not-last-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x|echo x
+      cmd
+    end
+    EOI
+    buildfile:11:19: error: for-loop must be last command in a pipe
+    EOE
+
+    : expression-after
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x && echo x
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: command expression involving for-loop
+    EOE
+
+    : expression-after-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x&&echo x
+      cmd
+    end
+    EOI
+    buildfile:11:19: error: command expression involving for-loop
+    EOE
+
+    : expression-before
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && echo x | for x
+      cmd
+    end
+    EOI
+    buildfile:11:24: error: command expression involving for-loop
+    EOE
+
+    : expression-before-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && echo x|for x
+      cmd
+    end
+    EOI
+    buildfile:11:22: error: command expression involving for-loop
+    EOE
+
+    : cleanup
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x &f
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: cleanup in for-loop
+    EOE
+
+    : cleanup-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x&f
+      cmd
+    end
+    EOI
+    buildfile:11:19: error: cleanup in for-loop
+    EOE
+
+    : stdout-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x >a
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x>a
+      cmd
+    end
+    EOI
+    buildfile:11:19: error: output redirect in for-loop
+    EOE
+
+    : stdin-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x <a
+      cmd
+    end
+    EOI
+    buildfile:11:20: error: stdin is both piped and redirected
+    EOE
+
+    : no-var
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for
+      cmd
+    end
+    EOI
+    buildfile:11:1: error: for: missing variable name
+    EOE
+
+    : untyped
+    :
+    $* <<EOI >>EOO
+    echo 'a b' | for -w x
+      cmd $x
+    end
+    EOI
+    echo 'a b' | for -w x
+    EOO
+
+    : expansion
+    :
+    $* <<EOI >>EOO
+    vs = a b
+    echo $vs | for x
+      cmd $x
+    end
+    EOI
+    echo a b | for x
+    EOO
+
+    : typed-var
+    :
+    $* <<EOI >>EOO
+    echo 'a b' | for -w [dir_paths] x
+      cmd $x
+    end
+    EOI
+    echo 'a b' | for -w [dir_paths] x
+    EOO
+  }
+
+  : end
+  :
+  {
+    : without-end
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      cmd
+    EOI
+    buildfile:13:1: error: expected closing 'end'
+    EOE
+  }
+
+  : elif
+  :
+  {
+    : without-if
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' | for x
+      elif true
+        cmd
+      end
+    end
+    EOI
+    buildfile:12:3: error: 'elif' without preceding 'if'
+    EOE
+  }
+
+  : nested
+  :
+  {
+    $* -l -r <<EOI >>EOO
+    echo 'a b' | for x      # 1
+      cmd1 $x               # 2
+      if ($x == "a")        # 3
+        cmd2                # 4
+        echo x y | for y    # 5
+          cmd3              # 6
+        end
+      else
+        cmd4                # 7
+      end
+      cmd5                  # 8
+    end
+    cmd6                    # 9
+    EOI
+    echo 'a b' | for x # 1
+    cmd6 # 9
+    EOO
+  }
+}
+
+: form-3
+:
+: for x <...
+:
+{
+  : for
+  :
+  {
+    : status
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a != 0
+      cmd
+    end
+    EOI
+    buildfile:11:10: error: for-loop exit code cannot be checked
+    EOE
+
+    : not-last
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a | echo x
+      cmd
+    end
+    EOI
+    buildfile:11:10: error: for-loop must be last command in a pipe
+    EOE
+
+    : not-last-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x|echo x
+      cmd
+    end
+    EOI
+    buildfile:11:9: error: for-loop must be last command in a pipe
+    EOE
+
+    : expression-after
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a && echo x
+      cmd
+    end
+    EOI
+    buildfile:11:10: error: command expression involving for-loop
+    EOE
+
+    : expression-after-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x&&echo x
+      cmd
+    end
+    EOI
+    buildfile:11:9: error: command expression involving for-loop
+    EOE
+
+    : expression-before
+    :
+    $* <<EOI 2>>EOE != 0
+    echo 'a b' && for x <a
+      cmd
+    end
+    EOI
+    buildfile:11:15: error: command expression involving for-loop
+    EOE
+
+    : cleanup
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <a &f
+      cmd
+    end
+    EOI
+    buildfile:11:10: error: cleanup in for-loop
+    EOE
+
+    : cleanup-before-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for &f x <a
+      cmd
+    end
+    EOI
+    buildfile:11:5: error: cleanup in for-loop
+    EOE
+
+    : cleanup-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a x&f
+      cmd
+    end
+    EOI
+    buildfile:11:9: error: cleanup in for-loop
+    EOE
+
+    : stdout-redirect
+    :
+    $* <<EOI 2>>EOE != 0
+    for x >a
+      cmd
+    end
+    EOI
+    buildfile:11:7: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-before-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for >a x
+      cmd
+    end
+    EOI
+    buildfile:11:5: error: output redirect in for-loop
+    EOE
+
+    : stdout-redirect-relex
+    :
+    $* <<EOI 2>>EOE != 0
+    for x>a
+      cmd
+    end
+    EOI
+    buildfile:11:6: error: output redirect in for-loop
+    EOE
+
+    : no-var
+    :
+    $* <<EOI 2>>EOE != 0
+    for <a
+      cmd
+    end
+    EOI
+    buildfile:11:1: error: for: missing variable name
+    EOE
+
+    : untyped
+    :
+    $* <<EOI >>EOO
+    for -w x <'a b'
+      cmd $x
+    end
+    EOI
+    for -w x <'a b'
+    EOO
+
+    : expansion
+    :
+    $* <<EOI >>EOO
+    vs = a b
+    for x <$vs
+      cmd $x
+    end
+    EOI
+    for x b <a
+    EOO
+
+    : typed-var
+    :
+    $* <<EOI >>EOO
+    for -w [dir_path] x <'a b'
+      cmd $x
+    end
+    EOI
+    for -w [dir_path] x <'a b'
+    EOO
+  }
+
+  : end
+  :
+  {
+    : without-end
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      cmd
+    EOI
+    buildfile:13:1: error: expected closing 'end'
+    EOE
+  }
+
+  : elif
+  :
+  {
+    : without-if
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+      elif true
+        cmd
+      end
+    end
+    EOI
+    buildfile:12:3: error: 'elif' without preceding 'if'
+    EOE
+  }
+
+  : nested
+  :
+  {
+    $* -l -r <<EOI >>EOO
+    for -w x <'a b'         # 1
+      cmd1 $x               # 2
+      if ($x == "a")        # 3
+        cmd2                # 4
+        for -w y <'x y'     # 5
+          cmd3              # 6
+        end
+      else
+        cmd4                # 7
+      end
+      cmd5                  # 8
+    end
+    cmd6                    # 9
+    EOI
+    for -w x <'a b' # 1
+    cmd6 # 9
+    EOO
+  }
+
+  : contained
+  :
+  {
+    : eos
+    :
+    $* <<EOI 2>>EOE != 0
+    for x <'a b'
+    EOI
+    buildfile:12:1: error: expected closing 'end'
+    EOE
+  }
+}
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index bcea7e0..035ab6b 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -205,10 +205,11 @@ namespace build2
         // enter: next token is peeked at (type in tt)
         // leave: newline
 
-        assert (!fct                         ||
-                *fct == line_type::cmd_if    ||
-                *fct == line_type::cmd_while ||
-                *fct == line_type::cmd_for);
+        assert (!fct                              ||
+                *fct == line_type::cmd_if         ||
+                *fct == line_type::cmd_while      ||
+                *fct == line_type::cmd_for_stream ||
+                *fct == line_type::cmd_for_args);
 
         // Determine the line type/start token.
         //
@@ -246,50 +247,107 @@ namespace build2
 
             break;
           }
-        case line_type::cmd_for:
+          //
+          // See pre_parse_line_start() for details.
+          //
+        case line_type::cmd_for_args: assert (false); break;
+        case line_type::cmd_for_stream:
           {
-            // First take care of the variable name.
+            // First we need to sense the next few tokens and detect which
+            // form of the loop we are dealing with, the first (for x: ...)
+            // or the third (x <...) one. Note that the second form (... | for
+            // x) is handled separately.
+            //
+            // @@ Do we diagnose `... | for x: ...`?
+            //
+            // If the next token doesn't introduce a variable (doesn't start
+            // attributes and doesn't look like a variable name), then this is
+            // the third form. Otherwise, if colon follows the variable name,
+            // then this is the first form and the third form otherwise.
+            //
+            // Note that for the third form we will need to pass the 'for'
+            // token as a program name to the command expression parsing
+            // function since it will be gone from the token stream by that
+            // time. Thus, we save it.
             //
-            mode (lexer_mode::normal);
+            // Note also that in this model it won't be possible to support
+            // options in the first form.
+            //
+            token pt (t);
+            assert (pt.type == type::word && pt.value == "for");
 
+            mode (lexer_mode::for_loop);
             next_with_attributes (t, tt);
-            attributes_push (t, tt);
 
-            if (tt != type::word || t.qtype != quote_type::unquoted)
-              fail (t) << "expected variable name instead of " << t;
+            // Note that we also consider special variable names (those that
+            // don't clash with the command line elements like redirects, etc)
+            // to later fail gracefully.
+            //
+            string& n (t.value);
 
-            const string& n (t.value);
+            if (tt == type::lsbrace ||                       // Attributes.
+                (tt == type::word                &&          // Variable name.
+                 t.qtype == quote_type::unquoted &&
+                 (n[0] == '_' || alpha (n[0]) || n == "~")))
+            {
+              attributes_push (t, tt);
 
-            if (special_variable (n))
-              fail (t) << "attempt to set '" << n << "' variable directly";
+              if (tt != type::word || t.qtype != quote_type::unquoted)
+                fail (t) << "expected variable name instead of " << t;
 
-            // We don't pre-enter variables.
-            //
-            ln.var = nullptr;
+              if (special_variable (n))
+                fail (t) << "attempt to set '" << n << "' special variable";
 
-            next (t, tt);
+              if (lexer_->peek_char ().first == ':')
+                lt = line_type::cmd_for_args;
+            }
 
-            if (tt != type::colon)
+            if (lt == line_type::cmd_for_stream) // for x <...
             {
-              // @@ TMP We will need to fallback to parsing the 'for x <...'
-              //        form instead.
+              // At this point `t` contains the token that follows the `for`
+              // token and, potentially, the attributes. Now pre-parse the
+              // command expression in the command_line lexer mode starting
+              // from this position and also passing the 'for' token as a
+              // program name.
               //
-              fail (t) << "expected ':' instead of " << t
-                       << " after variable name";
+              // Note that the fact that the potential attributes are already
+              // parsed doesn't affect the command expression pre-parsing.
+              // Also note that they will be available during the execution
+              // phase being replayed.
+              //
+              expire_mode (); // Expire the for-loop lexer mode.
+
+              parse_command_expr_result r (
+                parse_command_expr (t, tt,
+                                    lexer::redirect_aliases,
+                                    move (pt)));
+
+              assert (r.for_loop);
+
+              if (tt != type::newline)
+                fail (t) << "expected newline instead of " << t;
+
+              parse_here_documents (t, tt, r);
             }
+            else                                 // for x: ...
+            {
+              next (t, tt);
 
-            expire_mode (); // Expire the normal lexer mode.
+              assert (tt == type::colon);
 
-            // Parse the value similar to the var line type (see above).
-            //
-            mode (lexer_mode::variable_line);
-            parse_variable_line (t, tt);
+              expire_mode (); // Expire the for-loop lexer mode.
 
-            if (tt != type::newline)
-              fail (t) << "expected newline instead of " << t << " after for";
+              // Parse the value similar to the var line type (see above).
+              //
+              mode (lexer_mode::variable_line);
+              parse_variable_line (t, tt);
 
-            ++level_;
+              if (tt != type::newline)
+                fail (t) << "expected newline instead of " << t << " after for";
+            }
 
+            ln.var = nullptr;
+            ++level_;
             break;
           }
         case line_type::cmd_elif:
@@ -321,15 +379,24 @@ namespace build2
           // Fall through.
         case line_type::cmd:
           {
-            pair<command_expr, here_docs> p;
+            parse_command_expr_result r;
 
             if (lt != line_type::cmd_else && lt != line_type::cmd_end)
-              p = parse_command_expr (t, tt, lexer::redirect_aliases);
+              r = parse_command_expr (t, tt, lexer::redirect_aliases);
+
+            if (r.for_loop)
+            {
+              lt     = line_type::cmd_for_stream;
+              ln.var = nullptr;
+
+              ++level_;
+            }
 
             if (tt != type::newline)
               fail (t) << "expected newline instead of " << t;
 
-            parse_here_documents (t, tt, p);
+            parse_here_documents (t, tt, r);
+
             break;
           }
         }
@@ -358,7 +425,8 @@ namespace build2
             break;
           }
         case line_type::cmd_while:
-        case line_type::cmd_for:
+        case line_type::cmd_for_stream:
+        case line_type::cmd_for_args:
           {
             tt = peek (lexer_mode::first_token);
 
@@ -396,7 +464,8 @@ namespace build2
             break;
           }
         case line_type::cmd_while:
-        case line_type::cmd_for:
+        case line_type::cmd_for_stream:
+        case line_type::cmd_for_args:
           {
             fct = bt;
             break;
@@ -466,7 +535,9 @@ namespace build2
         // enter: peeked first token of next line (type in tt)
         // leave: newline
 
-        assert (lt == line_type::cmd_while || lt == line_type::cmd_for);
+        assert (lt == line_type::cmd_while      ||
+                lt == line_type::cmd_for_stream ||
+                lt == line_type::cmd_for_args);
 
         // Parse lines until we see closing 'end'.
         //
@@ -491,12 +562,12 @@ namespace build2
         //
         assert (!pre_parse_);
 
-        pair<command_expr, here_docs> p (
+        parse_command_expr_result pr (
           parse_command_expr (t, tt, lexer::redirect_aliases));
 
         assert (tt == type::newline);
 
-        parse_here_documents (t, tt, p);
+        parse_here_documents (t, tt, pr);
         assert (tt == type::newline);
 
         // @@ Note that currently running programs via a runner (e.g., see
@@ -509,7 +580,7 @@ namespace build2
         //    passed to the environment constructor, similar to passing the
         //    script deadline.
         //
-        return move (p.first);
+        return move (pr.expr);
       }
 
       //
@@ -1121,6 +1192,7 @@ namespace build2
         auto exec_cmd = [this] (token& t, build2::script::token_type& tt,
                                 const iteration_index* ii, size_t li,
                                 bool single,
+                                const function<command_function>& cf,
                                 const location& ll)
         {
           // We use the 0 index to signal that this is the only command.
@@ -1131,7 +1203,7 @@ namespace build2
           command_expr ce (
             parse_command_line (t, static_cast<token_type&> (tt)));
 
-          runner_->run (*environment_, ce, ii, li, ll);
+          runner_->run (*environment_, ce, ii, li, cf, ll);
         };
 
         exec_lines (s.body, exec_cmd);
@@ -1184,6 +1256,7 @@ namespace build2
                                        build2::script::token_type& tt,
                                        const iteration_index* ii, size_t li,
                                        bool /* single */,
+                                       const function<command_function>& cf,
                                        const location& ll)
         {
           // Note that we never reset the line index to zero (as we do in
@@ -1282,14 +1355,17 @@ namespace build2
             command_expr ce (
               parse_command_line (t, static_cast<token_type&> (tt)));
 
-            // Verify that this expression executes the set builtin.
+            // Verify that this expression executes the set builtin or is a
+            // for-loop.
             //
             if (find_if (ce.begin (), ce.end (),
-                         [] (const expr_term& et)
+                         [&cf] (const expr_term& et)
                          {
                            const process_path& p (et.pipe.back ().program);
                            return p.initial == nullptr &&
-                                  p.recall.string () == "set";
+                                  (p.recall.string () == "set" ||
+                                  (cf != nullptr &&
+                                   p.recall.string () == "for"));
                          }) == ce.end ())
             {
               const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens);
@@ -1301,7 +1377,7 @@ namespace build2
                 info (rt[0].location ()) << "depdb preamble ends here";
             }
 
-            runner_->run (*environment_, ce, ii, li, ll);
+            runner_->run (*environment_, ce, ii, li, cf, ll);
           }
         };
 
@@ -2336,18 +2412,36 @@ namespace build2
             {
               // Note: depdb is disallowed inside flow control constructs.
               //
-              string s;
-              build2::script::run (*environment_,
-                                   cmd,
-                                   nullptr /* iteration_index */, li,
-                                   ll,
-                                   !file ? &s : nullptr);
-
               if (!file)
               {
-                iss.str (move (s));
+                function<command_function> cf (
+                  [&iss]
+                  (build2::script::environment&,
+                   const strings&,
+                   auto_fd in,
+                   bool pipe,
+                   const optional<deadline>& dl,
+                   const command& deadline_cmd,
+                   const location& ll)
+                  {
+                    iss.str (stream_read (move (in),
+                                          pipe,
+                                          dl,
+                                          deadline_cmd,
+                                          ll));
+                  });
+
+                build2::script::run (*environment_,
+                                     cmd,
+                                     nullptr /* iteration_index */, li,
+                                     ll,
+                                     cf, false /* last_cmd */);
+
                 iss.exceptions (istream::badbit);
               }
+              else
+                build2::script::run (
+                  *environment_, cmd, nullptr /* iteration_index */, li, ll);
             }
 
             ifdstream ifs (ifdstream::badbit);
@@ -2490,7 +2584,8 @@ namespace build2
           environment_->set_special_variables (a);
       }
 
-      // When add a special variable don't forget to update lexer::word().
+      // When add a special variable don't forget to update lexer::word() and
+      // for-loop parsing in pre_parse_line().
       //
       bool parser::
       special_variable (const string& n) noexcept
diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx
index 061f3f8..7e9c612 100644
--- a/libbuild2/build/script/parser.test.cxx
+++ b/libbuild2/build/script/parser.test.cxx
@@ -37,11 +37,32 @@ namespace build2
         enter (environment&, const location&) override {}
 
         virtual void
-        run (environment&,
+        run (environment& env,
              const command_expr& e,
              const iteration_index* ii, size_t i,
-             const location&) override
+             const function<command_function>& cf,
+             const location& ll) override
         {
+          // If the functions is specified, then just execute it with an empty
+          // stdin so it can perform the housekeeping (stop replaying tokens,
+          // increment line index, etc).
+          //
+          if (cf != nullptr)
+          {
+            assert (e.size () == 1 && !e[0].pipe.empty ());
+
+            const command& c (e[0].pipe.back ());
+
+            // Must be enforced by the caller.
+            //
+            assert (!c.out && !c.err && !c.exit);
+
+            cf (env, c.arguments,
+                fdopen_null (), false /* pipe */,
+                nullopt /* deadline */, c,
+                ll);
+          }
+
           cout << e;
 
           if (line_ || iterations_)
diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx
index 157fc60..c52ef66 100644
--- a/libbuild2/build/script/runner.cxx
+++ b/libbuild2/build/script/runner.cxx
@@ -97,25 +97,28 @@ namespace build2
       run (environment& env,
            const command_expr& expr,
            const iteration_index* ii, size_t li,
+           const function<command_function>& cf,
            const location& ll)
       {
         if (verb >= 3)
           text << ":  " << expr;
 
         // Run the expression if we are not in the dry-run mode or if it
-        // executes the set or exit builtin and just print the expression
-        // otherwise at verbosity level 2 and up.
+        // executes the set or exit builtin or it is a for-loop. Otherwise,
+        // just print the expression otherwise at verbosity level 2 and up.
         //
         if (!env.context.dry_run ||
             find_if (expr.begin (), expr.end (),
-                     [] (const expr_term& et)
+                     [&cf] (const expr_term& et)
                      {
                        const process_path& p (et.pipe.back ().program);
                        return p.initial == nullptr &&
                               (p.recall.string () == "set" ||
-                               p.recall.string () == "exit");
+                               p.recall.string () == "exit" ||
+                               (cf != nullptr &&
+                                p.recall.string () == "for"));
                      }) != expr.end ())
-          build2::script::run (env, expr, ii, li, ll);
+          build2::script::run (env, expr, ii, li, ll, cf);
         else if (verb >= 2)
           text << expr;
       }
diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx
index 0652396..ec8a948 100644
--- a/libbuild2/build/script/runner.hxx
+++ b/libbuild2/build/script/runner.hxx
@@ -32,10 +32,14 @@ namespace build2
         // Location is the start position of this command line in the script.
         // It can be used in diagnostics.
         //
+        // Optionally, execute the specified function instead of the last
+        // pipe command.
+        //
         virtual void
         run (environment&,
              const command_expr&,
              const iteration_index*, size_t index,
+             const function<command_function>&,
              const location&) = 0;
 
         virtual bool
@@ -66,6 +70,7 @@ namespace build2
         run (environment&,
              const command_expr&,
              const iteration_index*, size_t,
+             const function<command_function>&,
              const location&) override;
 
         virtual bool
diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx
index 2e777b4..9d9b5a8 100644
--- a/libbuild2/build/script/script.cxx
+++ b/libbuild2/build/script/script.cxx
@@ -156,7 +156,7 @@ namespace build2
       }
 
       void environment::
-      set_variable (string&& nm,
+      set_variable (string nm,
                     names&& val,
                     const string& attrs,
                     const location& ll)
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index 2c5e6e0..ec27781 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -24,11 +24,13 @@ namespace build2
       using build2::script::lines;
       using build2::script::redirect;
       using build2::script::redirect_type;
+      using build2::script::command;
       using build2::script::expr_term;
       using build2::script::command_expr;
       using build2::script::iteration_index;
       using build2::script::deadline;
       using build2::script::timeout;
+      using build2::script::command_function;
 
       // Forward declarations.
       //
@@ -166,7 +168,7 @@ namespace build2
         size_t exec_line = 1;
 
         virtual void
-        set_variable (string&& name,
+        set_variable (string name,
                       names&&,
                       const string& attrs,
                       const location&) override;