Add support for for-loop

The semantics is similar to the C++11 range-based for:

list = 1 2 3
for i: $list
  print $i

Note that there is no scoping of any kind for the loop variable ('i' in
the above example).

See tests/loop/for.test for some examples/ideas.

In the future the plan is to also support more general while-loop as well
as break and continue.

14 files changed, 342 insertions, 20 deletions

diff --git a/build2/context.cxx b/build2/context.cxx
index 32af1c2..d074d9c 100644
--- a/build2/context.cxx
+++ b/build2/context.cxx
@@ -387,7 +387,7 @@ namespace build2
       // (basically what's necessary inside a double-quoted literal plus the
       // single quote).
       //
-      lexer l (is, path ("<cmdline>"), "\'\"\\$(");
+      lexer l (is, path ("<cmdline>"), 1 /* line */, "\'\"\\$(");
 
       // The first token should be a word, either the variable name or the
       // scope qualification.
diff --git a/build2/function.cxx b/build2/function.cxx
index 2d04b30..4421c86 100644
--- a/build2/function.cxx
+++ b/build2/function.cxx
@@ -160,7 +160,7 @@ namespace build2
 
             if (!perf && at != nullptr && ft != nullptr)
             {
-              while ((at = at->base) != nullptr && at != ft) ;
+              while ((at = at->base_type) != nullptr && at != ft) ;
 
               if (at != nullptr) // Types match via derived-to-base.
                 continue;
diff --git a/build2/lexer.hxx b/build2/lexer.hxx
index d88d5ba..a4eda1e 100644
--- a/build2/lexer.hxx
+++ b/build2/lexer.hxx
@@ -66,15 +66,18 @@ namespace build2
     lexer_mode (base_type v): base_type (v) {}
   };
 
-  class lexer: protected butl::char_scanner
+  class lexer: public butl::char_scanner
   {
   public:
     // If escape is not NULL then only escape sequences with characters from
     // this string are considered "effective escapes" with all others passed
     // through as is. Note that the escape string is not copied.
     //
-    lexer (istream& is, const path& name, const char* escapes = nullptr)
-        : lexer (is, name, escapes, true) {}
+    lexer (istream& is,
+           const path& name,
+           uint64_t line = 1, // Start line in the stream.
+           const char* escapes = nullptr)
+        : lexer (is, name, line, escapes, true /* set_mode */) {}
 
     const path&
     name () const {return name_;}
@@ -164,11 +167,18 @@ namespace build2
     // Lexer state.
     //
   protected:
-    lexer (istream& is, const path& n, const char* e, bool sm)
-        : char_scanner (is), fail ("error", &name_), name_ (n), sep_ (false)
+    lexer (istream& is,
+           const path& name,
+           uint64_t line,
+           const char* escapes,
+           bool set_mode)
+        : char_scanner (is, true /* crlf */, line),
+          fail ("error", &name_),
+          name_ (name),
+          sep_ (false)
     {
-      if (sm)
-        mode (lexer_mode::normal, '@', e);
+      if (set_mode)
+        mode (lexer_mode::normal, '@', escapes);
     }
 
     const path name_;
diff --git a/build2/name.hxx b/build2/name.hxx
index 388415b..6f329ef 100644
--- a/build2/name.hxx
+++ b/build2/name.hxx
@@ -134,7 +134,7 @@ namespace build2
 
   // Vector of names.
   //
-  // Quote often it will contain just one element so we use small_vector<1>.
+  // Quite often it will contain just one element so we use small_vector<1>.
   // Note also that it must be a separate type rather than an alias for
   // vector<name> in order to distinguish between untyped variable values
   // (names) and typed ones (vector<name>).
diff --git a/build2/parser.cxx b/build2/parser.cxx
index a33f0a3..1863f67 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -4,6 +4,7 @@
 
 #include <build2/parser.hxx>
 
+#include <sstream>
 #include <iostream> // cout
 
 #include <libbutl/filesystem.mxx> // path_search(), path_match()
@@ -385,6 +386,10 @@ namespace build2
           //
           fail (t) << n << " without if";
         }
+        else if (n == "for")
+        {
+          f = &parser::parse_for;
+        }
 
         if (f != nullptr)
         {
@@ -1554,6 +1559,159 @@ namespace build2
   }
 
   void parser::
+  parse_for (token& t, type& tt)
+  {
+    // for <varname>: <value>
+    //   <line>
+    //
+    // for <varname>: <value>
+    // {
+    //   <block>
+    // }
+    //
+
+    // First take care of the variable name. There is no reason not to
+    // support variable attributes.
+    //
+    next (t, tt);
+    attributes_push (t, tt);
+
+    // @@ PAT: currently we pattern-expand for var.
+    //
+    const location vloc (get_location (t));
+    names vns (parse_names (t, tt, pattern_mode::expand));
+
+    if (tt != type::colon)
+      fail (t) << "expected ':' instead of " << t << " after variable name";
+
+    const variable& var (
+      var_pool.rw (*scope_).insert (
+        parse_variable_name (move (vns), vloc),
+        true /* overridable */));
+
+    apply_variable_attributes (var);
+
+    if (var.visibility == variable_visibility::target)
+      fail (vloc) << "variable " << var << " has target visibility but "
+                  << "assigned in for-loop";
+
+    // Now the value (list of names) to iterate over. Parse it as a variable
+    // value to get expansion, attributes, etc.
+    //
+    value val;
+    apply_value_attributes (
+      nullptr, val, parse_variable_value (t, tt), type::assign);
+
+    // If this value is a vector, then save its element type so that we
+    // can typify each element below.
+    //
+    const value_type* etype (nullptr);
+
+    if (val && val.type != nullptr)
+    {
+      etype = val.type->element_type;
+      untypify (val);
+    }
+
+    if (tt != type::newline)
+      fail (t) << "expected newline instead of " << t << " after for";
+
+    // Finally the body. The initial thought was to use the token replay
+    // facility but on closer inspection this didn't turn out to be a good
+    // idea (no support for nested replays, etc). So instead we are going to
+    // do a full-blown re-lex. Specifically, we will first skip the line/block
+    // just as we do for non-taken if/else branches while saving the character
+    // sequence that comprises the body. Then we re-lex/parse it on each
+    // iteration.
+    //
+    string body;
+    uint64_t line (lexer_->line); // Line of the first character to be saved.
+    lexer::save_guard sg (*lexer_, body);
+
+    // This can be a block or a single line, similar to if-else.
+    //
+    bool block (next (t, tt) == type::lcbrace && peek () == type::newline);
+
+    if (block)
+    {
+      next (t, tt); // Get newline.
+      next (t, tt);
+
+      skip_block (t, tt);
+      sg.stop ();
+
+      if (tt != type::rcbrace)
+        fail (t) << "expected } instead of " << t << " at the end of for-block";
+
+      next (t, tt);
+
+      if (tt == type::newline)
+        next (t, tt);
+      else if (tt != type::eos)
+        fail (t) << "expected newline after }";
+    }
+    else
+    {
+      skip_line (t, tt);
+      sg.stop ();
+
+      if (tt == type::newline)
+        next (t, tt);
+    }
+
+    // Iterate.
+    //
+    names& ns (val.as<names> ());
+
+    if (ns.empty ())
+      return;
+
+    value& v (scope_->assign (var));
+
+    istringstream is (move (body));
+
+    for (auto i (ns.begin ()), e (ns.end ());; )
+    {
+      // Set the variable value.
+      //
+      bool pair (i->pair);
+      names n;
+      n.push_back (move (*i));
+      if (pair) n.push_back (move (*++i));
+      v = value (move (n));
+
+      if (etype != nullptr)
+        typify (v, *etype, &var);
+
+      lexer l (is, *path_, line);
+      lexer* ol (lexer_);
+      lexer_ = &l;
+
+      token t;
+      type tt;
+      next (t, tt);
+
+      if (block)
+      {
+        next (t, tt); // {
+        next (t, tt); // <newline>
+      }
+      parse_clause (t, tt);
+      assert (tt == (block ? type::rcbrace : type::eos));
+
+      lexer_ = ol;
+
+      if (++i == e)
+        break;
+
+      // Rewind the stream.
+      //
+      is.clear ();
+      is.seekg (0);
+    }
+  }
+
+  void parser::
   parse_assert (token& t, type& tt)
   {
     bool neg (t.value.back () == '!');
@@ -1730,12 +1888,15 @@ namespace build2
       n == "dir_path"       ? ptr (value_traits<dir_path>::value_type)       :
       n == "abs_dir_path"   ? ptr (value_traits<abs_dir_path>::value_type)   :
       n == "name"           ? ptr (value_traits<name>::value_type)           :
+      n == "name_pair"      ? ptr (value_traits<name_pair>::value_type)      :
       n == "target_triplet" ? ptr (value_traits<target_triplet>::value_type) :
+
+      n == "uint64s"        ? ptr (value_traits<uint64s>::value_type)        :
       n == "strings"        ? ptr (value_traits<strings>::value_type)        :
       n == "paths"          ? ptr (value_traits<paths>::value_type)          :
       n == "dir_paths"      ? ptr (value_traits<dir_paths>::value_type)      :
       n == "names"          ? ptr (value_traits<vector<name>>::value_type)   :
-      n == "name_pair"      ? ptr (value_traits<name_pair>::value_type)      :
+
       nullptr;
   }
 
@@ -4000,7 +4161,7 @@ namespace build2
     // We do "effective escaping" and only for ['"\$(] (basically what's
     // necessary inside a double-quoted literal plus the single quote).
     //
-    lexer l (is, *path_, "\'\"\\$(");
+    lexer l (is, *path_, 1 /* line */, "\'\"\\$(");
     lexer_ = &l;
     scope_ = root_ = scope::global_;
     pbase_ = &work; // Use current working directory.
@@ -4268,6 +4429,11 @@ namespace build2
     // Lookup.
     //
     const auto& var (var_pool.rw (*scope_).insert (move (name), true));
+
+    if (target_ == nullptr && var.visibility == variable_visibility::target)
+      fail (loc) << "variable " << var << " has target visibility but "
+                 << "expanded in a scope";
+
     return target_ != nullptr ? (*target_)[var] : (*scope_)[var];
 
     // Undefined/NULL namespace variables are not allowed.
diff --git a/build2/parser.hxx b/build2/parser.hxx
index 90b01e0..ce9ad2f 100644
--- a/build2/parser.hxx
+++ b/build2/parser.hxx
@@ -106,6 +106,9 @@ namespace build2
     parse_if_else (token&, token_type&);
 
     void
+    parse_for (token&, token_type&);
+
+    void
     parse_variable (token&, token_type&, const variable&, token_type);
 
     string
diff --git a/build2/test/script/lexer.hxx b/build2/test/script/lexer.hxx
index a262764..5e013b2 100644
--- a/build2/test/script/lexer.hxx
+++ b/build2/test/script/lexer.hxx
@@ -49,7 +49,11 @@ namespace build2
                const path& name,
                lexer_mode m,
                const char* escapes = nullptr)
-            : base_lexer (is, name, nullptr, false)
+            : base_lexer (is,
+                          name,
+                          1       /* line */,
+                          nullptr /* escapes */,
+                          false   /* set_mode */)
         {
           mode (m, '\0', escapes);
         }
diff --git a/build2/types.hxx b/build2/types.hxx
index f65d054..cf95a9d 100644
--- a/build2/types.hxx
+++ b/build2/types.hxx
@@ -55,6 +55,8 @@ namespace build2
   using std::uint64_t;
   using std::uintptr_t;
 
+  using uint64s = std::vector<uint64_t>;
+
   using std::size_t;
   using std::nullptr_t;
 
@@ -64,6 +66,9 @@ namespace build2
   using std::function;
   using std::reference_wrapper;
 
+  using strings = std::vector<string>;
+  using cstrings = std::vector<const char*>;
+
   using std::hash;
 
   using std::initializer_list;
@@ -77,9 +82,6 @@ namespace build2
   using butl::vector_view;  // <libbutl/vector-view.mxx>
   using butl::small_vector; // <libbutl/small-vector.mxx>
 
-  using strings = vector<string>;
-  using cstrings = vector<const char*>;
-
   using std::istream;
   using std::ostream;
 
diff --git a/build2/variable.cxx b/build2/variable.cxx
index 1bfd21c..1afa872 100644
--- a/build2/variable.cxx
+++ b/build2/variable.cxx
@@ -421,6 +421,7 @@ namespace build2
     type_name,
     sizeof (bool),
     nullptr,                      // No base.
+    nullptr,                      // No element.
     nullptr,                      // No dtor (POD).
     nullptr,                      // No copy_ctor (POD).
     nullptr,                      // No copy_assign (POD).
@@ -462,6 +463,7 @@ namespace build2
     type_name,
     sizeof (uint64_t),
     nullptr,                          // No base.
+    nullptr,                          // No element.
     nullptr,                          // No dtor (POD).
     nullptr,                          // No copy_ctor (POD).
     nullptr,                          // No copy_assign (POD).
@@ -538,6 +540,7 @@ namespace build2
     type_name,
     sizeof (string),
     nullptr,                      // No base.
+    nullptr,                      // No element.
     &default_dtor<string>,
     &default_copy_ctor<string>,
     &default_copy_assign<string>,
@@ -589,6 +592,7 @@ namespace build2
     type_name,
     sizeof (path),
     nullptr,                    // No base.
+    nullptr,                    // No element.
     &default_dtor<path>,
     &default_copy_ctor<path>,
     &default_copy_assign<path>,
@@ -632,7 +636,9 @@ namespace build2
   {
     type_name,
     sizeof (dir_path),
-    &value_traits<path>::value_type, // Assume direct cast works for both.
+    &value_traits<path>::value_type, // Base (assuming direct cast works for
+                                     // both).
+    nullptr,                         // No element.
     &default_dtor<dir_path>,
     &default_copy_ctor<dir_path>,
     &default_copy_assign<dir_path>,
@@ -678,7 +684,9 @@ namespace build2
   {
     type_name,
     sizeof (abs_dir_path),
-    &value_traits<dir_path>::value_type, // Assume direct cast works for both.
+    &value_traits<dir_path>::value_type, // Base (assuming direct cast works
+                                         // for both).
+    nullptr,                             // No element.
     &default_dtor<abs_dir_path>,
     &default_copy_ctor<abs_dir_path>,
     &default_copy_assign<abs_dir_path>,
@@ -716,6 +724,7 @@ namespace build2
     type_name,
     sizeof (name),
     nullptr,                    // No base.
+    nullptr,                    // No element.
     &default_dtor<name>,
     &default_copy_ctor<name>,
     &default_copy_assign<name>,
@@ -794,6 +803,7 @@ namespace build2
     type_name,
     sizeof (name_pair),
     nullptr,                         // No base.
+    nullptr,                         // No element.
     &default_dtor<name_pair>,
     &default_copy_ctor<name_pair>,
     &default_copy_assign<name_pair>,
@@ -930,6 +940,7 @@ namespace build2
     type_name,
     sizeof (process_path),
     nullptr,                         // No base.
+    nullptr,                         // No element.
     &default_dtor<process_path>,
     &process_path_copy_ctor,
     &process_path_copy_assign,
@@ -975,6 +986,7 @@ namespace build2
     type_name,
     sizeof (target_triplet),
     nullptr,                         // No base.
+    nullptr,                         // No element.
     &default_dtor<target_triplet>,
     &default_copy_ctor<target_triplet>,
     &default_copy_assign<target_triplet>,
diff --git a/build2/variable.hxx b/build2/variable.hxx
index 9f7a82c..f3eea63 100644
--- a/build2/variable.hxx
+++ b/build2/variable.hxx
@@ -35,7 +35,11 @@ namespace build2
     // below is expected to return the base pointer if its second argument
     // points to the base's value_type.
     //
-    const value_type* base;
+    const value_type* base_type;
+
+    // Element type, if this is a vector.
+    //
+    const value_type* element_type;
 
     // Destroy the value. If it is NULL, then the type is assumed to be POD
     // with a trivial destructor.
diff --git a/build2/variable.ixx b/build2/variable.ixx
index 04cf6dc..6126a4e 100644
--- a/build2/variable.ixx
+++ b/build2/variable.ixx
@@ -132,7 +132,9 @@ namespace build2
     // Find base if any.
     //
     const value_type* b (v.type);
-    for (; b != nullptr && b != &value_traits<T>::value_type; b = b->base) ;
+    for (;
+         b != nullptr && b != &value_traits<T>::value_type;
+         b = b->base_type) ;
     assert (b != nullptr);
 
     return *static_cast<const T*> (v.type->cast == nullptr
diff --git a/build2/variable.txx b/build2/variable.txx
index f75ffd6..aebbe1a 100644
--- a/build2/variable.txx
+++ b/build2/variable.txx
@@ -395,6 +395,7 @@ namespace build2
     nullptr,                          // Patched above.
     sizeof (vector<T>),
     nullptr,                          // No base.
+    &value_traits<T>::value_type,
     &default_dtor<vector<T>>,
     &default_copy_ctor<vector<T>>,
     &default_copy_assign<vector<T>>,
@@ -563,6 +564,7 @@ namespace build2
     nullptr,             // Patched above.
     sizeof (map<K, V>),
     nullptr,             // No base.
+    nullptr,             // No element.
     &default_dtor<map<K, V>>,
     &default_copy_ctor<map<K, V>>,
     &default_copy_assign<map<K, V>>,
diff --git a/tests/loop/buildfile b/tests/loop/buildfile
new file mode 100644
index 0000000..7517da5
--- /dev/null
+++ b/tests/loop/buildfile
@@ -0,0 +1,5 @@
+# file      : tests/loop/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: test{*} $b
diff --git a/tests/loop/for.test b/tests/loop/for.test
new file mode 100644
index 0000000..746f1bb
--- /dev/null
+++ b/tests/loop/for.test
@@ -0,0 +1,112 @@
+# file      : tests/loop/for.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Test for-loop.
+
+.include ../common.test
+
+: line
+:
+$* <<EOI >>EOO
+for i: 1 2 3
+  print $i
+EOI
+1
+2
+3
+EOO
+
+: block
+:
+$* <<EOI >>EOO
+for i: 1 2 3
+{
+  # This is a block if you haven't noticed.
+  j = $i
+  print $j
+}
+EOI
+1
+2
+3
+EOO
+
+: empty
+:
+$* <<EOI
+i = x
+nums =
+for i: $nums
+  print $i
+assert ($i == x)
+EOI
+
+: nested
+:
+$* <<EOI >>EOO
+for i: 1 2 3
+{
+  for j: + -
+  {
+    print $j$i
+  }
+}
+EOI
++1
+-1
++2
+-2
++3
+-3
+EOO
+
+: diag-line
+:
+$* <<EOI 2>>EOE != 0
+for i: true false
+{
+  assert $i
+}
+EOI
+<stdin>:3:3: error: assertion failed
+EOE
+
+: var-attribute
+:
+$* <<EOI >>EOO
+for [uint64] i: 0 1 2
+{
+  i += 1
+  print $i
+}
+EOI
+1
+2
+3
+EOO
+
+: val-attribute
+:
+$* <<EOI >>EOO
+for i: [uint64s] 0 1 2
+{
+  i += 1
+  print $i
+}
+EOI
+1
+2
+3
+EOO
+
+: pairs
+:
+$* <<EOI >>EOO
+for i: a@1 b@2 c@3
+  print $i
+EOI
+a@1
+b@2
+c@3
+EOO