Keep track of lexer mode in parser replay mechanism

3 files changed, 97 insertions, 24 deletions

diff --git a/build2/parser b/build2/parser
index 9142cf6..89f42b8 100644
--- a/build2/parser
+++ b/build2/parser
@@ -281,7 +281,7 @@ namespace build2
     peeked () const
     {
       assert (peeked_);
-      return peek_;
+      return peek_.token;
     }
 
     void
@@ -289,13 +289,37 @@ namespace build2
     {
       if (replay_ != replay::play)
         lexer_->mode (m, ps);
+      else
+        // As a sanity check, make sure the mode matches the next token. Note
+        // that we don't check the pair separator since it can be overriden by
+        // the lexer's mode() implementation.
+        //
+        assert (replay_i_ != replay_data_.size () &&
+                replay_data_[replay_i_].mode == m);
     }
 
     lexer_mode
     mode () const
     {
-      assert (replay_ != replay::play);
-      return lexer_->mode ();
+      if (replay_ != replay::play)
+        return lexer_->mode ();
+      else
+      {
+        assert (replay_i_ != replay_data_.size ());
+        return replay_data_[replay_i_].mode;
+      }
+    }
+
+    char
+    pair_separator () const
+    {
+      if (replay_ != replay::play)
+        return lexer_->pair_separator ();
+      else
+      {
+        assert (replay_i_ != replay_data_.size ());
+        return replay_data_[replay_i_].pair_separator;
+      }
     }
 
     void
@@ -306,9 +330,9 @@ namespace build2
     }
 
     // Token saving and replaying. Note that it can only be used in certain
-    // contexts. Specifically, the lexer mode should be the same and the code
-    // that parses a replay must not interact with the lexer directly (e.g.,
-    // the keyword() test). For now we don't enforce any of this.
+    // contexts. Specifically, the code that parses a replay must not interact
+    // with the lexer directly (e.g., the keyword() test). For now we don't
+    // enforce any of this.
     //
     // Note also that the peeked token is not part of the replay, until it
     // is "got".
@@ -338,13 +362,6 @@ namespace build2
       replay_ = replay::stop;
     }
 
-    const token&
-    replay_next ()
-    {
-      assert (replay_i_ != replay_data_.size ());
-      return replay_data_[replay_i_++];
-    }
-
     struct replay_guard
     {
       replay_guard (parser& p, bool start = true)
@@ -371,6 +388,57 @@ namespace build2
       parser* p_;
     };
 
+    struct replay_token
+    {
+      build2::token token;
+      lexer_mode mode;
+      char pair_separator;
+    };
+
+    using replay_tokens = vector<replay_token>;
+
+    // Stop saving and get the data.
+    //
+    replay_tokens
+    replay_data ()
+    {
+      assert (replay_ == replay::save);
+
+      replay_tokens r (move (replay_data_));
+      replay_ = replay::stop;
+      replay_data_.clear ();
+
+      return r;
+    }
+
+    // Set the data and start playing.
+    //
+    void
+    replay_data (replay_tokens&& d)
+    {
+      assert (replay_ == replay::stop);
+
+      replay_data_ = move (d);
+      replay_ = replay::play;
+    }
+
+    // Implementation details, don't call directly.
+    //
+    replay_token
+    lexer_next ()
+    {
+      lexer_mode m (lexer_->mode ()); // Get it first since it may expire.
+      char ps (lexer_->pair_separator ());
+      return replay_token {lexer_->next (), m, ps};
+    }
+
+    const replay_token&
+    replay_next ()
+    {
+      assert (replay_i_ != replay_data_.size ());
+      return replay_data_[replay_i_++];
+    }
+
     // Diagnostics.
     //
   protected:
@@ -390,12 +458,12 @@ namespace build2
     target* default_target_;
     names export_value_;
 
-    token peek_ = token (token_type::eos, false, 0, 0, token_printer);
+    replay_token peek_;
     bool peeked_ = false;
 
     enum class replay {stop, save, play} replay_ = replay::stop;
-    vector<token> replay_data_;
-    size_t replay_i_; // Position of the next token during replay.
+    replay_tokens replay_data_;
+    size_t replay_i_;         // Position of the next token during replay.
   };
 }
 
diff --git a/build2/parser.cxx b/build2/parser.cxx
index 24558ea..a81aec7 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -1865,8 +1865,8 @@ namespace build2
     // Manually expire the value mode if we haven't reached newline/eos (where
     // it expires automatically).
     //
-    if (lexer_->mode () == lexer_mode::value)
-      lexer_->expire_mode ();
+    if (mode () == lexer_mode::value)
+      expire_mode ();
 
     if (tt != type::rsbrace)
       fail (t) << "expected ']' instead of " << t;
@@ -2540,7 +2540,7 @@ namespace build2
           fail (t) << "multiple " << what << "s on the left hand side "
                    << "of a pair";
 
-        ns.back ().pair = lexer_->pair_separator ();
+        ns.back ().pair = pair_separator ();
         tt = peek ();
 
         // If the next token is separated, then we have an empty RHS. Note
@@ -3063,17 +3063,20 @@ namespace build2
   type parser::
   next (token& t, type& tt)
   {
+    replay_token r;
+
     if (peeked_)
     {
-      t = move (peek_);
+      r = move (peek_);
       peeked_ = false;
     }
     else
-      t = (replay_ == replay::play ? replay_next () : lexer_->next ());
+      r = replay_ != replay::play ? lexer_next () : replay_next ();
 
     if (replay_ == replay::save)
-      replay_data_.push_back (t);
+      replay_data_.push_back (r);
 
+    t = move (r.token);
     tt = t.type;
     return tt;
   }
@@ -3083,10 +3086,10 @@ namespace build2
   {
     if (!peeked_)
     {
-      peek_ = (replay_ == replay::play ? replay_next () : lexer_->next ());
+      peek_ = (replay_ != replay::play ? lexer_next () : replay_next ());
       peeked_ = true;
     }
 
-    return peek_.type;
+    return peek_.token.type;
   }
 }
diff --git a/build2/token b/build2/token
index d172e0d..517bf47 100644
--- a/build2/token
+++ b/build2/token
@@ -92,6 +92,8 @@ namespace build2
           printer (&token_printer) {}
   };
 
+  using tokens = vector<token>;
+
   // Output the token value in a format suitable for diagnostics.
   //
   inline ostream&