Various testscript lexer/parser fixes (testscript)

5 files changed, 162 insertions, 89 deletions

diff --git a/build2/test/script/lexer b/build2/test/script/lexer
index 5d77ab9..d79ef78 100644
--- a/build2/test/script/lexer
+++ b/build2/test/script/lexer
@@ -25,8 +25,8 @@ namespace build2
         enum
         {
           script_line = base_type::value_next,
+          assign_line,   // Auto-expires at the end of the token.
           variable_line, // Auto-expires at the end of the line.
-          test_line,
           command_line,
           here_line
         };
@@ -46,14 +46,13 @@ namespace build2
         virtual void
         mode (base_mode, char = '\0') override;
 
-        // Return true if we entered the quoted (double or single) mode since
-        // last reset.
+        // Number of quoted (double or single) tokens since last reset.
         //
-        bool
+        size_t
         quoted () const {return quoted_;}
 
         void
-        reset_quoted (bool q) {quoted_ = q;}
+        reset_quoted (size_t q) {quoted_ = q;}
 
       protected:
         virtual token
@@ -63,10 +62,10 @@ namespace build2
         next_line ();
 
         virtual token
-        word (bool) override;
+        word (state, bool) override;
 
       protected:
-        bool quoted_ = false;
+        size_t quoted_;
       };
     }
   }
diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index f752f39..f75ad4a 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -25,6 +25,14 @@ namespace build2
         {
         case lexer_mode::script_line:
           {
+            s1 = "=!|&<> $(#\t\n";
+            s2 = "==          ";
+            break;
+          }
+        case lexer_mode::assign_line:
+          {
+            // As script_line but with variable assignments.
+            //
             s1 = "=+!|&<> $(#\t\n";
             s2 = " ==          ";
             break;
@@ -37,14 +45,7 @@ namespace build2
             s2 = "        ";
             break;
           }
-        case lexer_mode::test_line:
-          {
-            // As script_line but without variable assignments.
-            //
-            s1 = "=!|&<> $(#\t\n";
-            s2 = "==          ";
-            break;
-          }
+
         case lexer_mode::command_line:
           {
             // Note that whitespaces are not word separators in this mode.
@@ -64,10 +65,6 @@ namespace build2
             s = false;
             break;
           }
-        case lexer_mode::single_quoted:
-        case lexer_mode::double_quoted:
-          quoted_ = true;
-          // Fall through.
         default:
           {
             // Disable pair separator.
@@ -83,15 +80,22 @@ namespace build2
       token lexer::
       next_impl ()
       {
+        token r;
+
         switch (state_.top ().mode)
         {
         case lexer_mode::script_line:
+        case lexer_mode::assign_line:
         case lexer_mode::variable_line:
-        case lexer_mode::test_line:
         case lexer_mode::command_line:
-        case lexer_mode::here_line:      return next_line ();
-        default:                         return base_lexer::next_impl ();
+        case lexer_mode::here_line:     r = next_line ();             break;
+        default:                        r = base_lexer::next_impl (); break;
         }
+
+        if (r.quoted)
+          ++quoted_;
+
+        return r;
       }
 
       token lexer::
@@ -110,7 +114,14 @@ namespace build2
         if (eos (c))
           return make_token (type::eos);
 
-        lexer_mode m (state_.top ().mode);
+        state st (state_.top ()); // Make copy (see assign_line).
+        lexer_mode m (st.mode);
+
+        // Expire the assign mode at the end of the token. Do it early in case
+        // we push any new mode (e.g., double quote).
+        //
+        if (m == lexer_mode::assign_line)
+          state_.pop ();
 
         // NOTE: remember to update mode() if adding new special characters.
 
@@ -148,7 +159,7 @@ namespace build2
 
         // Command line operator/separators.
         //
-        if (m == lexer_mode::script_line || m == lexer_mode::test_line)
+        if (m == lexer_mode::script_line || m == lexer_mode::assign_line)
         {
           switch (c)
           {
@@ -169,7 +180,7 @@ namespace build2
         // Command operators/separators.
         //
         if (m == lexer_mode::script_line ||
-            m == lexer_mode::test_line   ||
+            m == lexer_mode::assign_line ||
             m == lexer_mode::command_line)
         {
           switch (c)
@@ -234,7 +245,7 @@ namespace build2
 
         // Variable assignment (=, +=, =+).
         //
-        if (m == lexer_mode::script_line)
+        if (m == lexer_mode::assign_line)
         {
           switch (c)
           {
@@ -262,22 +273,24 @@ namespace build2
         // Otherwise it is a word.
         //
         unget (c);
-        return word (sep);
+        return word (st, sep);
       }
 
       token lexer::
-      word (bool sep)
+      word (state st, bool sep)
       {
+        lexer_mode m (st.mode);
+
         // Customized implementation that handles special variable names ($*,
         // $~, $NNN).
         //
-        if (state_.top ().mode != lexer_mode::variable)
-          return base_lexer::word (sep);
+        if (m != lexer_mode::variable)
+          return base_lexer::word (st, sep);
 
         xchar c (peek ());
 
         if (c != '*' && c != '~' && !digit (c))
-          return base_lexer::word (sep);
+          return base_lexer::word (st, sep);
 
         uint64_t ln (c.line), cn (c.column);
         string lexeme;
diff --git a/build2/test/script/parser b/build2/test/script/parser
index f9fbd98..adff8a3 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -40,7 +40,7 @@ namespace build2
         //
       protected:
         void
-        parse_script (token&, token_type&);
+        parse_script ();
 
         void
         parse_script_line (token&, token_type&);
@@ -63,6 +63,18 @@ namespace build2
         virtual lookup
         lookup_variable (name&&, string&&, const location&) override;
 
+        // Number of quoted tokens since last reset. Note that this includes
+        // the peeked token, if any.
+        //
+      protected:
+        size_t
+        quoted () const;
+
+        void
+        reset_quoted (token& current);
+
+        size_t replay_quoted_;
+
       protected:
         using base_parser = build2::parser;
 
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index bfd0667..74bacee 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -30,22 +30,28 @@ namespace build2
         runner_ = &r;
         scope_ = script_;
 
-        token t;
-        type tt;
-        next (t, tt);
-
-        parse_script (t, tt);
-
-        if (tt != type::eos)
-          fail (t) << "unexpected " << t;
+        parse_script ();
       }
 
       void parser::
-      parse_script (token& t, token_type& tt)
+      parse_script ()
       {
-        for (; tt != type::eos; next (t, tt))
+        token t;
+        type tt;
+
+        for (;;)
         {
+          // We need to start lexing each line in the assign mode in order to
+          // recognize assignment operators as separators.
+          //
+          mode (lexer_mode::assign_line);
+          next (t, tt);
+
+          if (tt == type::eos)
+            break;
+
           parse_script_line (t, tt);
+          assert (tt == type::newline);
         }
       }
 
@@ -53,19 +59,27 @@ namespace build2
       parse_script_line (token& t, token_type& tt)
       {
         // Decide whether this is a variable assignment or a command. It is a
-        // variable assignment if the first token is a word and the next is an
-        // assign/append/prepend operator. Assignment to a computed variable
-        // name must use the set builtin.
+        // variable assignment if the first token is an unquoted word and the
+        // next is an assign/append/prepend operator. Assignment to a computed
+        // variable name must use the set builtin.
         //
-        auto assign = [] (type t)
+        if (tt == type::word && !t.quoted)
         {
-          return t == type::assign || t == type::prepend || t == type::append;
-        };
+          // Switch recognition of variable assignments for one more token.
+          // This is safe to do because we know we cannot be in the quoted
+          // mode (since the current token is not quoted).
+          //
+          mode (lexer_mode::assign_line);
+          type p (peek ());
 
-        if (tt == type::word && assign (peek ()))
-          parse_variable_line (t, tt);
-        else
-          parse_test_line (t, tt);
+          if (p == type::assign || p == type::prepend || p == type::append)
+          {
+            parse_variable_line (t, tt);
+            return;
+          }
+        }
+
+        parse_test_line (t, tt);
       }
 
       // Return true if the string contains only digit characters (used to
@@ -84,16 +98,16 @@ namespace build2
       void parser::
       parse_variable_line (token& t, token_type& tt)
       {
-        location nl (get_location (t));
         string name (move (t.value));
 
         // Check if we are trying to modify any of the special aliases ($*,
         // $~, $N).
         //
-        if (name == "*" || name == "~" || digits (name))
-          fail (nl) << "attempt to set '" << name << "' variable directly";
-
-        const variable& var (script_->var_pool.insert (move (name)));
+        if (pre_parse_)
+        {
+          if (name == "*" || name == "~" || digits (name))
+            fail (t) << "attempt to set '" << name << "' variable directly";
+        }
 
         next (t, tt);
         type kind (tt); // Assignment kind.
@@ -106,39 +120,40 @@ namespace build2
         if (tt != type::newline)
           fail (t) << "unexpected " << t;
 
-        value& lhs (kind == type::assign
-                    ? scope_->assign (var)
-                    : scope_->append (var));
+        if (!pre_parse_)
+        {
+          const variable& var (script_->var_pool.insert (move (name)));
 
-        // @@ Need to adjust to make strings the default type.
-        //
-        apply_value_attributes (&var, lhs, move (rhs), kind);
+          value& lhs (kind == type::assign
+                      ? scope_->assign (var)
+                      : scope_->append (var));
 
-        // Handle the $*, $NN special aliases.
-        //
-        // The plan is as follows: in this function we detect modification of
-        // the source variables (test*), and (re)set $* to NULL on this scope
-        // (this is important to both invalidate any old values but also to
-        // "stake" the lookup position). This signals to the variable lookup
-        // function below that the $* and $NN values need to be recalculated
-        // from their sources. Note that we don't need to invalidate $NN since
-        // their lookup always checks $* first.
-        //
-        if (var.name == script_->test_var.name ||
-            var.name == script_->opts_var.name ||
-            var.name == script_->args_var.name)
-        {
-          scope_->assign (script_->cmd_var) = nullptr;
+          // @@ Need to adjust to make strings the default type.
+          //
+          apply_value_attributes (&var, lhs, move (rhs), kind);
+
+          // Handle the $*, $NN special aliases.
+          //
+          // The plan is as follows: in this function we detect modification
+          // of the source variables (test*), and (re)set $* to NULL on this
+          // scope (this is important to both invalidate any old values but
+          // also to "stake" the lookup position). This signals to the
+          // variable lookup function below that the $* and $NN values need to
+          // be recalculated from their sources. Note that we don't need to
+          // invalidate $NN since their lookup always checks $* first.
+          //
+          if (var.name == script_->test_var.name ||
+              var.name == script_->opts_var.name ||
+              var.name == script_->args_var.name)
+          {
+            scope_->assign (script_->cmd_var) = nullptr;
+          }
         }
       }
 
       void parser::
       parse_test_line (token& t, token_type& tt)
       {
-        // Stop recognizing variable assignments.
-        //
-        mode (lexer_mode::test_line);
-
         test ts;
 
         // Pending positions where the next word should go.
@@ -371,12 +386,14 @@ namespace build2
               // Note that we do it in the chunking mode to detect whether
               // anything in each chunk is quoted.
               //
-              lexer_->reset_quoted (t.quoted);
+              reset_quoted (t);
               parse_names (t, tt, ns, true, "command");
 
-              // Process what we got.
+              // Process what we got. Determine whether anything inside was
+              // quoted (note that the current token is not part of it).
               //
-              bool q (lexer_->quoted ());
+              bool q ((quoted () - (t.quoted ? 1 : 0)) != 0);
+
               for (name& n: ns)
               {
                 string s;
@@ -505,7 +522,7 @@ namespace build2
 
         // While we no longer need to recognize command line operators, we
         // also don't expect a valid test trailer to contain them. So we are
-        // going to continue lexing in the test_line mode.
+        // going to continue lexing in the script_line mode.
         //
         if (tt == type::equal || tt == type::not_equal)
         {
@@ -516,8 +533,6 @@ namespace build2
         if (tt != type::newline)
           fail (t) << "unexpected " << t;
 
-        expire_mode (); // Done parsing test-line.
-
         // Parse here-document fragments in the order they were mentioned on
         // the command line.
         //
@@ -702,6 +717,40 @@ namespace build2
 
         return lookup (nv, vars);
       }
+
+      size_t parser::
+      quoted () const
+      {
+        size_t r (0);
+
+        if (replay_ != replay::play)
+          r = lexer_->quoted ();
+        else
+        {
+          // Examine tokens we have replayed since last reset.
+          //
+          for (size_t i (replay_quoted_); i != replay_i_; ++i)
+            if (replay_data_[i].token.quoted)
+              ++r;
+        }
+
+        return r;
+      }
+
+      void parser::
+      reset_quoted (token& cur)
+      {
+        if (replay_ != replay::play)
+          lexer_->reset_quoted (cur.quoted ? 1 : 0);
+        else
+        {
+          replay_quoted_ = replay_i_ - 1;
+
+          // Must be the same token.
+          //
+          assert (replay_data_[replay_quoted_].token.quoted == cur.quoted);
+        }
+      }
     }
   }
 }
diff --git a/unit-tests/test/script/lexer/driver.cxx b/unit-tests/test/script/lexer/driver.cxx
index 7e237ed..b80eed9 100644
--- a/unit-tests/test/script/lexer/driver.cxx
+++ b/unit-tests/test/script/lexer/driver.cxx
@@ -30,8 +30,8 @@ namespace build2
           string s (argv[1]);
 
           if      (s == "script-line")   m = lexer_mode::script_line;
+          else if (s == "assign-line")   m = lexer_mode::assign_line;
           else if (s == "variable-line") m = lexer_mode::variable_line;
-          else if (s == "test-line")     m = lexer_mode::test_line;
           else if (s == "command-line")  m = lexer_mode::command_line;
           else if (s == "here-line")     m = lexer_mode::here_line;
           else if (s == "variable")      m = lexer_mode::variable;