Handle printing of extended tokens (testscript)

4 files changed, 74 insertions, 23 deletions

diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index e0a3272..3367d52 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -101,8 +101,13 @@ namespace build2
         xchar c (get ());
         uint64_t ln (c.line), cn (c.column);
 
+        auto make_token = [sep, ln, cn] (type t)
+        {
+          return token (t, sep, ln, cn, token_printer);
+        };
+
         if (eos (c))
-          return token (type::eos, sep, ln, cn);
+          return make_token (type::eos);
 
         lexer_mode m (state_.top ().mode);
 
@@ -114,13 +119,13 @@ namespace build2
           {
           case '\n':
             {
-              return token (type::newline, sep, ln, cn);
+              return make_token (type::newline);
             }
 
             // Variable expansion, function call, and evaluation context.
             //
-          case '$': return token (type::dollar, sep, ln, cn);
-          case '(': return token (type::lparen, sep, ln, cn);
+          case '$': return make_token (type::dollar);
+          case '(': return make_token (type::lparen);
           }
         }
 
@@ -130,8 +135,8 @@ namespace build2
           {
             // Attributes.
             //
-          case '[': return token (type::lsbrace, sep, ln, cn);
-          case ']': return token (type::rsbrace, sep, ln, cn);
+          case '[': return make_token (type::lsbrace);
+          case ']': return make_token (type::rsbrace);
           }
         }
 
@@ -149,8 +154,7 @@ namespace build2
               if (peek () == '=')
               {
                 get ();
-                return token (
-                  c == '=' ? type::equal : type::not_equal, sep, ln, cn);
+                return make_token (c == '=' ? type::equal : type::not_equal);
               }
             }
           }
@@ -171,10 +175,10 @@ namespace build2
               if (peek () == '|')
               {
                 get ();
-                return token (type::log_or, sep, ln, cn);
+                return make_token (type::log_or);
               }
               else
-                return token (type::pipe, sep, ln, cn);
+                return make_token (type::pipe);
             }
             // &, &&
             //
@@ -183,10 +187,10 @@ namespace build2
               if (peek () == '&')
               {
                 get ();
-                return token (type::log_and, sep, ln, cn);
+                return make_token (type::log_and);
               }
               else
-                return token (type::clean, sep, ln, cn);
+                return make_token (type::clean);
             }
             // <
             //
@@ -197,11 +201,11 @@ namespace build2
               if (p == '!' || p == '<')
               {
                 get ();
-                return token (
-                  p == '!' ? type::in_null : type::in_document, sep, ln, cn);
+                return make_token (
+                  p == '!' ? type::in_null : type::in_document);
               }
               else
-                return token (type::in_string, sep, ln, cn);
+                return make_token (type::in_string);
 
             }
             // >
@@ -213,11 +217,11 @@ namespace build2
               if (p == '!' || p == '>')
               {
                 get ();
-                return token (
-                  p == '!' ? type::out_null : type::out_document, sep, ln, cn);
+                return make_token (
+                  p == '!' ? type::out_null : type::out_document);
               }
               else
-                return token (type::out_string, sep, ln, cn);
+                return make_token (type::out_string);
             }
           }
         }
@@ -233,17 +237,17 @@ namespace build2
               if (peek () == '+')
               {
                 get ();
-                return token (type::prepend, sep, ln, cn);
+                return make_token (type::prepend);
               }
               else
-                return token (type::assign, sep, ln, cn);
+                return make_token (type::assign);
             }
           case '+':
             {
               if (peek () == '=')
               {
                 get ();
-                return token (type::append, sep, ln, cn);
+                return make_token (type::append);
               }
             }
           }
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index b99ab09..601cbd4 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -35,7 +35,7 @@ namespace build2
 
         runner_ = &r;
 
-        token t (type::eos, false, 0, 0);
+        token t;
         type tt;
         next (t, tt);
 
diff --git a/build2/test/script/token b/build2/test/script/token
index 51bf282..ac035f7 100644
--- a/build2/test/script/token
+++ b/build2/test/script/token
@@ -22,6 +22,8 @@ namespace build2
 
         enum
         {
+          // NOTE: remember to update token_printer()!
+
           pipe = base_type::value_next, // |
           clean,                        // &
           log_and,                      // &&
@@ -33,11 +35,14 @@ namespace build2
 
           out_null,                     // <!
           out_string,                   // <
-          out_document,                 // <<
+          out_document                  // <<
         };
 
         using base_type::base_type;
       };
+
+      void
+      token_printer (ostream&, const token&, bool);
     }
   }
 }
diff --git a/build2/test/script/token.cxx b/build2/test/script/token.cxx
new file mode 100644
index 0000000..bfff25a
--- /dev/null
+++ b/build2/test/script/token.cxx
@@ -0,0 +1,42 @@
+// file      : build2/test/script/token.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build2/test/script/token>
+
+using namespace std;
+
+namespace build2
+{
+  namespace test
+  {
+    namespace script
+    {
+      void
+      token_printer (ostream& os, const token& t, bool d)
+      {
+        // Only quote non-name tokens for diagnostics.
+        //
+        const char* q (d ? "'" : "");
+
+        switch (t.type)
+        {
+        case token_type::pipe:         os << q << '|' << q; break;
+        case token_type::clean:        os << q << '&' << q; break;
+        case token_type::log_and:      os << q << "&&" << q; break;
+        case token_type::log_or:       os << q << "||" << q; break;
+
+        case token_type::in_null:      os << q << "<!" << q; break;
+        case token_type::in_string:    os << q << '<' << q; break;
+        case token_type::in_document:  os << q << "<<" << q; break;
+
+        case token_type::out_null:     os << q << "<!" << q; break;
+        case token_type::out_string:   os << q << '<' << q; break;
+        case token_type::out_document: os << q << "<<" << q; break;
+
+        default: build2::token_printer (os, t, d);
+        }
+      }
+    }
+  }
+}