Add putback depth char_scanner template parameter

3 files changed, 42 insertions, 42 deletions

diff --git a/libbutl/char-scanner.ixx b/libbutl/char-scanner.ixx
index 7e9c4b0..57aefc2 100644
--- a/libbutl/char-scanner.ixx
+++ b/libbutl/char-scanner.ixx
@@ -3,36 +3,33 @@
 
 namespace butl
 {
-  template <typename V>
-  inline char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline char_scanner<V, N>::
   char_scanner (std::istream& is, bool crlf, std::uint64_t l, std::uint64_t p)
       : char_scanner (is, validator_type (), crlf, l, p)
   {
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   peek (std::string& what) -> xchar
   {
     return peek (&what);
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   peek () -> xchar
   {
     return peek (nullptr /* what */);
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   get (std::string* what) -> xchar
   {
-    if (unget_)
-    {
-      unget_ = false;
-      return ungetc_;
-    }
+    if (ungetn_ != 0)
+      return ungetb_[--ungetn_];
     else
     {
       xchar c (peek (what));
@@ -41,33 +38,34 @@ namespace butl
     }
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   get (std::string& what) -> xchar
   {
     return get (&what);
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   get () -> xchar
   {
     return get (nullptr /* what */);
   }
 
-  template <typename V>
-  inline void char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline void char_scanner<V, N>::
   unget (const xchar& c)
   {
     // Because iostream::unget cannot work once eos is reached, we have to
     // provide our own implementation.
     //
-    unget_ = true;
-    ungetc_ = c;
+    assert (ungetn_ != N); // Make sure the buffer is not filled.
+
+    ungetb_[ungetn_++] = c;
   }
 
-  template <typename V>
-  inline auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline auto char_scanner<V, N>::
   peek_ () -> int_type
   {
     if (gptr_ != egptr_)
@@ -86,8 +84,8 @@ namespace butl
     return r;
   }
 
-  template <typename V>
-  inline void char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline void char_scanner<V, N>::
   get_ ()
   {
     int_type c;
@@ -106,8 +104,8 @@ namespace butl
       save_->push_back (static_cast<char_type> (c));
   }
 
-  template <typename V>
-  inline std::uint64_t char_scanner<V>::
+  template <typename V, std::size_t N>
+  inline std::uint64_t char_scanner<V, N>::
   pos_ () const
   {
     return buf_ != nullptr ? buf_->tellg () : 0;
diff --git a/libbutl/char-scanner.mxx b/libbutl/char-scanner.mxx
index e57245b..60994cf 100644
--- a/libbutl/char-scanner.mxx
+++ b/libbutl/char-scanner.mxx
@@ -9,6 +9,7 @@
 
 #ifndef __cpp_lib_modules_ts
 #include <string>  // char_traits
+#include <cstddef> // size_t
 #include <cstdint> // uint64_t
 #include <climits> // INT_*
 #include <utility> // pair, make_pair()
@@ -46,11 +47,12 @@ LIBBUTL_MODEXPORT namespace butl
   // Low-level character stream scanner. Normally used as a base for
   // higher-level lexers.
   //
-  template <typename V = noop_validator>
+  template <typename V = noop_validator, std::size_t N = 1>
   class char_scanner
   {
   public:
     using validator_type = V;
+    static constexpr const std::size_t unget_depth = N;
 
     // If the crlf argument is true, then recognize Windows newlines (0x0D
     // 0x0A) and convert them to just '\n' (0x0A). Note that a standalone
@@ -119,7 +121,7 @@ LIBBUTL_MODEXPORT namespace butl
           : char_type (0);
       }
 
-      xchar (int_type v,
+      xchar (int_type v = 0,
              std::uint64_t l = 0,
              std::uint64_t c = 0,
              std::uint64_t p = 0)
@@ -247,10 +249,10 @@ LIBBUTL_MODEXPORT namespace butl
     bool crlf_;
     bool eos_ = false;
 
-    bool unget_ = false;
-    bool unpeek_ = false;
+    std::size_t ungetn_ = 0;
+    xchar ungetb_[N];
 
-    xchar ungetc_ = '\0';
+    bool unpeek_ = false;
     xchar unpeekc_ = '\0';
   };
 }
diff --git a/libbutl/char-scanner.txx b/libbutl/char-scanner.txx
index d4e2082..35edf42 100644
--- a/libbutl/char-scanner.txx
+++ b/libbutl/char-scanner.txx
@@ -7,8 +7,8 @@
 
 namespace butl
 {
-  template <typename V>
-  char_scanner<V>::
+  template <typename V, std::size_t N>
+  char_scanner<V, N>::
   char_scanner (std::istream& is,
                 validator_type v,
                 bool crlf,
@@ -26,12 +26,12 @@ namespace butl
   {
   }
 
-  template <typename V>
-  auto char_scanner<V>::
+  template <typename V, std::size_t N>
+  auto char_scanner<V, N>::
   peek (std::string* what) -> xchar
   {
-    if (unget_)
-      return ungetc_;
+    if (ungetn_ > 0)
+      return ungetb_[ungetn_ - 1];
 
     if (unpeek_)
       return unpeekc_;
@@ -108,12 +108,12 @@ namespace butl
     return xchar (v, line, column, position);
   }
 
-  template <typename V>
-  void char_scanner<V>::
+  template <typename V, std::size_t N>
+  void char_scanner<V, N>::
   get (const xchar& c)
   {
-    if (unget_)
-      unget_ = false;
+    if (ungetn_ > 0)
+      --ungetn_;
     else
     {
       if (unpeek_)