Add support for shared here-documents

7 files changed, 269 insertions, 49 deletions

diff --git a/build2/test/script/parser b/build2/test/script/parser
index edd64a3..fce372e 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -95,11 +95,20 @@ namespace build2
         // Ordered sequence of here-document redirects that we can expect to
         // see after the command line.
         //
+        struct here_redirect
+        {
+          size_t expr; // Index in command_expr.
+          size_t pipe; // Index in command_pipe.
+          int fd;      // Redirect fd (0 - in, 1 - out, 2 - err).
+        };
+
         struct here_doc
         {
-          size_t expr;      // Index in command_expr.
-          size_t pipe;      // Index in command_pipe.
-          int fd;           // Redirect fd (0 - in, 1 - out, 2 - err).
+          // Redirects that share here_doc. Most of the time we will have no
+          // more than 2 (2 - for the roundtrip test cases).
+          //
+          small_vector<here_redirect, 2> redirects;
+
           string end;
           bool literal;     // Literal (single-quote).
           string modifiers;
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index c3dc233..0b00861 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -1810,6 +1810,8 @@ namespace build2
                  : redirect_fmode::compare);
 
             break;
+
+          case redirect_type::here_doc_ref: assert (false); break;
           }
         };
 
@@ -1967,13 +1969,45 @@ namespace build2
                     end = move (r.value); // The "cleared" end marker.
                   }
 
-                  hd.push_back (
-                    here_doc {
-                      0, 0, 0,
-                      move (end),
-                      qt == quote_type::single,
-                      move (mod),
-                      r.intro, move (r.flags)});
+                  bool literal (qt == quote_type::single);
+                  bool shared (false);
+
+                  for (const auto& d: hd)
+                  {
+                    if (d.end == end)
+                    {
+                      auto check = [&t, &end, &re, this] (bool c,
+                                                          const char* what)
+                      {
+                        if (!c)
+                          fail (t) << "different " << what
+                                   << " for shared here-document "
+                                   << (re ? "regex '" : "'") << end << "'";
+                      };
+
+                      check (d.modifiers == mod, "modifiers");
+                      check (d.literal == literal, "quoting");
+
+                      if (re)
+                      {
+                        check (d.regex == r.intro, "introducers");
+                        check (d.regex_flags == r.flags, "global flags");
+                      }
+
+                      shared = true;
+                      break;
+                    }
+                  }
+
+                  if (!shared)
+                    hd.push_back (
+                      here_doc {
+                        {},
+                        move (end),
+                        literal,
+                        move (mod),
+                        r.intro, move (r.flags)});
+
                   break;
                 }
 
@@ -2067,7 +2101,11 @@ namespace build2
 
                 if (fd != -1)
                 {
+                  here_redirect rd {
+                    expr.size () - 1, expr.back ().pipe.size (), fd};
+
                   string end (move (t.value));
+
                   regex_parts r;
 
                   if (p == pending::out_doc_regex ||
@@ -2081,16 +2119,30 @@ namespace build2
                     end = move (r.value); // The "cleared" end marker.
                   }
 
-                  hd.push_back (
-                    here_doc {
-                      expr.size () - 1,
-                      expr.back ().pipe.size (),
-                      fd,
-                      move (end),
-                      (t.qtype == quote_type::unquoted ||
-                       t.qtype == quote_type::single),
-                      move (mod),
-                      r.intro, move (r.flags)});
+                  bool shared (false);
+                  for (auto& d: hd)
+                  {
+                    // No need to check that redirects that share here-document
+                    // have the same modifiers, etc. That have been done during
+                    // pre-parsing.
+                    //
+                    if (d.end == end)
+                    {
+                      d.redirects.emplace_back (rd);
+                      shared = true;
+                      break;
+                    }
+                  }
+
+                  if (!shared)
+                    hd.push_back (
+                      here_doc {
+                        {rd},
+                        move (end),
+                        (t.qtype == quote_type::unquoted ||
+                         t.qtype == quote_type::single),
+                        move (mod),
+                        r.intro, move (r.flags)});
 
                   p = pending::none;
                   mod.clear ();
@@ -2394,8 +2446,11 @@ namespace build2
 
           if (!pre_parse_)
           {
-            command& c (p.first[h.expr].pipe[h.pipe]);
-            redirect& r (h.fd == 0 ? c.in : h.fd == 1 ? c.out : c.err);
+            assert (!h.redirects.empty ());
+            auto i (h.redirects.cbegin ());
+
+            command& c (p.first[i->expr].pipe[i->pipe]);
+            redirect& r (i->fd == 0 ? c.in : i->fd == 1 ? c.out : c.err);
 
             if (v.re)
             {
@@ -2408,6 +2463,18 @@ namespace build2
             r.end        = move (h.end);
             r.end_line   = v.end_line;
             r.end_column = v.end_column;
+
+            // Note that our references cannot be invalidated because the
+            // command_expr/command-pipe vectors already contain all their
+            // elements.
+            //
+            for (++i; i != h.redirects.cend (); ++i)
+            {
+              command& c (p.first[i->expr].pipe[i->pipe]);
+
+              (i->fd == 0 ? c.in : i->fd == 1 ? c.out : c.err) =
+                redirect (redirect_type::here_doc_ref, r);
+            }
           }
 
           expire_mode ();
diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx
index 41527cf..4a0c8e7 100644
--- a/build2/test/script/runner.cxx
+++ b/build2/test/script/runner.cxx
@@ -492,8 +492,7 @@ namespace build2
               {
                 try
                 {
-                  string s (
-                    transform (l.value, true, rd.modifiers, *sp.root));
+                  string s (transform (l.value, true, rd.modifiers, *sp.root));
 
                   c = line_char (
                     char_regex (s, gf | parse_flags (l.flags)), pool);
@@ -799,18 +798,19 @@ namespace build2
         //
         path isp;
         auto_fd ifd;
-        int in (0); // @@ TMP
+        int id (0); // @@ TMP
+        const redirect& in (c.in.effective ());
 
         // Open a file for passing to the command stdin.
         //
-        auto open_stdin = [&isp, &ifd, &in, &ll] ()
+        auto open_stdin = [&isp, &ifd, &id, &ll] ()
         {
           assert (!isp.empty ());
 
           try
           {
             ifd = fdopen (isp, fdopen_mode::in);
-            in  = ifd.get ();
+            id  = ifd.get ();
           }
           catch (const io_error& e)
           {
@@ -818,14 +818,14 @@ namespace build2
           }
         };
 
-        switch (c.in.type)
+        switch (in.type)
         {
         case redirect_type::pass:
           {
             try
             {
-              ifd = fddup (in);
-              in  = 0;
+              ifd = fddup (id);
+              id  = 0;
             }
             catch (const io_error& e)
             {
@@ -860,7 +860,7 @@ namespace build2
                 throw io_error (
                   error_code (errno, system_category ()).message ());
 
-              in = -2;
+              id = -2;
             }
             catch (const io_error& e)
             {
@@ -872,7 +872,7 @@ namespace build2
 
         case redirect_type::file:
           {
-            isp = normalize (c.in.file.path, sp, ll);
+            isp = normalize (in.file.path, sp, ll);
 
             open_stdin ();
             break;
@@ -886,8 +886,7 @@ namespace build2
             //
             isp = std_path ("stdin");
 
-            const redirect& r (c.in);
-            save (isp, transform (r.str, false, r.modifiers, *sp.root), ll);
+            save (isp, transform (in.str, false, in.modifiers, *sp.root), ll);
             sp.clean ({cleanup_type::always, isp}, true);
 
             open_stdin ();
@@ -896,7 +895,8 @@ namespace build2
 
         case redirect_type::merge:
         case redirect_type::here_str_regex:
-        case redirect_type::here_doc_regex: assert (false); break;
+        case redirect_type::here_doc_regex:
+        case redirect_type::here_doc_ref:   assert (false); break;
         }
 
         // Dealing with stdout and stderr redirect types other than 'null'
@@ -998,6 +998,8 @@ namespace build2
               m |= fdopen_mode::truncate;
               break;
             }
+
+          case redirect_type::here_doc_ref: assert (false); break;
           }
 
           try
@@ -1017,16 +1019,18 @@ namespace build2
 
         path osp;
         auto_fd ofd;
-        int out (open (c.out, 1, osp, ofd));
+        const redirect& out (c.out.effective ());
+        int od (open (out, 1, osp, ofd));
 
         path esp;
         auto_fd efd;
-        int err (open (c.err, 2, esp, efd));
+        const redirect& err (c.err.effective ());
+        int ed (open (err, 2, esp, efd));
 
         // Merge standard streams.
         //
-        bool mo (c.out.type == redirect_type::merge);
-        if (mo || c.err.type == redirect_type::merge)
+        bool mo (out.type == redirect_type::merge);
+        if (mo || err.type == redirect_type::merge)
         {
           auto_fd& self  (mo ? ofd : efd);
           auto_fd& other (mo ? efd : ofd);
@@ -1084,7 +1088,7 @@ namespace build2
             process pr (sp.wd_path.string ().c_str (),
                         pp,
                         args.data (),
-                        in, out, err);
+                        id, od, ed);
 
             ifd.reset ();
             ofd.reset ();
@@ -1172,8 +1176,8 @@ namespace build2
         // Exit code is correct. Check if the standard outputs match the
         // expectations.
         //
-        check_output (p, osp, isp, c.out, ll, sp, "stdout");
-        check_output (p, esp, isp, c.err, ll, sp, "stderr");
+        check_output (p, osp, isp, out, ll, sp, "stdout");
+        check_output (p, esp, isp, err, ll, sp, "stderr");
       }
 
       bool default_runner::
diff --git a/build2/test/script/script b/build2/test/script/script
index fbf3dd5..90b71bf 100644
--- a/build2/test/script/script
+++ b/build2/test/script/script
@@ -71,10 +71,11 @@ namespace build2
         null,
         merge,
         here_str_literal,
-        here_doc_literal,
         here_str_regex,
+        here_doc_literal,
         here_doc_regex,
-        file
+        here_doc_ref,     // Reference to here_doc literal or regex.
+        file,
       };
 
       // Pre-parsed (but not instantiated) regex lines. The idea here is that
@@ -154,9 +155,10 @@ namespace build2
         union
         {
           int         fd;    // Merge-to descriptor.
-          string      str;   // Note: includes trailing newline, if requested.
-          regex_lines regex; // Note: includes trailing blank, if requested.
+          string      str;   // Note: with trailing newline, if requested.
+          regex_lines regex; // Note: with trailing blank, if requested.
           file_type   file;
+          reference_wrapper<const redirect> ref; // Note: no chains.
         };
 
         string modifiers;   // Redirect modifiers.
@@ -164,15 +166,34 @@ namespace build2
         uint64_t end_line;  // Here-document end marker location.
         uint64_t end_column;
 
+        // Create redirect of a type other than reference.
+        //
         explicit
         redirect (redirect_type = redirect_type::none);
 
+        // Create redirect of the reference type.
+        //
+        redirect (redirect_type t, const redirect& r)
+            : type (redirect_type::here_doc_ref), ref (r)
+        {
+          // There is no support (and need) for reference chains.
+          //
+          assert (t == redirect_type::here_doc_ref &&
+                  r.type != redirect_type::here_doc_ref);
+        }
+
         // Move constuctible/assignable-only type.
         //
         redirect (redirect&&);
         redirect& operator= (redirect&&);
 
         ~redirect ();
+
+        const redirect&
+        effective () const noexcept
+        {
+          return type == redirect_type::here_doc_ref ? ref.get () : *this;
+        }
       };
 
       // cleanup
diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx
index f5cec44..342cae9 100644
--- a/build2/test/script/script.cxx
+++ b/build2/test/script/script.cxx
@@ -153,6 +153,8 @@ namespace build2
               print_path (r.file.path);
               break;
             }
+
+          case redirect_type::here_doc_ref: assert (false); break;
           }
         };
 
@@ -206,9 +208,14 @@ namespace build2
 
           // Redirects.
           //
-          if (c.in.type  != redirect_type::none) print_redirect (c.in,   "<");
-          if (c.out.type != redirect_type::none) print_redirect (c.out,  ">");
-          if (c.err.type != redirect_type::none) print_redirect (c.err, "2>");
+          if (c.in.effective ().type  != redirect_type::none)
+            print_redirect (c.in.effective (), "<");
+
+          if (c.out.effective ().type != redirect_type::none)
+            print_redirect (c.out.effective (),  ">");
+
+          if (c.err.effective ().type != redirect_type::none)
+            print_redirect (c.err.effective (), "2>");
 
           for (const auto& p: c.cleanups)
           {
@@ -322,6 +329,8 @@ namespace build2
           }
 
         case redirect_type::file: new (&file) file_type (); break;
+
+        case redirect_type::here_doc_ref: assert (false); break;
         }
       }
 
@@ -358,6 +367,11 @@ namespace build2
             new (&file) file_type (move (r.file));
             break;
           }
+        case redirect_type::here_doc_ref:
+          {
+            new (&ref) reference_wrapper<const redirect> (r.ref);
+            break;
+          }
         }
       }
 
@@ -378,6 +392,12 @@ namespace build2
         case redirect_type::here_doc_regex: regex.~regex_lines (); break;
 
         case redirect_type::file: file.~file_type (); break;
+
+        case redirect_type::here_doc_ref:
+          {
+            ref.~reference_wrapper<const redirect> ();
+            break;
+          }
         }
       }
 
diff --git a/tests/test/script/runner/redirect.test b/tests/test/script/runner/redirect.test
index f32492a..7cb6316 100644
--- a/tests/test/script/runner/redirect.test
+++ b/tests/test/script/runner/redirect.test
@@ -264,6 +264,16 @@ psr = ($cxx.target.class != 'windows' ? '/' : '\\') # Path separator in regex.
     EOI
     $b
 
+    : shared
+    :
+    $c <<EOI;
+    $* -i 1 <<EOF >>EOF
+    foo
+    bar
+    EOF
+    EOI
+    $b
+
     : extra-newline
     :
     $c <<EOI;
@@ -434,6 +444,15 @@ psr = ($cxx.target.class != 'windows' ? '/' : '\\') # Path separator in regex.
     EOI
     $b
 
+    : shared
+    :
+    $c <<EOI;
+    $* -o foo -e foo >>~/EOF/ 2>>~/EOF/
+    foo
+    EOF
+    EOI
+    $b
+
     : mismatch
     :
     $c <<EOI;
diff --git a/unit-tests/test/script/parser/redirect.test b/unit-tests/test/script/parser/redirect.test
index b1c1209..2e53c4d 100644
--- a/unit-tests/test/script/parser/redirect.test
+++ b/unit-tests/test/script/parser/redirect.test
@@ -59,6 +59,46 @@
     baz
     EOE_
     EOO
+
+    : sharing
+    :
+    {
+      : in-out
+      :
+      $* <<EOI >>EOO
+      cmd <<:/EOF >>:/EOF
+      foo
+      EOF
+      EOI
+      cmd <<:/EOF >>:/EOF
+      foo
+      EOF
+      EOO
+
+      : different
+      :
+      {
+        : modifiers
+        :
+        $* <<EOI 2>>EOE != 0
+        cmd <<:/EOF >>:EOF
+        foo
+        EOF
+        EOI
+        testscript:1:16: error: different modifiers for shared here-document 'EOF'
+        EOE
+
+        : quoting
+        :
+        $* <<EOI 2>>EOE != 0
+        cmd <<EOF >>"EOF"
+        foo
+        EOF
+        EOI
+        testscript:1:13: error: different quoting for shared here-document 'EOF'
+        EOE
+      }
+    }
   }
 
   : regex
@@ -79,6 +119,46 @@
     bar
     EOE
     EOO
+
+    : sharing
+    :
+    {
+      : in-out
+      :
+      $* <<EOI >>EOO
+      cmd >>~/EOF/ 2>>~/EOF/
+      foo
+      EOF
+      EOI
+      cmd >>~/EOF/ 2>>~/EOF/
+      foo
+      EOF
+      EOO
+
+      : different
+      :
+      {
+        : introducers
+        :
+        $* <<EOI 2>>EOE != 0
+        cmd >>~/EOF/ 2>>~%EOF%
+        foo
+        EOF
+        EOI
+        testscript:1:18: error: different introducers for shared here-document regex 'EOF'
+        EOE
+
+        : flags
+        :
+        $* <<EOI 2>>EOE != 0
+        cmd >>~/EOF/ 2>>~/EOF/i
+        foo
+        EOF
+        EOI
+        testscript:1:18: error: different global flags for shared here-document regex 'EOF'
+        EOE
+      }
+    }
   }
 }