Invent diag preamble for buildscript

5 files changed, 293 insertions, 108 deletions

diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript
index 30eb859..18f7f83 100644
--- a/libbuild2/build/script/parser+diag.test.testscript
+++ b/libbuild2/build/script/parser+diag.test.testscript
@@ -19,16 +19,76 @@ $* <<EOI >>EOO
   name: echo
   EOO
 
-: diag
+: preamble
 :
-$* <<EOI >>~%EOO%
-  echo abc
-  cat abc
-  diag copy >= $>
-  cp <- $>
-  EOI
-  %diag: copy >= .+file\{driver\.\}%
-  EOO
+{
+  : disambiguate
+  :
+  $* <<EOI >>~%EOO%
+    echo abc | set v
+    cat abc | set v
+    diag copy >= $>
+    cp <- $>
+    EOI
+    echo abc | set v
+    cat abc | set v
+    %diag: copy >= .+file\{driver\.\}%
+    EOO
+
+  : name
+  :
+  $* <<EOI >>EOO
+    n = foo
+    diag copy $n
+    cp $n $>
+    EOI
+    diag: copy foo
+    EOO
+
+  : temp_dir
+  :
+  {
+    test.options += -t
+
+    : no
+    :
+    $* <<EOI >false
+      f = foo
+      diag $f
+      f = $~/f
+      foo "$f"
+      EOI
+
+    : no-depdb
+    :
+    $* <<EOI >false
+      f = $~/f
+      depdb hash "$f"
+      diag $f
+      f = $~/f
+      foo "$f"
+      EOI
+
+    : yes
+    :
+    $* <<EOI >true
+      f = $~/f
+      diag $f
+      foo $f
+      EOI
+
+    : yes-depdb
+    :
+    $* <<EOI >true
+      f = $~/f
+      depdb hash "$f"
+      f = $~/t
+      diag $f
+      f = $~/f
+      foo "$f"
+      EOI
+  }
+}
 
 : ambiguity
 :
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index 3098bc9..77bd21b 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -107,7 +107,7 @@ namespace build2
         {
           diag_record dr;
 
-          if (!diag_name_ && !diag_line_)
+          if (!diag_name_ && diag_preamble_.empty ())
           {
             dr << fail (s.start_loc)
                << "unable to deduce low-verbosity script diagnostics name";
@@ -136,12 +136,12 @@ namespace build2
 
         // Save the script name or custom diagnostics line.
         //
-        assert (diag_name_.has_value () != diag_line_.has_value ());
+        assert (diag_name_.has_value () == diag_preamble_.empty ());
 
         if (diag_name_)
           s.diag_name = move (diag_name_->first);
         else
-          s.diag_line = move (diag_line_->first);
+          s.diag_preamble = move (diag_preamble_);
 
         // Save the custom dependency change tracking lines, if present.
         //
@@ -641,6 +641,12 @@ namespace build2
               fail (l) << "'" << v << "' call via 'env' builtin";
           };
 
+          auto diag_loc = [this] ()
+          {
+            assert (!diag_preamble_.empty ());
+            return diag_preamble_.back ().tokens[0].location ();
+          };
+
           if (v == "diag")
           {
             verify ();
@@ -657,24 +663,41 @@ namespace build2
               }
               else           // Custom diagnostics.
               {
-                assert (diag_line_);
-
                 fail (l) << "multiple 'diag' builtin calls" <<
-                  info (diag_line_->second) << "previous call is here";
+                  info (diag_loc ()) << "previous call is here";
               }
             }
 
-            // Instruct the parser to save the diag builtin line separately
-            // from the script lines, when it is fully parsed. Note that it
-            // will be executed prior to the script body execution to obtain
-            // the custom diagnostics.
+            // Move the script body to the end of the diag preamble.
             //
-            diag_line_   = make_pair (line (), l);
-            save_line_   = &diag_line_->first;
-            diag_weight_ = 4;
+            // Note that we move into the preamble whatever is there and delay
+            // the check until the execution (see the depdb preamble
+            // collecting for the reasoning).
+            //
+            lines& ls (script_->body);
+            diag_preamble_.insert (diag_preamble_.end (),
+                                   make_move_iterator (ls.begin ()),
+                                   make_move_iterator (ls.end ()));
+            ls.clear ();
 
-            diag_name_  = nullopt;
-            diag_name2_ = nullopt;
+            // Also move the body_temp_dir flag, if it is true.
+            //
+            if (script_->body_temp_dir)
+            {
+              script_->diag_preamble_temp_dir = true;
+              script_->body_temp_dir = false;
+            }
+
+            // Similar to the depdb preamble collection, instruct the parser
+            // to save the depdb builtin line separately from the script
+            // lines.
+            //
+            diag_preamble_.push_back (line ());
+            save_line_ = &diag_preamble_.back ();
+
+            diag_weight_ = 4;
+            diag_name_   = nullopt;
+            diag_name2_  = nullopt;
 
             // Note that the rest of the line contains the builtin argument to
             // be printed, thus we parse it in the value lexer mode.
@@ -704,9 +727,8 @@ namespace build2
               fail (l) << "'depdb' builtin can only be used for file-based "
                        << "targets";
 
-            if (diag_line_)
-              fail (diag_line_->second)
-                << "'diag' builtin call before 'depdb' call" <<
+            if (!diag_preamble_.empty ())
+              fail (diag_loc ()) << "'diag' builtin call before 'depdb' call" <<
                 info (l) << "'depdb' call is here";
 
             // Note that the rest of the line contains the builtin command
@@ -1212,6 +1234,24 @@ namespace build2
           runner_->leave (e, s.end_loc);
       }
 
+      // Return true if the specified expression executes the set builtin or
+      // is a for-loop.
+      //
+      static bool
+      valid_preamble_cmd (const command_expr& ce,
+                          const function<command_function>& cf)
+      {
+        return find_if (
+          ce.begin (), ce.end (),
+          [&cf] (const expr_term& et)
+          {
+            const process_path& p (et.pipe.back ().program);
+            return p.initial == nullptr &&
+                   (p.recall.string () == "set" ||
+                    (cf != nullptr && p.recall.string () == "for"));
+          }) != ce.end ();
+      }
+
       void parser::
       exec_depdb_preamble (action a, const scope& bs, const file& t,
                            environment& e, const script& s, runner& r,
@@ -1355,18 +1395,7 @@ namespace build2
             command_expr ce (
               parse_command_line (t, static_cast<token_type&> (tt)));
 
-            // Verify that this expression executes the set builtin or is a
-            // for-loop.
-            //
-            if (find_if (ce.begin (), ce.end (),
-                         [&cf] (const expr_term& et)
-                         {
-                           const process_path& p (et.pipe.back ().program);
-                           return p.initial == nullptr &&
-                                  (p.recall.string () == "set" ||
-                                  (cf != nullptr &&
-                                   p.recall.string () == "for"));
-                         }) == ce.end ())
+            if (!valid_preamble_cmd (ce, cf))
             {
               const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens);
               assert (!rt.empty ());
@@ -1384,6 +1413,79 @@ namespace build2
         exec_lines (begin, end, exec_cmd);
       }
 
+      names parser::
+      execute_diag_preamble (const scope& rs, const scope& bs,
+                             environment& e, const script& s, runner& r,
+                             bool diag, bool enter, bool leave)
+      {
+        tracer trace ("exec_diag_preamble");
+
+        assert (!s.diag_preamble.empty ());
+
+        const line& dl (s.diag_preamble.back ()); // Diag builtin line.
+
+        pre_exec (rs, bs, e, &s, &r);
+
+        if (enter)
+          runner_->enter (e, s.start_loc);
+
+        // Perform the variable assignments.
+        //
+        auto exec_cmd = [&dl, this] (token& t,
+                                     build2::script::token_type& tt,
+                                     const iteration_index* ii, size_t li,
+                                     bool /* single */,
+                                     const function<command_function>& cf,
+                                     const location& ll)
+        {
+          // Note that we never reset the line index to zero (as we do in
+          // execute_body()) assuming that there are some script body commands
+          // to follow.
+          //
+          command_expr ce (
+            parse_command_line (t, static_cast<token_type&> (tt)));
+
+          if (!valid_preamble_cmd (ce, cf))
+          {
+            const replay_tokens& rt (dl.tokens);
+            assert (!rt.empty ());
+
+            fail (ll) << "disallowed command in diag preamble" <<
+              info << "only variable assignments are allowed in diag preamble"
+                   << info (rt[0].location ()) << "diag preamble ends here";
+          }
+
+          runner_->run (*environment_, ce, ii, li, cf, ll);
+        };
+
+        exec_lines (s.diag_preamble.begin (), s.diag_preamble.end () - 1,
+                    exec_cmd);
+
+        // Execute the diag line, if requested.
+        //
+        names ns;
+
+        if (diag)
+        {
+          // Copy the tokens and start playing.
+          //
+          replay_data (replay_tokens (dl.tokens));
+
+          token t;
+          build2::script::token_type tt;
+          next (t, tt);
+
+          ns = exec_special (t, tt, true /* skip_first */);
+
+          replay_stop ();
+        }
+
+        if (leave)
+          runner_->leave (e, s.end_loc);
+
+        return ns;
+      }
+
       void parser::
       pre_exec (const scope& rs, const scope& bs,
                 environment& e, const script* s, runner* r)
@@ -1487,28 +1589,6 @@ namespace build2
                : names ();
       }
 
-      names parser::
-      execute_special (const scope& rs, const scope& bs,
-                       environment& e,
-                       const line& ln,
-                       bool omit_builtin)
-      {
-        pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */);
-
-        // Copy the tokens and start playing.
-        //
-        replay_data (replay_tokens (ln.tokens));
-
-        token t;
-        build2::script::token_type tt;
-        next (t, tt);
-
-        names r (exec_special (t, tt, omit_builtin));
-
-        replay_stop ();
-        return r;
-      }
-
       void parser::
       exec_depdb_dyndep (token& lt, build2::script::token_type& ltt,
                          size_t li, const location& ll,
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index a81583c..1328bae 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -96,7 +96,6 @@ namespace build2
                       environment&, const script&, runner&,
                       bool enter = true, bool leave = true);
 
-
         // Execute the first or the second (dyndep) half of the depdb
         // preamble.
         //
@@ -174,15 +173,21 @@ namespace build2
           return v;
         }
 
-        // Parse a special builtin line into names, performing the variable
-        // and pattern expansions. If omit_builtin is true, then omit the
-        // builtin name from the result.
+        // If the diag argument is true, then execute the preamble including
+        // the (trailing) diagnostics line and return the resulting names (see
+        // exec_special() for the diagnostics line execution semantics).
+        // Otherwise, execute the preamble excluding the diagnostics line and
+        // return an empty names list. If requested, call the runner's enter()
+        // and leave() functions that initialize/clean up the environment
+        // before/after the preamble execution.
+        //
+        // Note: having both root and base scopes for testing (where we pass
+        // global scope for both).
         //
         names
-        execute_special (const scope& root, const scope& base,
-                         environment&,
-                         const line&,
-                         bool omit_builtin = true);
+        execute_diag_preamble (const scope& root, const scope& base,
+                               environment&, const script&, runner&,
+                               bool diag, bool enter, bool leave);
 
       protected:
         // Setup the parser for subsequent exec_*() function calls.
@@ -203,6 +208,10 @@ namespace build2
           exec_lines (l.begin (), l.end (), c);
         }
 
+        // Parse a special builtin line into names, performing the variable
+        // and pattern expansions. Optionally, skip the first token (builtin
+        // name, etc).
+        //
         names
         exec_special (token&, build2::script::token_type&, bool skip_first);
 
@@ -293,18 +302,20 @@ namespace build2
         //
         // If the diag builtin is encountered, then its whole line is saved
         // (including the leading 'diag' word) for later execution and the
-        // diagnostics weight is set to 4.
+        // diagnostics weight is set to 4. The preceding lines, which can only
+        // contain variable assignments (including via the set builtin,
+        // potentially inside the flow control constructs), are also saved.
         //
         // Any attempt to manually set the custom diagnostics twice (the diag
         // builtin after the script name or after another diag builtin) is
         // reported as ambiguity.
         //
-        // At the end of pre-parsing either diag_name_ or diag_line_ (but not
-        // both) are present.
+        // At the end of pre-parsing either diag_name_ is present or
+        // diag_preamble_ is not empty (but not both).
         //
         optional<pair<string, location>> diag_name_;
         optional<pair<string, location>> diag_name2_; // Ambiguous script name.
-        optional<pair<line, location>>   diag_line_;
+        lines                            diag_preamble_;
         uint8_t                          diag_weight_ = 0;
 
         // Custom dependency change tracking.
@@ -368,9 +379,9 @@ namespace build2
         // Before the script line gets parsed, it is set to a temporary value
         // that will by default be appended to the script. However,
         // parse_program() can point it to a different location where the line
-        // should be saved instead (e.g., diag_line_, etc) or set it to NULL
-        // if the line is handled in an ad-hoc way and should be dropped
-        // (e.g., depdb_clear_, etc).
+        // should be saved instead (e.g., diag_preamble_ back, etc) or set it
+        // to NULL if the line is handled in an ad-hoc way and should be
+        // dropped (e.g., depdb_clear_, etc).
         //
         line* save_line_;
 
diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx
index 7e9c612..d69929c 100644
--- a/libbuild2/build/script/parser.test.cxx
+++ b/libbuild2/build/script/parser.test.cxx
@@ -119,8 +119,8 @@ namespace build2
       // argv[0] [-l] [-r]
       // argv[0] -b [-t]
       // argv[0] -d [-t]
+      // argv[0] -g [-t] [<diag-name>]
       // argv[0] -q
-      // argv[0] -g [<diag-name>]
       //
       // In the first form read the script from stdin and trace the script
       // body execution to stdout using the custom print runner.
@@ -131,14 +131,14 @@ namespace build2
       // In the third form read the script from stdin, parse it and dump the
       // depdb preamble lines to stdout.
       //
-      // In the forth form read the script from stdin, parse it and print
-      // line tokens quoting information to stdout.
-      //
-      // In the fifth form read the script from stdin, parse it and print the
+      // In the forth form read the script from stdin, parse it and print the
       // low-verbosity script diagnostics name or custom low-verbosity
       // diagnostics to stdout. If the script doesn't deduce any of them, then
       // print the diagnostics and exit with non-zero code.
       //
+      // In the fifth form read the script from stdin, parse it and print
+      // line tokens quoting information to stdout.
+      //
       // -l
       //    Print the script line number for each executed expression.
       //
@@ -151,9 +151,13 @@ namespace build2
       // -d
       //    Dump the parsed script depdb preamble to stdout.
       //
+      // -g
+      //    Dump the low-verbosity script diagnostics name or custom
+      //    low-verbosity diagnostics to stdout.
+      //
       // -t
-      //    Print true if the body (-b) or depdb preamble (-d) references the
-      //    temporary directory and false otherwise.
+      //    Print true if the body (-b), depdb preamble (-d), or diag preamble
+      //    (-g) references the temporary directory and false otherwise.
       //
       // -q
       //    Print the parsed script tokens quoting information to sdout. If a
@@ -163,10 +167,6 @@ namespace build2
       //    <quoting>      := 'S' | 'D' | 'M'
       //    <completeness> := 'C' | 'P'
       //
-      // -g
-      //    Dump the low-verbosity script diagnostics name or custom
-      //    low-verbosity diagnostics to stdout.
-      //
       int
       main (int argc, char* argv[])
       {
@@ -177,8 +177,8 @@ namespace build2
           run,
           body,
           depdb_preamble,
-          quoting,
-          diag
+          diag,
+          quoting
         } m (mode::run);
 
         bool print_line (false);
@@ -198,15 +198,17 @@ namespace build2
             m = mode::body;
           else if (a == "-d")
             m = mode::depdb_preamble;
+          else if (a == "-g")
+            m = mode::diag;
           else if (a == "-t")
           {
-            assert (m == mode::body || m == mode::depdb_preamble);
+            assert (m == mode::body           ||
+                    m == mode::depdb_preamble ||
+                    m == mode::diag);
             temp_dir = true;
           }
           else if (a == "-q")
             m = mode::quoting;
-          else if (a == "-g")
-            m = mode::diag;
           else
           {
             if (m == mode::diag)
@@ -219,8 +221,8 @@ namespace build2
           }
         }
 
-        assert (!print_line       || m == mode::run);
-        assert (!print_iterations || m == mode::run);
+        assert (!print_line       || m == mode::run || m == mode::diag);
+        assert (!print_iterations || m == mode::run || m == mode::diag);
         assert (!diag_name        || m == mode::diag);
 
         // Fake build system driver, default verbosity.
@@ -274,9 +276,29 @@ namespace build2
           {
           case mode::run:
             {
-              environment e (perform_update_id, tt, bs, s.body_temp_dir);
+              environment e (perform_update_id, tt, bs, false /* temp_dir */);
               print_runner r (print_line, print_iterations);
-              p.execute_body (ctx.global_scope, ctx.global_scope, e, s, r);
+
+              bool exec_diag (!s.diag_preamble.empty ());
+
+              if (exec_diag)
+              {
+                if (s.diag_preamble_temp_dir)
+                  e.set_temp_dir_variable ();
+
+                p.execute_diag_preamble (ctx.global_scope, ctx.global_scope,
+                                         e, s, r,
+                                         false /* diag */,
+                                         true  /* enter */,
+                                         false /* leave */);
+              }
+
+              if (s.body_temp_dir && !s.diag_preamble_temp_dir)
+                e.set_temp_dir_variable ();
+
+              p.execute_body (ctx.global_scope, ctx.global_scope,
+                              e, s, r,
+                              !exec_diag /* enter */);
               break;
             }
           case mode::diag:
@@ -287,14 +309,26 @@ namespace build2
               }
               else
               {
-                assert (s.diag_line);
+                if (!temp_dir)
+                {
+                  environment e (perform_update_id,
+                                 tt,
+                                 bs,
+                                 s.diag_preamble_temp_dir);
 
-                environment e (perform_update_id, tt, bs, false /* temp_dir */);
+                  print_runner r (print_line, print_iterations);
 
-                cout << "diag: " << p.execute_special (ctx.global_scope,
+                  names diag (p.execute_diag_preamble (ctx.global_scope,
                                                        ctx.global_scope,
-                                                       e,
-                                                       *s.diag_line) << endl;
+                                                       e, s, r,
+                                                       true /* diag */,
+                                                       true /* enter */,
+                                                       true /* leave */));
+
+                  cout << "diag: " << diag << endl;
+                }
+                else
+                  cout << (s.diag_preamble_temp_dir ? "true" : "false") << endl;
               }
 
               break;
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index ec27781..b2b886c 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -47,13 +47,11 @@ namespace build2
       class script
       {
       public:
-        using lines_type = build::script::lines;
-
         // Note that the variables are not pre-entered into a pool during the
         // parsing phase, so the line variable pointers are NULL.
         //
-        lines_type body;
-        bool       body_temp_dir = false; // True if the body references $~.
+        lines body;
+        bool  body_temp_dir = false; // True if the body references $~.
 
         // Referenced ordinary (non-special) variables.
         //
@@ -68,11 +66,13 @@ namespace build2
         small_vector<string, 2> vars; // 2 for command and options.
 
         // Command name for low-verbosity diagnostics and custom low-verbosity
-        // diagnostics line. Note: cannot be both (see the script parser for
+        // diagnostics line, potentially preceded with the variable
+        // assignments. Note: cannot be both (see the script parser for
         // details).
         //
         optional<string> diag_name;
-        optional<line>   diag_line;
+        lines            diag_preamble;
+        bool             diag_preamble_temp_dir = false; // True if refs $~.
 
         // The script's custom dependency change tracking lines (see the
         // script parser for details).
@@ -81,7 +81,7 @@ namespace build2
         bool             depdb_value;                    // String or hash.
         optional<size_t> depdb_dyndep;                   // Pos of first dyndep.
         bool             depdb_dyndep_byproduct = false; // dyndep --byproduct
-        lines_type       depdb_preamble;                 // Note include vars.
+        lines            depdb_preamble;                 // Note include vars.
         bool             depdb_preamble_temp_dir = false; // True if refs $~.
 
         location start_loc;