Properly handle diag directive in build script parser

5 files changed, 273 insertions, 58 deletions

diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript
new file mode 100644
index 0000000..bb0672e
--- /dev/null
+++ b/libbuild2/build/script/parser+diag.test.testscript
@@ -0,0 +1,57 @@
+# file      : libbuild2/build/script/parser+diag.test.testscript
+# license   : MIT; see accompanying LICENSE file
+
+test.options += -g
+
+: name
+:
+$* test <<EOI >>EOO
+  echo abc
+  EOI
+  name: test
+  EOO
+
+: name-deduce
+:
+$* <<EOI >>EOO
+  echo abc
+  EOI
+  name: echo
+  EOO
+
+: diag
+:
+$* <<EOI >>~%EOO%
+  echo abc
+  cat abc
+  diag abc '==>' $>
+  cp abc xyz
+  EOI
+  %diag: abc ==> .+file\{driver\.\}%
+  EOO
+
+: ambiguity
+:
+{
+: name
+:
+  $* test <<EOI 2>>EOE != 0
+    echo abc
+    diag xyz
+    EOI
+    buildfile:12:1: error: both low-verbosity script diagnostics name and 'diag' builtin call
+      buildfile:10: info: script name specified here
+    EOE
+
+  : diag
+  :
+  $* <<EOI 2>>EOE != 0
+    echo abc
+    diag abc
+    cat abc
+    diag xyz
+    EOI
+    buildfile:14:1: error: multiple 'diag' builtin calls
+      buildfile:12:1: info: previous call is here
+    EOE
+}
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index c4c4b03..72c99ad 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -28,7 +28,7 @@ namespace build2
       script parser::
       pre_parse (const target& tg,
                  istream& is, const path_name& pn, uint64_t line,
-                 optional<string> diag_name, const location& diag_loc)
+                 optional<string> diag, const location& diag_loc)
       {
         path_ = &pn;
 
@@ -50,9 +50,9 @@ namespace build2
         runner_ = nullptr;
         environment_ = nullptr;
 
-        if (diag_name)
+        if (diag)
         {
-          diag = make_pair (move (*diag_name), diag_loc);
+          diag_name   = make_pair (move (*diag), diag_loc);
           diag_weight = 4;
         }
 
@@ -69,17 +69,21 @@ namespace build2
         {
           diag_record dr;
 
-          if (!diag)
+          if (!diag_name && !diag_line)
           {
             dr << fail (s.start_loc)
                << "unable to deduce low-verbosity script diagnostics name";
           }
-          else if (diag2)
+          else if (diag_name2)
           {
+            assert (diag_name);
+
             dr << fail (s.start_loc)
                << "low-verbosity script diagnostics name is ambiguous" <<
-              info (diag->second) << "could be '" << diag->first << "'" <<
-              info (diag2->second) << "could be '" << diag2->first << "'";
+              info (diag_name->second) << "could be '" << diag_name->first
+               << "'" <<
+              info (diag_name2->second) << "could be '" << diag_name2->first
+               << "'";
           }
 
           if (!dr.empty ())
@@ -92,7 +96,12 @@ namespace build2
           }
         }
 
-        s.diag = move (diag->first);
+        assert (diag_name.has_value () != diag_line.has_value ());
+
+        if (diag_name)
+          s.diag_name = move (diag_name->first);
+        else
+          s.diag_line = move (diag_line->first);
 
         return s;
       }
@@ -141,6 +150,8 @@ namespace build2
         line_type lt (
           pre_parse_line_start (t, tt, lexer_mode::second_token));
 
+        save_line_ = nullptr;
+
         line ln;
         switch (lt)
         {
@@ -198,26 +209,20 @@ namespace build2
 
         assert (tt == type::newline);
 
-        //@@ TODO: we need to make sure special builtin is the first command.
+        ln.type = lt;
+        ln.tokens = replay_data ();
 
-        // Save the script line, unless this is a special builtin (indicated
-        // by the replay::stop mode).
-        //
-        if (replay_ == replay::save)
-        {
-          ln.type = lt;
-          ln.tokens = replay_data ();
+        if (save_line_ != nullptr)
+          *save_line_ = move (ln);
+        else
           script_->lines.push_back (move (ln));
 
-          if (lt == line_type::cmd_if || lt == line_type::cmd_ifn)
-          {
-            tt = peek (lexer_mode::first_token);
+        if (lt == line_type::cmd_if || lt == line_type::cmd_ifn)
+        {
+          tt = peek (lexer_mode::first_token);
 
-            pre_parse_if_else (t, tt);
-          }
+          pre_parse_if_else (t, tt);
         }
-        else
-          assert (replay_ == replay::stop && lt == line_type::cmd);
       }
 
       void parser::
@@ -303,7 +308,9 @@ namespace build2
       //
 
       optional<process_path> parser::
-      parse_program (token& t, build2::script::token_type& tt, names& ns)
+      parse_program (token& t, build2::script::token_type& tt,
+                     bool first,
+                     names& ns)
       {
         const location l (get_location (t));
 
@@ -318,37 +325,57 @@ namespace build2
         {
           if (diag_weight < w)
           {
-            diag        = make_pair (move (d), l);
+            diag_name   = make_pair (move (d), l);
             diag_weight = w;
-            diag2       = nullopt;
+            diag_name2  = nullopt;
           }
-          else if (w != 0 && w == diag_weight && d != diag->first && !diag2)
-            diag2 = make_pair (move (d), l);
+          else if (w != 0                &&
+                   w == diag_weight      &&
+                   d != diag_name->first &&
+                   !diag_name2)
+            diag_name2 = make_pair (move (d), l);
         };
 
         // Handle special builtins.
         //
-        if (pre_parse_)
+        if (pre_parse_ && first && tt == type::word)
         {
-          if (tt == type::word && t.value == "diag")
+          if (t.value == "diag")
           {
-            // @@ Redo the diag directive handling (save line separately and
-            //    execute later, before script execution).
+            // Check for ambiguity.
             //
             if (diag_weight == 4)
             {
-              fail (script_->start_loc)
-                << "low-verbosity script diagnostics name is ambiguous" <<
-                info (diag->second) << "could be '" << diag->first << "'" <<
-                info (l) << "could be '<name>'";
+              if (diag_name) // Script name.
+              {
+                fail (l) << "both low-verbosity script diagnostics name "
+                         << "and 'diag' builtin call" <<
+                  info (diag_name->second) << "script name specified here";
+              }
+              else           // Custom diagnostics.
+              {
+                assert (diag_line);
+
+                fail (l) << "multiple 'diag' builtin calls" <<
+                  info (diag_line->second) << "previous call is here";
+              }
             }
 
-            set_diag ("<name>", 4);
+            // Instruct the parser to save the diag builtin line separately
+            // from the script lines, when it is fully parsed. Note that it
+            // will be executed prior to the script execution to obtain the
+            // custom diagnostics.
+            //
+            diag_line  = make_pair (line (), l);
+            save_line_ = &diag_line->first;
+            diag_weight  = 4;
 
-            build2::script::parser::parse_program (t, tt, ns);
-            replay_stop ();
+            diag_name  = nullopt;
+            diag_name2 = nullopt;
 
-            return nullopt;
+            // Parse the leading chunk and bail out.
+            //
+            return build2::script::parser::parse_program (t, tt, first, ns);
           }
         }
 
@@ -719,6 +746,55 @@ namespace build2
         runner_->leave (*environment_, s.end_loc);
       }
 
+      names parser::
+      execute_special (const scope& rs, const scope& bs,
+                       environment& e,
+                       const line& ln,
+                       bool omit_builtin)
+      {
+        path_ = nullptr; // Set by replays.
+
+        pre_parse_ = false;
+
+        set_lexer (nullptr);
+
+        // The script shouldn't be able to modify the scopes.
+        //
+        // Note that for now we don't set target_ since it's not clear what
+        // it could be used for (we need scope_ for calling functions such as
+        // $target.path()).
+        //
+        root_ = const_cast<scope*> (&rs);
+        scope_ = const_cast<scope*> (&bs);
+        pbase_ = scope_->src_path_;
+
+        script_ = nullptr;
+        runner_ = nullptr;
+        environment_ = &e;
+
+        // Copy the tokens and start playing.
+        //
+        replay_data (replay_tokens (ln.tokens));
+
+        token t;
+        build2::script::token_type tt;
+        next (t, tt);
+
+        if (omit_builtin)
+        {
+          assert (tt != type::newline && tt != type::eos);
+
+          next (t, tt);
+        }
+
+        names r (tt != type::newline && tt != type::eos
+                 ? parse_names (t, tt, pattern_mode::expand)
+                 : names ());
+
+        replay_stop ();
+        return r;
+      }
+
       // When add a special variable don't forget to update lexer::word().
       //
       bool parser::
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index 15d4ede..4b98cbc 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -36,7 +36,7 @@ namespace build2
         script
         pre_parse (const target&,
                    istream&, const path_name&, uint64_t line,
-                   optional<string> diag, const location& diag_loc);
+                   optional<string> diag_name, const location& diag_loc);
 
         // Recursive descent parser.
         //
@@ -67,6 +67,16 @@ namespace build2
         execute (const scope& root, const scope& base,
                  environment&, const script&, runner&);
 
+        // Parse a special builtin line into names, performing the variable
+        // and pattern expansions. If omit_builtin is true, then omit the
+        // builtin name from the result.
+        //
+        names
+        execute_special (const scope& root, const scope& base,
+                         environment&,
+                         const line&,
+                         bool omit_builtin = true);
+
       protected:
         void
         exec_script ();
@@ -87,24 +97,28 @@ namespace build2
         // leaving the rest for the base parser to handle.
         //
         // During pre-parsing try to deduce the low-verbosity script
-        // diagnostics name.
+        // diagnostics name as a program/builtin name or obtain the custom
+        // low-verbosity diagnostics specified with the diag builtin. Note
+        // that the diag builtin can only appear at the beginning of the
+        // command line.
         //
         virtual optional<process_path>
-        parse_program (token&, build2::script::token_type&, names&) override;
-
-        void
-        parse_program_diag (token&, build2::script::token_type&, names&);
+        parse_program (token&, build2::script::token_type&,
+                       bool first,
+                       names&) override;
 
       protected:
         script* script_;
 
-        // Current low-verbosity script diagnostics name and weight.
+        // Current low-verbosity script diagnostics and its weight.
         //
         // During pre-parsing each command leading names are translated into a
-        // potential script name, unless it is set manually (with the diag
-        // directive or via the constructor). The potential script name has a
-        // weight associated with it, so script names with greater weights
-        // override names with lesser weights. The possible weights are:
+        // potential low-verbosity script diagnostics name, unless the
+        // diagnostics is set manually (script name via the constructor or
+        // custom diagnostics via the diag builtin). The potential script
+        // name has a weight associated with it, so script names with greater
+        // weights override names with lesser weights. The possible weights
+        // are:
         //
         // 0     - builtins that do not add to the script semantics (exit,
         //         true, etc) and are never picked up as a script name
@@ -119,8 +133,20 @@ namespace build2
         // then this ambiguity is reported unless a higher-weighted name is
         // encountered later.
         //
-        optional<pair<string, location>> diag;
-        optional<pair<string, location>> diag2;
+        // If the diag builtin is encountered, then its whole line is saved
+        // (including the leading 'diag' word) for later execution and the
+        // diagnostics weight is set to 4.
+        //
+        // Any attempt to manually set the custom diagnostics twice (the diag
+        // builtin after the script name or after another diag builtin) is
+        // reported as ambiguity.
+        //
+        // At the end of pre-parsing either diag_name or diag_line (but not
+        // both) are present.
+        //
+        optional<pair<string, location>> diag_name;
+        optional<pair<string, location>> diag_name2; // Ambiguous script name.
+        optional<pair<line, location>>   diag_line;
         uint8_t                          diag_weight = 0;
 
         // True during pre-parsing when the pre-parse mode is temporarily
@@ -128,6 +154,15 @@ namespace build2
         //
         bool pre_parse_suspended_ = false;
 
+        // The alternative location where the next line should be saved.
+        //
+        // It is set to NULL before the script line get parsed, indicating
+        // that the line should by default be appended to the script. However,
+        // parse_program() can point it to a different location where the line
+        // should be saved instead (e.g., diag_line, etc).
+        //
+        line* save_line_;
+
         // Execute state.
         //
         runner* runner_;
diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx
index de3e839..7f2840d 100644
--- a/libbuild2/build/script/parser.test.cxx
+++ b/libbuild2/build/script/parser.test.cxx
@@ -73,6 +73,7 @@ namespace build2
       // argv[0] [-l]
       // argv[0] -d
       // argv[0] -p
+      // argv[0] -g [<diag-name>]
       //
       // In the first form read the script from stdin and trace the script
       // execution to stdout using the custom print runner.
@@ -83,6 +84,11 @@ namespace build2
       // In the third form read the script from stdin, parse it and print
       // line tokens quoting information to stdout.
       //
+      // In the forth form read the script from stdin, parse it and print the
+      // low-verbosity script diagnostics name or custom low-verbosity
+      // diagnostics to stdout. If the script doesn't deduce any of them, then
+      // print the diagnostics and exit with non-zero code.
+      //
       // -l
       //    Print the script line number for each executed expression.
       //
@@ -97,6 +103,10 @@ namespace build2
       //    <quoting>      := 'S' | 'D' | 'M'
       //    <completeness> := 'C' | 'P'
       //
+      // -g
+      //    Dump the low-verbosity script diagnostics name or custom
+      //    low-verbosity diagnostics to stdout.
+      //
       int
       main (int argc, char* argv[])
       {
@@ -106,10 +116,12 @@ namespace build2
         {
           run,
           dump,
-          print
+          print,
+          diag
         } m (mode::run);
 
         bool print_line (false);
+        optional<string> diag_name;
 
         for (int i (1); i != argc; ++i)
         {
@@ -121,11 +133,22 @@ namespace build2
             m = mode::dump;
           else if (a == "-p")
             m = mode::print;
+          else if (a == "-g")
+            m = mode::diag;
           else
+          {
+            if (m == mode::diag)
+            {
+              diag_name = move (a);
+              break;
+            }
+
             assert (false);
+          }
         }
 
-        assert (m == mode::run || !print_line);
+        assert (!print_line || m == mode::run);
+        assert (!diag_name  || m == mode::diag);
 
         // Fake build system driver, default verbosity.
         //
@@ -164,7 +187,9 @@ namespace build2
           script s (p.pre_parse (tt,
                                  cin, nm,
                                  11 /* line */,
-                                 string ("test"),
+                                 (m != mode::diag
+                                  ? optional<string> ("test")
+                                  : move (diag_name)),
                                  location (nm, 10)));
 
           switch (m)
@@ -176,6 +201,26 @@ namespace build2
               p.execute (ctx.global_scope, ctx.global_scope, e, s, r);
               break;
             }
+          case mode::diag:
+            {
+              if (s.diag_name)
+              {
+                cout << "name: " << *s.diag_name << endl;
+              }
+              else
+              {
+                assert (s.diag_line);
+
+                environment e (perform_update_id, tt, false /* temp_dir */);
+
+                cout << "diag: " << p.execute_special (ctx.global_scope,
+                                                       ctx.global_scope,
+                                                       e,
+                                                       *s.diag_line) << endl;
+              }
+
+              break;
+            }
           case mode::dump:
             {
               dump (cout, "", s.lines);
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index de759de..5fd8561 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -58,9 +58,11 @@ namespace build2
         //
         bool temp_dir = false;
 
-        // Command name for low-verbosity diagnostics.
+        // Command name for low-verbosity diagnostics and custom low-verbosity
+        // diagnostics line. Note: cannot be both.
         //
-        optional<string> diag;
+        optional<string> diag_name;
+        optional<line>   diag_line;
 
         location start_loc;
         location end_loc;