diff options
| -rw-r--r-- | build2/test/rule.cxx | 3 | ||||
| -rw-r--r-- | build2/test/script/lexer | 4 | ||||
| -rw-r--r-- | build2/test/script/parser | 13 | ||||
| -rw-r--r-- | build2/test/script/parser.cxx | 223 | ||||
| -rw-r--r-- | build2/test/script/script | 19 | ||||
| -rw-r--r-- | unit-tests/test/script/lexer/script-line.test | 14 |
6 files changed, 217 insertions, 59 deletions
diff --git a/build2/test/rule.cxx b/build2/test/rule.cxx index 7e4101c..188cc90 100644 --- a/build2/test/rule.cxx +++ b/build2/test/rule.cxx @@ -373,7 +373,8 @@ namespace build2 ifdstream ifs (sp); script::parser p; - p.parse (ifs, sp, s, r); + p.pre_parse (ifs, sp, s); + p.parse (sp, s, r); } catch (const io_error& e) { diff --git a/build2/test/script/lexer b/build2/test/script/lexer index d79ef78..80e6036 100644 --- a/build2/test/script/lexer +++ b/build2/test/script/lexer @@ -31,7 +31,9 @@ namespace build2 here_line }; - using base_type::base_type; + lexer_mode () = default; + lexer_mode (value_type v): base_type (v) {} + lexer_mode (base_type v): base_type (v) {} }; class lexer: public build2::lexer diff --git a/build2/test/script/parser b/build2/test/script/parser index adff8a3..6531aba 100644 --- a/build2/test/script/parser +++ b/build2/test/script/parser @@ -29,7 +29,10 @@ namespace build2 // Issue diagnostics and throw failed in case of an error. // void - parse (istream&, const path& name, script&, runner&); + pre_parse (istream&, const path& name, script&); + + void + parse (const path& name, script&, runner&); // Recursive descent parser. // @@ -40,10 +43,16 @@ namespace build2 // protected: void + pre_parse_script (); + + void parse_script (); + line_type + pre_parse_script_line (token&, token_type&); + void - parse_script_line (token&, token_type&); + parse_script_line (token&, token_type&, line_type); void parse_variable_line (token&, token_type&); diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index e515532..ea75d1b 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -18,7 +18,7 @@ namespace build2 using type = token_type; void parser:: - parse (istream& is, const path& p, script& s, runner& r) + pre_parse (istream& is, const path& p, script& s) { path_ = &p; @@ -27,20 +27,43 @@ namespace build2 base_parser::lexer_ = &l; script_ = &s; + runner_ = nullptr; + scope_ = script_; + + pre_parse_ = true; + + pre_parse_script (); + } + + void parser:: + parse (const path& p, script& s, runner& r) + { + path_ = &p; + + lexer_ = nullptr; + base_parser::lexer_ = nullptr; + + script_ = &s; runner_ = &r; scope_ = script_; + pre_parse_ = false; + parse_script (); } void parser:: - parse_script () + pre_parse_script () { token t; type tt; for (;;) { + // Start saving tokens for the next (logical) line. + // + replay_save (); + // We need to start lexing each line in the assign mode in order to // recognize assignment operators as separators. // @@ -50,18 +73,46 @@ namespace build2 if (tt == type::eos) break; - parse_script_line (t, tt); + line_type lt (pre_parse_script_line (t, tt)); assert (tt == type::newline); + + // Stop saving and get the tokens. + // + scope_->lines.push_back (line {lt, replay_data ()}); } + + replay_stop (); // Discard replay of eos. } void parser:: - parse_script_line (token& t, token_type& tt) + parse_script () + { + token t; + type tt; + + for (line& l: scope_->lines) + { + replay_data (move (l.tokens)); // Set the tokens and start playing. + + // We don't really need the assign mode since we already know the + // line type. + // + next (t, tt); + + parse_script_line (t, tt, l.type); + assert (tt == type::newline); + + replay_stop (); // Stop playing. + } + } + + line_type parser:: + pre_parse_script_line (token& t, token_type& tt) { // Decide whether this is a variable assignment or a command. It is a - // variable assignment if the first token is an unquoted word and the - // next is an assign/append/prepend operator. Assignment to a computed - // variable name must use the set builtin. + // variable assignment if the first token is an unquoted word (name) + // and the next is an assign/append/prepend operator. Assignment to a + // computed variable name must use the set builtin. // if (tt == type::word && !t.quoted) { @@ -75,11 +126,22 @@ namespace build2 if (p == type::assign || p == type::prepend || p == type::append) { parse_variable_line (t, tt); - return; + return line_type::variable; } } parse_test_line (t, tt); + return line_type::test; + } + + void parser:: + parse_script_line (token& t, token_type& tt, line_type lt) + { + switch (lt) + { + case line_type::variable: parse_variable_line (t, tt); break; + case line_type::test: parse_test_line (t, tt); break; + } } // Return true if the string contains only digit characters (used to @@ -174,20 +236,21 @@ namespace build2 // Ordered sequence of here-document redirects that we can expect to // see after the command line. // - vector<reference_wrapper<redirect>> hd; + struct here_doc + { + redirect* redir; + string end; + }; + vector<here_doc> hd; // Add the next word to either one of the pending positions or // to program arguments by default. // auto add_word = [&ts, &p, &hd, this] (string&& w, const location& l) { - auto add_here_end = [&w, &hd, &l, this] (redirect& r) + auto add_here_end = [&hd] (redirect& r, string&& w) { - if (w.empty ()) - fail (l) << "empty here-document end marker"; - - hd.push_back (r); - r.here_end = move (w); + hd.push_back (here_doc {&r, move (w)}); }; switch (p) @@ -208,11 +271,13 @@ namespace build2 } break; } - case pending::in_document: add_here_end (ts.in); break; - case pending::in_string: ts.in.value = move (w); break; - case pending::out_document: add_here_end (ts.out); break; + + case pending::in_document: add_here_end (ts.in, move (w)); break; + case pending::out_document: add_here_end (ts.out, move (w)); break; + case pending::err_document: add_here_end (ts.err, move (w)); break; + + case pending::in_string: ts.in.value = move (w); break; case pending::out_string: ts.out.value = move (w); break; - case pending::err_document: add_here_end (ts.err); break; case pending::err_string: ts.err.value = move (w); break; } @@ -364,6 +429,33 @@ namespace build2 case type::out_string: case type::out_document: { + if (pre_parse_) + { + // The only thing we need to handle here are the here-document + // end markers since we need to know how many of the to pre- + // parse after the command. + // + switch (tt) + { + case type::in_document: + case type::out_document: + // We require the end marker to be a literal, unquoted word. + // In particularm, we don't allow quoted because of cases + // like foo"$bar" (where we will see word 'foo'). + // + next (t, tt); + + if (tt != type::word || t.quoted) + fail (l) << "here-document end marker expected"; + + hd.push_back (here_doc {nullptr, move (t.value)}); + break; + } + + next (t, tt); + break; + } + // If this is one of the operators/separators, check that we // don't have any pending locations to be filled. // @@ -395,8 +487,12 @@ namespace build2 reset_quoted (t); parse_names (t, tt, ns, true, "command"); + if (pre_parse_) // Nothing else to do if we are pre-parsing. + break; + // Process what we got. Determine whether anything inside was - // quoted (note that the current token is not part of it). + // quoted (note that the current token is "next" and is not part + // of this). // bool q ((quoted () - (t.quoted ? 1 : 0)) != 0); @@ -500,12 +596,14 @@ namespace build2 { case type::in_null: case type::in_string: - case type::in_document: case type::out_null: case type::out_string: - case type::out_document: parse_redirect (t, l); break; + case type::in_document: + case type::out_document: + fail (l) << "here-document redirect in expansion"; + break; } } @@ -524,7 +622,8 @@ namespace build2 // Verify we don't have anything pending to be filled. // - check_pending (l); + if (!pre_parse_) + check_pending (l); // While we no longer need to recognize command line operators, we // also don't expect a valid test trailer to contain them. So we are @@ -539,7 +638,7 @@ namespace build2 // Parse here-document fragments in the order they were mentioned on // the command line. // - for (redirect& r: hd) + for (here_doc& h: hd) { // Switch to the here-line mode which is like double-quoted but // recognized the newline as a separator. @@ -547,14 +646,22 @@ namespace build2 mode (lexer_mode::here_line); next (t, tt); - r.value = parse_here_document (t, tt, r.here_end); + string v (parse_here_document (t, tt, h.end)); + + if (!pre_parse_) + { + redirect& r (*h.redir); + r.value = move (v); + r.here_end = move (h.end); + } expire_mode (); } // Now that we have all the pieces, run the test. // - runner_->run (ts); + if (!pre_parse_) + runner_->run (ts); } command_exit parser:: @@ -570,18 +677,19 @@ namespace build2 names ns (parse_names (t, tt, true, "exit status")); unsigned long es (256); - try - { - if (ns.size () == 1 && ns[0].simple () && !ns[0].empty ()) - es = stoul (ns[0].value); - } - catch (const exception&) + if (!pre_parse_) { - } + try + { + if (ns.size () == 1 && ns[0].simple () && !ns[0].empty ()) + es = stoul (ns[0].value); + } + catch (const exception&) {} // Fall through. - if (es > 255) - fail (t) << "command exit status expected instead of " << ns << - info << "must be an unsigned integer less than 256"; + if (es > 255) + fail (t) << "exit status expected instead of '" << ns << "'" << + info << "exit status is an unsigned integer less than 256"; + } return command_exit {comp, static_cast<uint8_t> (es)}; } @@ -608,29 +716,32 @@ namespace build2 // names ns (parse_names (t, tt, false, "here-document line")); - // What shall we do if the expansion results in multiple names? For, - // example if the line contains just the variable expansion and it - // is of type strings. Adding all the elements space-separated seems - // like the natural thing to do. - // - for (auto b (ns.begin ()), i (b); i != ns.end (); ++i) + if (!pre_parse_) { - string s; - - try - { - s = value_traits<string>::convert (move (*i), nullptr); - } - catch (const invalid_argument&) + // What shall we do if the expansion results in multiple names? + // For, example if the line contains just the variable expansion + // and it is of type strings. Adding all the elements space- + // separated seems like the natural thing to do. + // + for (auto b (ns.begin ()), i (b); i != ns.end (); ++i) { - fail (t) << "invalid string value '" << *i << "'"; - } + string s; - if (i != b) - r += ' '; + try + { + s = value_traits<string>::convert (move (*i), nullptr); + } + catch (const invalid_argument&) + { + fail (t) << "invalid string value '" << *i << "'"; + } + + if (i != b) + r += ' '; - r += s; - r += '\n'; // Here-document line always includes a newline. + r += s; + r += '\n'; // Here-document line always includes a newline. + } } // We should expand the whole line at once so this would normally be @@ -651,6 +762,8 @@ namespace build2 lookup parser:: lookup_variable (name&& qual, string&& name, const location& loc) { + assert (!pre_parse_); + if (!qual.empty ()) fail (loc) << "qualified variable name"; diff --git a/build2/test/script/script b/build2/test/script/script index 590013a..e3afbca 100644 --- a/build2/test/script/script +++ b/build2/test/script/script @@ -12,6 +12,8 @@ #include <build2/test/target> +#include <build2/test/script/token> // replay_tokens + namespace build2 { class target; @@ -20,6 +22,18 @@ namespace build2 { namespace script { + // Pre-parse representation. + // + enum class line_type {variable, test}; + + struct line + { + line_type type; + replay_tokens tokens; + }; + + // Parse object model. + // enum class redirect_type { none, @@ -118,6 +132,11 @@ namespace build2 // value& append (const variable&); + + // Pre-parse. + // + public: + vector<line> lines; }; class script: public scope diff --git a/unit-tests/test/script/lexer/script-line.test b/unit-tests/test/script/lexer/script-line.test index 9739cec..64d7c18 100644 --- a/unit-tests/test/script/lexer/script-line.test +++ b/unit-tests/test/script/lexer/script-line.test @@ -24,3 +24,17 @@ test.arguments += bar $2 $* ($3 == [null]) + +x = [uint64] 001 +foo $x +cmd abc$(x)23 +cmd >>EOO << EOI +foo +$x +bar +EOO +$x$x$x +EOI + +y = >> +cmd "2"$y |