From 4a4e5ad3c50619ad7653b01b562af9794c97aa80 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 1 Nov 2016 16:35:47 +0200 Subject: Implement command-pipe, command-expr in testscript parser --- build2/test/script/parser.cxx | 138 ++++++++++++++++++++++----- build2/test/script/runner | 10 +- build2/test/script/runner.cxx | 84 ++++++++-------- build2/test/script/script | 48 ++++++++++ build2/test/script/script.cxx | 48 ++++++++++ build2/test/script/script.ixx | 21 ++++ doc/testscript.cli | 9 +- unit-tests/test/script/parser/buildfile | 4 +- unit-tests/test/script/parser/driver.cxx | 4 +- unit-tests/test/script/parser/exit.test | 23 +++++ unit-tests/test/script/parser/pipe-expr.test | 47 +++++++++ 11 files changed, 357 insertions(+), 79 deletions(-) create mode 100644 unit-tests/test/script/parser/exit.test create mode 100644 unit-tests/test/script/parser/pipe-expr.test diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index 7fb0608..27d8888 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -783,7 +783,17 @@ namespace build2 pair> parser:: parse_command_line (token& t, type& tt, line_type lt, size_t li) { - command c; + command_expr expr {{expr_operator::log_and, {}}}; + command c; // Command being assembled. + + // Make sure the command makes sense. + // + auto check_command = [&c, this] (const location& l) + { + if (c.out.type == redirect_type::merge && + c.err.type == redirect_type::merge) + fail (l) << "stdout and stderr redirected to each other"; + }; // Pending positions where the next word should go. // @@ -814,7 +824,10 @@ namespace build2 // struct here_doc { - redirect* redir; + size_t expr; // Index in command_expr. + size_t pipe; // Index in command_pipe. + size_t redir; // Redirect (0 - in, 1 - out, 2 - err). + string end; bool no_newline; }; @@ -823,8 +836,8 @@ namespace build2 // Add the next word to either one of the pending positions or // to program arguments by default. // - auto add_word = - [&c, &p, &nn, &app, &ct, &hd, this] (string&& w, const location& l) + auto add_word = [&expr, &c, &p, &nn, &app, &ct, &hd, this] + (string&& w, const location& l) { auto add_merge = [&l, this] (redirect& r, const string& w, int fd) { @@ -849,9 +862,11 @@ namespace build2 r.str = move (w); }; - auto add_here_end = [&hd, &nn] (redirect& r, string&& w) + auto add_here_end = [&expr, &hd, &nn] (size_t r, string&& w) { - hd.push_back (here_doc {&r, move (w), nn}); + hd.push_back ( + here_doc { + expr.size () - 1, expr.back ().pipe.size (), r, move (w), nn}); }; auto parse_path = [&l, this] (string&& w, const char* what) -> path @@ -903,9 +918,9 @@ namespace build2 case pending::out_string: add_here_str (c.out, move (w)); break; case pending::err_string: add_here_str (c.err, move (w)); break; - case pending::in_document: add_here_end (c.in, move (w)); break; - case pending::out_document: add_here_end (c.out, move (w)); break; - case pending::err_document: add_here_end (c.err, move (w)); break; + case pending::in_document: add_here_end (0, move (w)); break; + case pending::out_document: add_here_end (1, move (w)); break; + case pending::err_document: add_here_end (2, move (w)); break; case pending::in_file: add_file (c.in, 0, move (w)); break; case pending::out_file: add_file (c.out, 1, move (w)); break; @@ -1120,8 +1135,6 @@ namespace build2 { switch (tt) { - case type::equal: - case type::not_equal: case type::semi: case type::colon: case type::newline: @@ -1130,6 +1143,38 @@ namespace build2 break; } + case type::equal: + case type::not_equal: + { + if (!pre_parse_) + check_pending (l); + + c.exit = parse_command_exit (t, tt); + + // Only a limited set of things can appear after the exit status + // so we check this here. + // + switch (tt) + { + case type::semi: + case type::colon: + case type::newline: + + case type::pipe: + case type::log_or: + case type::log_and: + break; + default: + fail (t) << "unexpected " << t << " after command exit status"; + } + + break; + } + + case type::pipe: + case type::log_or: + case type::log_and: + case type::in_pass: case type::out_pass: @@ -1181,7 +1226,7 @@ namespace build2 if (tt != type::word || t.quoted) fail (l) << "expected here-document end marker"; - hd.push_back (here_doc {nullptr, move (t.value), nn}); + hd.push_back (here_doc {0, 0, 0, move (t.value), nn}); break; } @@ -1198,6 +1243,29 @@ namespace build2 // switch (tt) { + case type::pipe: + case type::log_or: + case type::log_and: + { + // Check that the previous command makes sense. + // + check_command (l); + expr.back ().pipe.push_back (move (c)); + + c = command (); + p = pending::program; + + if (tt != type::pipe) + { + expr_operator o (tt == type::log_or + ? expr_operator::log_or + : expr_operator::log_and); + expr.push_back ({o, command_pipe ()}); + } + + break; + } + case type::in_pass: case type::out_pass: @@ -1367,6 +1435,29 @@ namespace build2 // switch (tt) { + case type::pipe: + case type::log_or: + case type::log_and: + { + // Check that the previous command makes sense. + // + check_command (l); + expr.back ().pipe.push_back (move (c)); + + c = command (); + p = pending::program; + + if (tt != type::pipe) + { + expr_operator o (tt == type::log_or + ? expr_operator::log_or + : expr_operator::log_and); + expr.push_back ({o, command_pipe ()}); + } + + break; + } + case type::in_pass: case type::out_pass: @@ -1424,24 +1515,17 @@ namespace build2 if (!pre_parse_) { - // Verify we don't have anything pending to be filled. + // Verify we don't have anything pending to be filled and the + // command makes sense. // check_pending (l); + check_command (l); - if (c.out.type == redirect_type::merge && - c.err.type == redirect_type::merge) - fail (l) << "stdout and stderr redirected to each other"; + expr.back ().pipe.push_back (move (c)); } - // While we no longer need to recognize command line operators, we - // also don't expect a valid test trailer to contain them. So we are - // going to continue lexing in the script_line mode. - // - if (tt == type::equal || tt == type::not_equal) - c.exit = parse_command_exit (t, tt); - // Colon and semicolon are only valid in test command lines. Note that - // we still recognize them lexically, they are just not a valid tokens + // we still recognize them lexically, they are just not valid tokens // per the grammar. // if (tt == type::colon || tt == type::semi) @@ -1514,7 +1598,9 @@ namespace build2 if (!pre_parse_) { - redirect& r (*h.redir); + command& c (expr[h.expr].pipe[h.pipe]); + redirect& r (h.redir == 0 ? c.in : h.redir == 1 ? c.out : c.err); + r.doc.doc = move (v); r.doc.end = move (h.end); } @@ -1525,7 +1611,7 @@ namespace build2 // Now that we have all the pieces, run the command. // if (!pre_parse_) - runner_->run (*scope_, c, li, ll); + runner_->run (*scope_, expr, li, ll); return r; } diff --git a/build2/test/script/runner b/build2/test/script/runner index 0180108..b78628c 100644 --- a/build2/test/script/runner +++ b/build2/test/script/runner @@ -26,16 +26,16 @@ namespace build2 virtual void enter (scope&, const location&) = 0; - // Index is the 1-base index of this command in the command list + // Index is the 1-base index of this command line in the command list // (e.g., in a compound test). If it is 0 then it means there is only // one command (e.g., a simple test). This information can be used, // for example, to derive file names. // - // Location is the start position of this command in the testscript. - // It can be used in diagnostics. + // Location is the start position of this command line in the + // testscript. It can be used in diagnostics. // virtual void - run (scope&, const command&, size_t index, const location&) = 0; + run (scope&, const command_expr&, size_t index, const location&) = 0; // Location is the scope end location (for diagnostics, etc). // @@ -50,7 +50,7 @@ namespace build2 enter (scope&, const location&) override; virtual void - run (scope&, const command&, size_t, const location&) override; + run (scope&, const command_expr&, size_t, const location&) override; virtual void leave (scope&, const location&) override; diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx index ee7b5df..7271234 100644 --- a/build2/test/script/runner.cxx +++ b/build2/test/script/runner.cxx @@ -21,7 +21,7 @@ namespace build2 // empty. // static bool - non_empty (const path& p, const location& cl) + non_empty (const path& p, const location& ll) { if (p.empty () || !exists (p)) return false; @@ -37,7 +37,7 @@ namespace build2 // executed let's add the location anyway to ease the // troubleshooting. And let's stick to that principle down the road. // - error (cl) << "unable to read " << p << ": " << e.what (); + error (ll) << "unable to read " << p << ": " << e.what (); throw failed (); } } @@ -51,13 +51,13 @@ namespace build2 const path& op, const path& ip, const redirect& rd, - const location& cl, + const location& ll, scope& sp, const char* what) { - auto input_info = [&ip, &cl] (diag_record& d) + auto input_info = [&ip, &ll] (diag_record& d) { - if (non_empty (ip, cl)) + if (non_empty (ip, ll)) d << info << "stdin: " << ip; }; @@ -67,9 +67,9 @@ namespace build2 // Check that there is no output produced. // - if (non_empty (op, cl)) + if (non_empty (op, ll)) { - diag_record d (fail (cl)); + diag_record d (fail (ll)); d << pr << " unexpectedly writes to " << what << info << what << ": " << op; @@ -96,7 +96,7 @@ namespace build2 } catch (const io_error& e) { - fail (cl) << "unable to write " << orp << ": " << e.what (); + fail (ll) << "unable to write " << orp << ": " << e.what (); } // Use diff utility to compare the output with the expected result. @@ -130,13 +130,13 @@ namespace build2 // Output doesn't match the expected result. // - diag_record d (error (cl)); + diag_record d (error (ll)); d << pr << " " << what << " doesn't match the expected output"; auto output_info = - [&d, &what, &cl] (const path& p, const char* prefix) + [&d, &what, &ll] (const path& p, const char* prefix) { - if (non_empty (p, cl)) + if (non_empty (p, ll)) d << info << prefix << what << ": " << p; else d << info << prefix << what << " is empty"; @@ -156,7 +156,7 @@ namespace build2 // p.wait (); // Check throw. - error (cl) << "failed to compare " << what + error (ll) << "failed to compare " << what << " with the expected output"; } @@ -165,7 +165,7 @@ namespace build2 } catch (const process_error& e) { - error (cl) << "unable to execute " << pp << ": " << e.what (); + error (ll) << "unable to execute " << pp << ": " << e.what (); if (e.child ()) exit (1); @@ -194,7 +194,7 @@ namespace build2 } void concurrent_runner:: - leave (scope& sp, const location& cl) + leave (scope& sp, const location& ll) { // Remove files and directories in the order opposite to the order of // cleanup registration. @@ -206,10 +206,10 @@ namespace build2 // working directory. // auto verify = - [&sp, &cl] (const path& p, const path& orig, const char* what) + [&sp, &ll] (const path& p, const path& orig, const char* what) { if (!p.sub (sp.wd_path)) - fail (cl) << "registered for cleanup " << what << " " << orig + fail (ll) << "registered for cleanup " << what << " " << orig << " is out of working directory " << sp.wd_path; }; @@ -233,7 +233,7 @@ namespace build2 rmdir_status r (rmdir (d, 2)); if (r != rmdir_status::success) - fail (cl) << "registered for cleanup directory " << d + fail (ll) << "registered for cleanup directory " << d << (r == rmdir_status::not_empty ? " is not empty" : " does not exist"); @@ -262,7 +262,7 @@ namespace build2 // for completeness. // if (r == rmdir_status::not_empty) - fail (cl) << "registered for cleanup wildcard " << p + fail (ll) << "registered for cleanup wildcard " << p << " matches the current directory"; continue; @@ -273,14 +273,16 @@ namespace build2 verify (p, p, "file"); if (rmfile (p, 2) == rmfile_status::not_exist) - fail (cl) << "registered for cleanup file " << p + fail (ll) << "registered for cleanup file " << p << " does not exist"; } } void concurrent_runner:: - run (scope& sp, const command& c, size_t ci, const location& cl) + run (scope& sp, const command_expr& expr, size_t li, const location& ll) { + const command& c (expr.back ().pipe.back ()); // @@ TMP + if (verb >= 3) text << c; @@ -315,7 +317,7 @@ namespace build2 // Normalize a path. Also make relative path absolute using the // scope's working directory unless it is already absolute. // - auto normalize = [&sp, &cl] (path p) -> path + auto normalize = [&sp, &ll] (path p) -> path { path r (p.absolute () ? move (p) : sp.wd_path / move (p)); @@ -325,7 +327,7 @@ namespace build2 } catch (const invalid_path& e) { - fail (cl) << "invalid file path " << e.path; + fail (ll) << "invalid file path " << e.path; } return r; @@ -333,15 +335,15 @@ namespace build2 // Create unique path for a test command standard stream cache file. // - auto std_path = [&ci, &normalize] (const char* n) -> path + auto std_path = [&li, &normalize] (const char* n) -> path { path p (n); - // 0 if belongs to a single-command test scope, otherwise is the - // command number (start from one) in the test scope. + // 0 if belongs to a single-line test scope, otherwise is the + // command line number (start from one) in the test scope. // - if (ci > 0) - p += "-" + to_string (ci); + if (li > 0) + p += "-" + to_string (li); return normalize (move (p)); }; @@ -352,7 +354,7 @@ namespace build2 // Open a file for passing to the test command stdin. // - auto open_stdin = [&stdin, &si, &in, &cl] () + auto open_stdin = [&stdin, &si, &in, &ll] () { assert (!stdin.empty ()); @@ -362,7 +364,7 @@ namespace build2 } catch (const io_error& e) { - fail (cl) << "unable to read " << stdin << ": " << e.what (); + fail (ll) << "unable to read " << stdin << ": " << e.what (); } in = si.fd (); @@ -399,7 +401,7 @@ namespace build2 } catch (const io_error& e) { - fail (cl) << "unable to write " << stdin << ": " << e.what (); + fail (ll) << "unable to write " << stdin << ": " << e.what (); } open_stdin (); @@ -432,7 +434,7 @@ namespace build2 // descriptors for merge, pass or null redirects respectively not // opening a file. // - auto open = [&sp, &cl, &std_path, &normalize] (const redirect& r, + auto open = [&sp, &ll, &std_path, &normalize] (const redirect& r, int dfd, path& p, ofdstream& os) -> int @@ -472,7 +474,7 @@ namespace build2 } catch (const io_error& e) { - fail (cl) << "unable to write " << p << ": " << e.what (); + fail (ll) << "unable to write " << p << ": " << e.what (); } sp.clean ({cleanup_type::always, p}, true); @@ -534,14 +536,14 @@ namespace build2 } catch (const io_error& e) { - fail (cl) << "unable to read " << stderr << ": " + fail (ll) << "unable to read " << stderr << ": " << e.what (); } } // Fail with a proper diagnostics. // - diag_record d (fail (cl)); + diag_record d (fail (ll)); if (!status) d << pp << " terminated abnormally"; @@ -555,20 +557,20 @@ namespace build2 else assert (false); - if (non_empty (stderr, cl)) + if (non_empty (stderr, ll)) d << info << "stderr: " << stderr; - if (non_empty (stdout, cl)) + if (non_empty (stdout, ll)) d << info << "stdout: " << stdout; - if (non_empty (stdin, cl)) + if (non_empty (stdin, ll)) d << info << "stdin: " << stdin; } // Check if the standard outputs match expectations. // - check_output (pp, stdout, stdin, c.out, cl, sp, "stdout"); - check_output (pp, stderr, stdin, c.err, cl, sp, "stderr"); + check_output (pp, stdout, stdin, c.out, ll, sp, "stdout"); + check_output (pp, stderr, stdin, c.err, ll, sp, "stderr"); } catch (const io_error& e) { @@ -577,12 +579,12 @@ namespace build2 // pr.wait (); // Check throw. - fail (cl) << "IO operation failed for " << pp << ": " << e.what (); + fail (ll) << "IO operation failed for " << pp << ": " << e.what (); } } catch (const process_error& e) { - error (cl) << "unable to execute " << pp << ": " << e.what (); + error (ll) << "unable to execute " << pp << ": " << e.what (); if (e.child ()) exit (1); diff --git a/build2/test/script/script b/build2/test/script/script index 0b3a05e..9c65b96 100644 --- a/build2/test/script/script +++ b/build2/test/script/script @@ -26,6 +26,7 @@ namespace build2 // Pre-parse representation. // + enum class line_type {variable, setup, tdown, test}; struct line @@ -38,6 +39,9 @@ namespace build2 // Parse object model. // + + // redirect + // enum class redirect_type { none, @@ -85,6 +89,8 @@ namespace build2 ~redirect (); }; + // cleanup + // enum class cleanup_type { always, // &foo - cleanup, fail if does not exist. @@ -115,6 +121,8 @@ namespace build2 }; using cleanups = vector; + // command_exit + // enum class exit_comparison {eq, ne}; struct command_exit @@ -142,6 +150,8 @@ namespace build2 uint8_t status; }; + // command + // struct command { path program; @@ -169,6 +179,36 @@ namespace build2 ostream& operator<< (ostream&, const command&); + // command_pipe + // + using command_pipe = vector; + + void + to_stream (ostream&, const command_pipe&, command_to_stream); + + ostream& + operator<< (ostream&, const command_pipe&); + + // command_expr + // + enum class expr_operator {log_or, log_and}; + + struct expr_term + { + expr_operator op; // Ignored for the first term. + command_pipe pipe; + }; + + using command_expr = vector; + + void + to_stream (ostream&, const command_expr&, command_to_stream); + + ostream& + operator<< (ostream&, const command_expr&); + + // description + // struct description { string id; @@ -182,6 +222,8 @@ namespace build2 } }; + // scope + // class script; class scope @@ -254,6 +296,8 @@ namespace build2 location end_loc_; }; + // group + // class group: public scope { public: @@ -280,6 +324,8 @@ namespace build2 lines tdown_; }; + // test + // class test: public scope { public: @@ -293,6 +339,8 @@ namespace build2 lines tests_; }; + // script + // class script_base // Make sure certain things are initialized early. { protected: diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx index 77b8902..e453d44 100644 --- a/build2/test/script/script.cxx +++ b/build2/test/script/script.cxx @@ -155,6 +155,54 @@ namespace build2 } } + void + to_stream (ostream& o, const command_pipe& p, command_to_stream m) + { + if ((m & command_to_stream::header) == command_to_stream::header) + { + for (auto b (p.begin ()), i (b); i != p.end (); ++i) + { + if (i != b) + o << " | "; + + to_stream (o, *i, command_to_stream::header); + } + } + + if ((m & command_to_stream::here_doc) == command_to_stream::here_doc) + { + for (const command& c: p) + to_stream (o, c, command_to_stream::here_doc); + } + } + + void + to_stream (ostream& o, const command_expr& e, command_to_stream m) + { + if ((m & command_to_stream::header) == command_to_stream::header) + { + for (auto b (e.begin ()), i (b); i != e.end (); ++i) + { + if (i != b) + { + switch (i->op) + { + case expr_operator::log_or: o << " || "; break; + case expr_operator::log_and: o << " && "; break; + } + } + + to_stream (o, i->pipe, command_to_stream::header); + } + } + + if ((m & command_to_stream::here_doc) == command_to_stream::here_doc) + { + for (const expr_term& t: e) + to_stream (o, t.pipe, command_to_stream::here_doc); + } + } + // redirect // redirect:: diff --git a/build2/test/script/script.ixx b/build2/test/script/script.ixx index 2e215da..c156ec1 100644 --- a/build2/test/script/script.ixx +++ b/build2/test/script/script.ixx @@ -28,12 +28,33 @@ namespace build2 inline command_to_stream operator| (command_to_stream x, command_to_stream y) {return x |= y;} + + // command + // inline ostream& operator<< (ostream& o, const command& c) { to_stream (o, c, command_to_stream::all); return o; } + + // command_pipe + // + inline ostream& + operator<< (ostream& o, const command_pipe& p) + { + to_stream (o, p, command_to_stream::all); + return o; + } + + // command_pipe + // + inline ostream& + operator<< (ostream& o, const command_expr& e) + { + to_stream (o, e, command_to_stream::all); + return o; + } } } } diff --git a/doc/testscript.cli b/doc/testscript.cli index 7ed9649..e4f59e4 100644 --- a/doc/testscript.cli +++ b/doc/testscript.cli @@ -734,9 +734,12 @@ description: variable-line: ('='|'+='|'=+') value-attributes? value-attributes: '[' ']' -setup-line: '+' command -teardown-line: '-' command -test-line: command +setup-line: '+' command-expr +teardown-line: '-' command-expr +test-line: command-expr + +command-expr: command-pipe (('||'|'&&') command-pipe)* +command-pipe: command ('|' command)* command: (' '+(|redirect|cleanup))* command-exit? *here-document diff --git a/unit-tests/test/script/parser/buildfile b/unit-tests/test/script/parser/buildfile index c65dd73..60c556c 100644 --- a/unit-tests/test/script/parser/buildfile +++ b/unit-tests/test/script/parser/buildfile @@ -11,7 +11,7 @@ filesystem config/{utility init operation} dump types-parsers \ test/{target script/{token lexer parser script}} exe{driver}: cxx{driver} ../../../../build2/cxx{$src} $libs \ -test{cleanup command-re-parse description expansion here-document here-string \ - pre-parse redirect scope setup-teardown} +test{cleanup command-re-parse description exit expansion here-document \ + here-string pipe-expr pre-parse redirect scope setup-teardown} include ../../../../build2/ diff --git a/unit-tests/test/script/parser/driver.cxx b/unit-tests/test/script/parser/driver.cxx index 18fcdce..20e1e6a 100644 --- a/unit-tests/test/script/parser/driver.cxx +++ b/unit-tests/test/script/parser/driver.cxx @@ -81,9 +81,9 @@ namespace build2 } virtual void - run (scope&, const command& t, size_t, const location&) override + run (scope&, const command_expr& e, size_t, const location&) override { - cout << ind_ << t << endl; + cout << ind_ << e << endl; } virtual void diff --git a/unit-tests/test/script/parser/exit.test b/unit-tests/test/script/parser/exit.test new file mode 100644 index 0000000..263179b --- /dev/null +++ b/unit-tests/test/script/parser/exit.test @@ -0,0 +1,23 @@ +: eq +: +$* <>EOO +cmd == 1 +EOI +cmd == 1 +EOO + +: ne +: +$* <>EOO +cmd!=1 +EOI +cmd != 1 +EOO + +: end +: +$* <>EOE != 0 +cmd != 1 <"foo" +EOI +testscript:1:10: error: unexpected '<' after command exit status +EOE diff --git a/unit-tests/test/script/parser/pipe-expr.test b/unit-tests/test/script/parser/pipe-expr.test new file mode 100644 index 0000000..5a6e6ab --- /dev/null +++ b/unit-tests/test/script/parser/pipe-expr.test @@ -0,0 +1,47 @@ +: pipe +: +$* <>EOO +cmd1 | cmd2|cmd3 +EOI +cmd1 | cmd2 | cmd3 +EOO + +: log +: +$* <>EOO +cmd1 || cmd2&&cmd3 +EOI +cmd1 || cmd2 && cmd3 +EOO + +: pipe-log +: +$* <>EOO +cmd1 | cmd2 && cmd3 | cmd4 +EOI +cmd1 | cmd2 && cmd3 | cmd4 +EOO + +: exit +: +$* <>EOO +cmd1|cmd2==1&&cmd3!=0|cmd4 +EOI +cmd1 | cmd2 == 1 && cmd3 != 0 | cmd4 +EOO + +: leading +: +$* <>EOE != 0 +| cmd +EOI +testscript:1:1: error: missing program +EOE + +: trailing +: +$* <>EOE != 0 +cmd && +EOI +testscript:1:7: error: missing program +EOE -- cgit v1.1