diff options
| -rw-r--r-- | build2/parser | 2 | ||||
| -rw-r--r-- | build2/test/script/parser | 13 | ||||
| -rw-r--r-- | build2/test/script/parser.cxx | 54 | ||||
| -rw-r--r-- | build2/test/script/runner.cxx | 245 | ||||
| -rw-r--r-- | build2/test/script/script | 1 | ||||
| -rw-r--r-- | doc/testscript.cli | 5 | ||||
| -rw-r--r-- | tests/test/script/runner/buildfile | 2 | ||||
| -rw-r--r-- | tests/test/script/runner/set.test | 275 |
8 files changed, 579 insertions, 18 deletions
diff --git a/build2/parser b/build2/parser index e589948..df2797b 100644 --- a/build2/parser +++ b/build2/parser @@ -108,7 +108,7 @@ namespace build2 apply_value_attributes (const variable*, // Optional. value& lhs, value&& rhs, - token_type kind); + token_type assign_kind); // Return the value pack (values can be NULL/typed). Note that for an // empty eval context ('()' potentially with whitespaces in between) the diff --git a/build2/test/script/parser b/build2/test/script/parser index cb51042..cb536e6 100644 --- a/build2/test/script/parser +++ b/build2/test/script/parser @@ -34,6 +34,19 @@ namespace build2 void pre_parse (istream&, script&); + // Helpers. + // + // Parse attribute string and perform attribute-guided assignment. + // Issue diagnostics and throw failed in case of an error. + // + void + apply_value_attributes (const variable*, // Optional. + value& lhs, + value&& rhs, + const string& attributes, + token_type assign_kind, + const path& name); // For diagnostics. + // Recursive descent parser. // // Usually (but not always) parse functions receive the token/type diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index 59580a3..2ea42b5 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -3034,7 +3034,8 @@ namespace build2 ? scope_->assign (var) : scope_->append (var)); - apply_value_attributes (&var, lhs, move (rhs), kind); + build2::parser::apply_value_attributes ( + &var, lhs, move (rhs), kind); // If we changes any of the test.* values, then reset the $*, // $N special aliases. @@ -3203,13 +3204,23 @@ namespace build2 // only look for buildfile variables. // // Otherwise, every variable that is ever set in a script has been - // pre-entered during pre-parse. Which means that if one is not found - // in the script pool then it can only possibly be set in the - // buildfile. + // pre-entered during pre-parse or introduced with the set builtin + // during test execution. Which means that if one is not found in the + // script pool then it can only possibly be set in the buildfile. // - const variable* pvar (scope_ != nullptr - ? script_->var_pool.find (name) - : nullptr); + // Note that we need to acquire the variable pool lock. The pool can + // be changed from multiple threads by the set builtin. The obtained + // variable pointer can safelly be used with no locking as the variable + // pool is an associative container (underneath) and we are only adding + // new variables into it. + // + const variable* pvar (nullptr); + + if (scope_ != nullptr) + { + slock sl (script_->var_pool_mutex); + pvar = script_->var_pool.find (name); + } return pvar != nullptr ? scope_->find (*pvar) @@ -3269,6 +3280,35 @@ namespace build2 base_parser::lexer_ = l; } + void parser:: + apply_value_attributes (const variable* var, + value& lhs, + value&& rhs, + const string& attributes, + token_type kind, + const path& name) + { + path_ = &name; + + istringstream is (attributes); + lexer l (is, name, lexer_mode::attribute); + set_lexer (&l); + + token t; + type tt; + next (t, tt); + + if (tt != type::lsbrace && tt != type::eos) + fail (t) << "expected '[' instead of " << t; + + attributes_push (t, tt, true); + + if (tt != type::eos) + fail (t) << "trailing junk after ']'"; + + build2::parser::apply_value_attributes (var, lhs, move (rhs), kind); + } + // parser::parsed_doc // parser::parsed_doc:: diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx index bc3c1ce..e8040e2 100644 --- a/build2/test/script/runner.cxx +++ b/build2/test/script/runner.cxx @@ -10,11 +10,13 @@ #include <butl/fdstream> // fdopen_mode, fdnull(), fddup() #include <build2/regex> +#include <build2/variable> #include <build2/filesystem> #include <build2/test/common> #include <build2/test/script/regex> +#include <build2/test/script/parser> #include <build2/test/script/builtin> using namespace std; @@ -792,6 +794,203 @@ namespace build2 : sp.wd_path.directory ()); } + // The set pseudo-builtin: set variable from the stdin input. + // + // set [-e|--exact] [(-n|--newline)|(-w|--whitespace)] [<attr>] <var> + // + // -e|--exact + // Unless the option is specified, a single final newline is ignored + // in the input. + // + // -n|--newline + // Split the input into a list of elements at newlines, including a + // final blank element in case of -e. Multiple consecutive newlines + // are not collapsed. + // + // -w|--whitespace + // Split the input into a list of elements at whitespaces, including a + // final blank element in case of -e. Multiple consecutive whitespaces + // (including newlines) are collapsed. + // + // If the attr argument is specified, then it must contain a list of + // value attributes enclosed in []. + // + static void + set_builtin (scope& sp, + const strings& args, + auto_fd in, + const location& ll) + { + try + { + // Do not throw when eofbit is set (end of stream reached), and + // when failbit is set (read operation failed to extract any + // character). + // + ifdstream cin (move (in), ifdstream::badbit); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool exact (false); + bool newline (false); + bool whitespace (false); + + for (; i != e; ++i) + { + const string& o (*i); + + if (o == "-e" || o == "--exact") + exact = true; + else if (o == "-n" || o == "--newline") + newline = true; + else if (o == "-w" || o == "--whitespace") + whitespace = true; + else + { + if (*i == "--") + ++i; + + break; + } + } + + // Process arguments. + // + if (i == e) + fail (ll) << "missing variable name"; + + const string& a (*i++); // Either attributes or variable name. + const string* ats (i == e ? nullptr : &a); + const string& vname (i == e ? a : *i++); + + if (i != e) + fail (ll) << "unexpected argument"; + + if (ats != nullptr && ats->empty ()) + fail (ll) << "empty variable attributes"; + + if (vname.empty ()) + fail (ll) << "empty variable name"; + + // Read the input. + // + cin.peek (); // Sets eofbit for an empty stream. + + names ns; + while (!cin.eof ()) + { + // Read next element that depends on the whitespace mode being + // enabled or not. For the later case it also make sense to strip + // the trailing CRs that can appear while cross-testing Windows + // target or as a part of msvcrt junk production (see above). + // + string s; + if (whitespace) + cin >> s; + else + { + getline (cin, s); + + while (!s.empty () && s.back () == '\r') + s.pop_back (); + } + + // If failbit is set then we read nothing into the string as eof is + // reached. That in particular means that the stream has trailing + // whitespaces (possibly including newlines) if the whitespace mode + // is enabled, or the trailing newline otherwise. If so then + // we append the "blank" to the variable value in the exact mode + // prior to bailing out. + // + if (cin.fail ()) + { + if (exact) + { + if (whitespace || newline) + ns.emplace_back (move (s)); // Reuse empty string. + else if (ns.empty ()) + ns.emplace_back ("\n"); + else + ns[0].value += '\n'; + } + + break; + } + + if (whitespace || newline || ns.empty ()) + ns.emplace_back (move (s)); + else + { + ns[0].value += '\n'; + ns[0].value += s; + } + } + + cin.close (); + + // Set the variable value and attributes. Note that we need to aquire + // unique lock before potentially changing the script's variable + // pool. The obtained variable reference can safelly be used with no + // locking as the variable pool is an associative container + // (underneath) and we are only adding new variables into it. + // + ulock ul (sp.root->var_pool_mutex); + const variable& var (sp.root->var_pool.insert (move (vname))); + ul.unlock (); + + value& lhs (sp.assign (var)); + + // If there are no attributes specified then the variable assignment + // is straightforward. Otherwise we will use the build2 parser helper + // function. + // + if (ats == nullptr) + lhs.assign (move (ns), &var); + else + { + // Come up with a "path" that contains both the expression line + // location as well as the attributes string. The resulting + // diagnostics will look like this: + // + // testscript:10:1: ([x]):1:1: error: unknown value attribute x + // + path name; + { + string n (ll.file->string ()); + n += ':'; + + if (!ops.no_line ()) + { + n += to_string (ll.line); + n += ':'; + + if (!ops.no_column ()) + { + n += to_string (ll.column); + n += ':'; + } + } + + n += " ("; + n += *ats; + n += ')'; + name = path (move (n)); + } + + parser p; + p.apply_value_attributes( + &var, lhs, value (move (ns)), *ats, token_type::assign, name); + } + } + catch (const io_error& e) + { + fail (ll) << "set: " << e; + } + } + static bool run_pipe (scope& sp, command_pipe::const_iterator bc, @@ -864,11 +1063,14 @@ namespace build2 return normalize (move (p), sp, ll); }; + const redirect& in (c.in.effective ()); + const redirect& out (c.out.effective ()); + const redirect& err (c.err.effective ()); + // If stdin file descriptor is not open then this is the first pipeline // command. Open stdin descriptor according to the redirect specified. // path isp; - const redirect& in (c.in.effective ()); if (ifd.get () != -1) assert (in.type == redirect_type::none); // No redirect expected. @@ -973,6 +1175,40 @@ namespace build2 } } + assert (ifd.get () != -1); + + command_pipe::const_iterator nc (bc + 1); + bool last (nc == ec); + + // Prior to follow up with opening file descriptors for command + // outputs redirects let's check if the command is the set builtin. + // Being a builtin syntactically it differs from the regular ones in a + // number of ways. It either succeeds or terminates abnormally, so + // redirecting stderr is meaningless. It also never produces any output + // and may appear only as a terminal command in a pipeline. That means + // we can short-circuit here calling the builtin and returning right + // after that. Checking that the user didn't specify any meaningless + // redirects or exit code check sounds as a right thing to do. + // + if (c.program.string () == "set") + { + if (!last) + fail (ll) << "set builtin must be the last command in a pipe"; + + if (out.type != redirect_type::none) + fail (ll) << "set builtin stdout must not be redirected"; + + if (err.type != redirect_type::none) + fail (ll) << "set builtin stderr must not be redirected"; + + if ((c.exit.comparison == exit_comparison::eq) != + (c.exit.status == 0)) + fail (ll) << "set builtin exit status must not be other than zero"; + + set_builtin (sp, c.arguments, move (ifd), ll); + return true; + } + // Open a file for command output redirect if requested explicitly // (file overwrite/append redirects) or for the purpose of the output // validation (none, here_*, file comparison redirects), register the @@ -1079,7 +1315,6 @@ namespace build2 }; path osp; - const redirect& out (c.out.effective ()); auto_fd ofd; // If this is the last command in the pipeline than redirect the @@ -1096,9 +1331,6 @@ namespace build2 // test failures investigation and for tests "tightening". // fdpipe p; - command_pipe::const_iterator nc (bc + 1); - bool last (nc == ec); - if (last) ofd = open (out, 1, osp); else @@ -1118,7 +1350,6 @@ namespace build2 } path esp; - const redirect& err (c.err.effective ()); auto_fd efd (open (err, 2, esp)); // Merge standard streams. @@ -1143,7 +1374,7 @@ namespace build2 // All descriptors should be open to the date. // - assert (ifd.get () != -1 && ofd.get () != -1 && efd.get () != -1); + assert (ofd.get () != -1 && efd.get () != -1); optional<process_exit> exit; builtin* b (builtins.find (c.program.string ())); diff --git a/build2/test/script/script b/build2/test/script/script index 2438fa5..04c54db 100644 --- a/build2/test/script/script +++ b/build2/test/script/script @@ -495,6 +495,7 @@ namespace build2 public: variable_pool var_pool; + mutable shared_mutex var_pool_mutex; const variable& test_var; // test const variable& options_var; // test.options diff --git a/doc/testscript.cli b/doc/testscript.cli index 918dd4a..5b4e8ee 100644 --- a/doc/testscript.cli +++ b/doc/testscript.cli @@ -2457,8 +2457,9 @@ list of elements at newlines, including a final blank element in case of If the \c{-w|--whitespace} option is specified, then the input is split into a list of elements at whitespaces, including a final blank element in case of -\c{-e|--exact}. Multiple consecutive whitespaces (including newlines) are -collapsed. +\c{-e|--exact}. In this mode if \c{-e|--exact} is not specified, then all (and +not just newline) trailing whitespaces are ignored. Multiple consecutive +whitespaces (including newlines) are collapsed. If neither \c{-n|--newline} nor \c{-w|--whitespace} is specified, then the entire input is used as a single element, including a final newline in case diff --git a/tests/test/script/runner/buildfile b/tests/test/script/runner/buildfile index df37c6d..c3df228 100644 --- a/tests/test/script/runner/buildfile +++ b/tests/test/script/runner/buildfile @@ -2,7 +2,7 @@ # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: test{cleanup expr if pipe redirect regex status} exe{driver} $b +./: test{cleanup expr if pipe redirect regex set status} exe{driver} $b test{*}: target = exe{driver} diff --git a/tests/test/script/runner/set.test b/tests/test/script/runner/set.test new file mode 100644 index 0000000..7c24669 --- /dev/null +++ b/tests/test/script/runner/set.test @@ -0,0 +1,275 @@ +# file : tests/test/script/runner/set.test +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +.include ../common.test + +: special +: +{ + : pipelining + : + $c <'set foo | cat >bar' && $b 2>>EOE != 0 + testscript:1:1: error: set builtin must be the last command in a pipe + EOE + + : redirecting + : + { + : stdout + : + $c <'set foo >bar' && $b 2>>EOE != 0 + testscript:1:1: error: set builtin stdout must not be redirected + EOE + + : stderr + : + $c <'set foo 2>bar' && $b 2>>EOE != 0 + testscript:1:1: error: set builtin stderr must not be redirected + EOE + } + + : status + : + $c <'set foo == 1' && $b 2>>EOE != 0 + testscript:1:1: error: set builtin exit status must not be other than zero + EOE +} + +: arguments +: +{ + : none + : + $c <'set -e' && $b 2>>EOE != 0 + testscript:1:1: error: missing variable name + EOE + + : unexpected + : + $c <'set foo bar baz' && $b 2>>EOE != 0 + testscript:1:1: error: unexpected argument + EOE + + : empty-attrs + : + $c <"set '' baz" && $b 2>>EOE != 0 + testscript:1:1: error: empty variable attributes + EOE + + : empty-var + : + $c <"set ''" && $b 2>>EOE != 0 + testscript:1:1: error: empty variable name + EOE +} + +: whitespace-separated-list +: +{ + : non-exact + : + $c <<EOI && $b + set -w baz <' foo bar '; + echo $baz >'foo bar' + EOI + + : exact + : + { + : trailing-ws + : + $c <<EOI && $b + set -e -w baz <' foo bar '; + echo $baz >'foo bar ' + EOI + + : no-trailing-ws + : + : Note that we need to strip the default trailing newline as well with the + : ':' modifier. + : + $c <<EOI && $b + set -e -w baz <:' foo bar'; + echo $baz >'foo bar' + EOI + } +} + +: newline-separated-list +: +{ + : non-exact + : + $c <<EOI && $b + set -n baz <<EOF; + + foo + + bar + + EOF + echo $baz >' foo bar ' + EOI + + : exact + : + { + : trailing-newline + : + $c <<EOI && $b + set -e -n baz <<EOF; + + foo + + bar + + EOF + echo $baz >' foo bar ' + EOI + + : no-trailing-newline + : + $c <<EOI && $b + set -e -n baz <<:EOF; + + foo + + bar + EOF + echo $baz >' foo bar' + EOI + } +} + +: string +: +{ + : non-exact + : + $c <<EOI && $b + set baz <<EOF; + + foo + + bar + + EOF + echo $baz >>EOO + + foo + + bar + + EOO + EOI + + : roundtrip + : + echo 'foo' | set bar; + echo "$bar" >'foo' + + : exact + : + : Note that echo adds the trailing newline, so EOF and EOO here-documents + : differ by this newline. + : + { + : trailing-newline + : + $c <<EOI && $b + set -e baz <<EOF; + + foo + + bar + EOF + echo "$baz" >>EOO + + foo + + bar + + EOO + EOI + + : no-trailing-newline + : + $c <<EOI && $b + set -e baz <<:EOF; + + foo + + bar + EOF + echo "$baz" >>EOO + + foo + + bar + EOO + EOI + } +} + +: attributes +: +{ + : dir_path + : + $c <<EOI && $b + set [dir_path] bar <'foo'; + echo $bar >/'foo/' + EOI + + : null + : + $c <<EOI && $b + set [null] foo <-; + echo $foo >'' + EOI + + : none + : + $c <<EOI && $b 2>>EOE != 0 + set -w baz <'foo bar'; + echo "$baz" + EOI + testscript:2:8: error: concatenating variable expansion contains multiple values + EOE + + # @@ Move the following tests to build2 parser unit tests when created. + # + : empty-brackets + : + $c <<EOI && $b 2>>EOE != 0 + set -w '[]' baz <'foo bar'; + echo "$baz" + EOI + testscript:2:8: error: concatenating variable expansion contains multiple values + EOE + + : no-left-bracket + : + $c <<EOI && $b 2>>EOE != 0 + set -w x baz + EOI + testscript:1:1: (x):1:1: error: expected '[' instead of 'x' + EOE + + : unknown + : + $c <<EOI && $b 2>>EOE != 0 + set -w [x] baz + EOI + testscript:1:1: ([x]):1:1: error: unknown value attribute x + EOE + + : junk + : + $c <<EOI && $b 2>>EOE != 0 + set -w '[string] x' baz + EOI + testscript:1:1: ([string] x):1:10: error: trailing junk after ']' + EOE +} |