diff options
| -rwxr-xr-x | bootstrap.sh | 2 | ||||
| -rw-r--r-- | build2/buildfile | 3 | ||||
| -rw-r--r-- | build2/test/script/builtin | 21 | ||||
| -rw-r--r-- | build2/test/script/builtin.cxx | 293 | ||||
| -rw-r--r-- | build2/test/script/parser | 6 | ||||
| -rw-r--r-- | build2/test/script/parser.cxx | 19 | ||||
| -rw-r--r-- | build2/test/script/runner.cxx | 21 | ||||
| -rw-r--r-- | build2/types | 3 | ||||
| -rw-r--r-- | build2/utility | 16 | ||||
| -rw-r--r-- | tests/test/script/buildfile | 2 | ||||
| -rw-r--r-- | tests/test/script/builtin/buildfile | 5 | ||||
| -rw-r--r-- | tests/test/script/builtin/cat.test | 57 | ||||
| -rw-r--r-- | tests/test/script/builtin/echo.test | 11 | ||||
| -rw-r--r-- | tests/test/script/builtin/mkdir.test (renamed from tests/test/script/runner/mkdir.test) | 0 | ||||
| -rw-r--r-- | tests/test/script/builtin/touch.test (renamed from tests/test/script/runner/touch.test) | 11 | ||||
| -rw-r--r-- | tests/test/script/runner/buildfile | 5 | ||||
| -rw-r--r-- | tests/test/script/runner/redirect.test | 2 | ||||
| -rw-r--r-- | tests/test/script/runner/status.test | 4 |
18 files changed, 421 insertions, 60 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index 15410cb..3496a1f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -131,4 +131,4 @@ src="$src build2/pkgconfig/*.cxx" src="$src $libbutl/butl/*.cxx" set -x -"$cxx" "-I$libbutl" -I. '-DBUILD2_HOST_TRIPLET="'"$host"'"' -std=c++1y "$@" -o build2/b-boot $src +"$cxx" "-I$libbutl" -I. '-DBUILD2_HOST_TRIPLET="'"$host"'"' -std=c++1y "$@" -o build2/b-boot $src -lpthread diff --git a/build2/buildfile b/build2/buildfile index 86b67a9..cd04ce7 100644 --- a/build2/buildfile +++ b/build2/buildfile @@ -88,6 +88,9 @@ test/script/{hxx cxx}{ token } \ # obj{b context}: cxx.poptions += -DBUILD2_HOST_TRIPLET=\"$cxx.target\" +if ($cxx.target.class != "windows") + cxx.libs += -lpthread + # Generated options parser. # {hxx ixx cxx}{b-options}: cli{b} diff --git a/build2/test/script/builtin b/build2/test/script/builtin index e3c16b3..bd5fe50 100644 --- a/build2/test/script/builtin +++ b/build2/test/script/builtin @@ -5,11 +5,11 @@ #ifndef BUILD2_TEST_SCRIPT_BUILTIN #define BUILD2_TEST_SCRIPT_BUILTIN +#include <map> + #include <build2/types> #include <build2/utility> -#include <map> - namespace build2 { namespace test @@ -18,21 +18,26 @@ namespace build2 { class scope; + // Start builtin command. Throw system_error on failure. + // // Note that unlike argc/argv, our args don't include the program name. // - using builtin = int (*) (scope&, - const strings& args, - auto_fd in, auto_fd out, auto_fd err); + // Also note that the future object being returned doesn't block in dtor + // until the builtin command terminates. + // + using builtin = future<uint8_t> (scope&, + const strings& args, + auto_fd in, auto_fd out, auto_fd err); - class builtin_map: public std::map<string, builtin> + class builtin_map: public std::map<string, builtin*> { public: - using base = std::map<string, builtin>; + using base = std::map<string, builtin*>; using base::base; // Return NULL if not a builtin. // - builtin + builtin* find (const string& n) const { auto i (base::find (n)); diff --git a/build2/test/script/builtin.cxx b/build2/test/script/builtin.cxx index 9241447..2dcfde6 100644 --- a/build2/test/script/builtin.cxx +++ b/build2/test/script/builtin.cxx @@ -10,12 +10,22 @@ # include <sys/utime.h> #endif +#include <thread> + #include <butl/path-io> // use default operator<< implementation -#include <butl/fdstream> // fdopen_mode +#include <butl/fdstream> // fdopen_mode, fdstream_mode #include <butl/filesystem> // mkdir_status #include <build2/test/script/script> +// Strictly speaking a builtin which reads/writes from/to standard streams +// must be asynchronous so that the caller can communicate with it through +// pipes without being blocked on I/O operations. However, as an optimization, +// we allow builtins that only print diagnostics to STDERR to be synchronous +// assuming that their output will always fit the pipe buffer. Synchronous +// builtins must not read from STDIN and write to STDOUT. Later we may relax +// this rule to allow a "short" output for such builtins. +// using namespace std; using namespace butl; @@ -25,6 +35,18 @@ namespace build2 { namespace script { + using builtin_impl = uint8_t (scope&, + const strings& args, + auto_fd in, auto_fd out, auto_fd err); + static future<uint8_t> + to_future (uint8_t status) + { + promise<uint8_t> p; + future<uint8_t> f (p.get_future ()); + p.set_value (status); + return f; + } + // Operation failed, diagnostics has already been issued. // struct failed {}; @@ -48,13 +70,141 @@ namespace build2 return p; } + // Builtin commands functions. + // + + // cat <file>... + // + // Read files in sequence and write their contents to STDOUT in the same + // sequence. Read from STDIN if no argumements provided or '-' is + // specified as a file path. STDIN, STDOUT and file streams are set to + // binary mode prior to I/O operations. + // + // Note that POSIX doesn't specify if after I/O operation failure the + // command should proceed with the rest of the arguments. The current + // implementation exits immediatelly in such a case. + // + // @@ Shouldn't we check that we don't print a nonempty regular file to + // itself, as that would merely exhaust the output device? POSIX + // allows (but not requires) such a check and some implementations do + // this. That would require to fstat() file descriptors and complicate + // the code a bit. Was able to reproduce on a big file (should be + // bigger than the stream buffer size) with the test + // 'cat file >>>&file'. + // + // Note: must be executed asynchronously. + // + static uint8_t + cat (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + try + { + ifdstream cin (move (in), fdstream_mode::binary); + ofdstream cout (move (out), fdstream_mode::binary); + + // Copy input stream to STDOUT. + // + auto copy = [&cout] (istream& is) + { + if (is.peek () != ifdstream::traits_type::eof ()) + cout << is.rdbuf (); + + is.clear (istream::eofbit); // Sets eofbit. + }; + + // Path of a file being printed to STDOUT. An empty path represents + // STDIN. Used in diagnostics. + // + path p; + + try + { + // Print STDIN. + // + if (args.empty ()) + copy (cin); + + // Print files. + // + for (auto i (args.begin ()); i != args.end (); ++i) + { + if (*i == "-") + { + if (!cin.eof ()) + { + p.clear (); + copy (cin); + } + + continue; + } + + p = parse_path (*i, sp.wd_path); + + ifdstream is (p, ifdstream::binary); + copy (is); + is.close (); + } + } + catch (const io_error& e) + { + cerr << "cat: unable to print "; + + if (p.empty ()) + cerr << "stdin"; + else + cerr << "'" << p << "'"; + + cerr << ": " << e.what () << endl; + throw failed (); + } + + cin.close (); + cout.close (); + r = 0; + } + catch (const invalid_path& e) + { + cerr << "cat: invalid path '" << e.path << "'" << endl; + } + // Can be thrown while closing cin, cout or writing to cerr (that's + // why need to check its state before writing). + // + catch (const io_error& e) + { + if (cerr.good ()) + cerr << "cat: " << e.what () << endl; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + // echo <string>... // - static int - echo (scope&, const strings& args, auto_fd in, auto_fd out, auto_fd err) + // Note: must be executed asynchronously. + // + static uint8_t + echo (scope&, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept try { - int r (1); + uint8_t r (1); ofdstream cerr (move (err)); try @@ -83,6 +233,26 @@ namespace build2 return 1; } + // false + // + // Return 1. Failure to close the file descriptors is silently ignored. + // + static future<uint8_t> + false_ (scope&, const strings&, auto_fd, auto_fd, auto_fd) + { + return to_future (1); + } + + // true + // + // Return 0. Failure to close the file descriptors is silently ignored. + // + static future<uint8_t> + true_ (scope&, const strings&, auto_fd, auto_fd, auto_fd) + { + return to_future (0); + } + // Create a directory if not exist and its parent directories if // necessary. Throw system_error on failure. Register created // directories for cleanup. The directory path must be absolute. @@ -106,20 +276,19 @@ namespace build2 // Create any missing intermediate pathname components. Each argument // that names an existing directory must be ignored without error. // - static int + // Note that POSIX doesn't specify if after a directory creation failure + // the command should proceed with the rest of the arguments. The current + // implementation exits immediatelly in such a case. + // + // Note: can be executed synchronously. + // + static uint8_t mkdir (scope& sp, const strings& args, - auto_fd in, auto_fd out, auto_fd err) + auto_fd in, auto_fd out, auto_fd err) noexcept try { - // @@ Should we set a proper verbosity so paths get printed as - // relative? Can be inconvenient for end-user when build2 runs from - // a testscript. - // - // No, don't think so. If this were an external program, there - // won't be such functionality. - // - int r (1); + uint8_t r (1); ofdstream cerr (move (err)); try @@ -170,7 +339,6 @@ namespace build2 { cerr << "mkdir: unable to create directory '" << p << "': " << e.what () << endl; - throw failed (); } } @@ -181,6 +349,14 @@ namespace build2 { cerr << "mkdir: invalid path '" << e.path << "'" << endl; } + // Can be thrown while closing in, out or writing to cerr (that's why + // need to check its state before writing). + // + catch (const io_error& e) + { + if (cerr.good ()) + cerr << "mkdir: " << e.what () << endl; + } catch (const failed&) { // Diagnostics has already been issued. @@ -194,7 +370,7 @@ namespace build2 return 1; } - // touch <path>... + // touch <file>... // // Change file access and modification times to the current time. Create // a file if doesn't exist. Fail if a file system entry other than file @@ -203,13 +379,19 @@ namespace build2 // Note that POSIX doesn't specify the behavior for touching an entry // other than file. // - static int + // Also note that POSIX doesn't specify if after a file touch failure the + // command should proceed with the rest of the arguments. The current + // implementation exits immediatelly in such a case. + // + // Note: can be executed synchronously. + // + static uint8_t touch (scope& sp, const strings& args, - auto_fd in, auto_fd out, auto_fd err) + auto_fd in, auto_fd out, auto_fd err) noexcept try { - int r (1); + uint8_t r (1); ofdstream cerr (move (err)); try @@ -283,6 +465,14 @@ namespace build2 { cerr << "touch: invalid path '" << e.path << "'" << endl; } + // Can be thrown while closing in, out or writing to cerr (that's why + // need to check its state before writing). + // + catch (const io_error& e) + { + if (cerr.good ()) + cerr << "touch: " << e.what () << endl; + } catch (const failed&) { // Diagnostics has already been issued. @@ -296,11 +486,70 @@ namespace build2 return 1; } + static void + thread_thunk (builtin_impl* fn, + scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err, + promise<uint8_t> p) + { + // The use of set_value_at_thread_exit() would be more appropriate but + // the function is not supported by old versions of g++ (e.g., not in + // 4.9). There could also be overhead associated with it. + // + p.set_value (fn (sp, args, move (in), move (out), move (err))); + } + + // Run builtin implementation asynchronously. + // + static future<uint8_t> + async_impl (builtin_impl fn, + scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) + { + promise<uint8_t> p; + future<uint8_t> f (p.get_future ()); + + thread t (thread_thunk, + fn, + ref (sp), + cref (args), + move (in), move (out), move (err), + move (p)); + + t.detach (); + return f; + } + + template <builtin_impl fn> + static future<uint8_t> + async_impl (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) + { + return async_impl (fn, sp, args, move (in), move (out), move (err)); + } + + // Run builtin implementation synchronously. + // + template <builtin_impl fn> + static future<uint8_t> + sync_impl (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) + { + return to_future (fn (sp, args, move (in), move (out), move (err))); + } + const builtin_map builtins { - {"echo", &echo}, - {"mkdir", &mkdir}, - {"touch", &touch} + {"cat", &async_impl<&cat>}, + {"echo", &async_impl<&echo>}, + {"false", &false_}, + {"mkdir", &sync_impl<&mkdir>}, + {"touch", &sync_impl<&touch>}, + {"true", &true_} }; } } diff --git a/build2/test/script/parser b/build2/test/script/parser index a81ddf0..fdfbe11 100644 --- a/build2/test/script/parser +++ b/build2/test/script/parser @@ -158,6 +158,12 @@ namespace build2 const string& insert_id (string, location); + // Set lexer pointers for both the current and the base classes. + // + protected: + void + set_lexer (lexer* l); + protected: using base_parser = build2::parser; diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index 10103f3..438e1f2 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -60,8 +60,7 @@ namespace build2 pre_parse_ = true; lexer l (is, *path_, lexer_mode::script_line); - lexer_ = &l; - base_parser::lexer_ = &l; + set_lexer (&l); id_prefix_.clear (); @@ -948,8 +947,7 @@ namespace build2 path_ = &p; lexer* ol (lexer_); - lexer_ = &l; - base_parser::lexer_ = &l; + set_lexer (&l); string oip (id_prefix_); id_prefix_ += to_string (dl.line); @@ -963,8 +961,7 @@ namespace build2 fail (t) << "stray " << t; id_prefix_ = oip; - base_parser::lexer_ = ol; - lexer_ = ol; + set_lexer (ol); path_ = op; } catch (const io_error& e) @@ -2219,8 +2216,7 @@ namespace build2 pre_parse_ = false; - lexer_ = nullptr; - base_parser::lexer_ = nullptr; + set_lexer (nullptr); script_ = &s; runner_ = &r; @@ -2694,6 +2690,13 @@ namespace build2 return p.first->first; } + + void parser:: + set_lexer (lexer* l) + { + lexer_ = l; + base_parser::lexer_ = l; + } } } } diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx index 1d6920a..aa35612 100644 --- a/build2/test/script/runner.cxx +++ b/build2/test/script/runner.cxx @@ -406,7 +406,8 @@ namespace build2 ifd.reset (fdnull ()); // @@ Eventually will be throwing. if (ifd.get () == -1) // @@ TMP - throw io_error ("", error_code (errno, system_category ())); + throw io_error ( + error_code (errno, system_category ()).message ()); in = -2; } @@ -512,7 +513,8 @@ namespace build2 fd.reset (fdnull ()); // @@ Eventully will be throwing. if (fd.get () == -1) // @@ TMP - throw io_error ("", error_code (errno, system_category ())); + throw io_error ( + error_code (errno, system_category ()).message ()); } catch (const io_error& e) { @@ -590,13 +592,24 @@ namespace build2 } optional<process::status_type> status; - builtin b (builtins.find (c.program.string ())); + builtin* b (builtins.find (c.program.string ())); if (b != nullptr) { // Execute the builtin. // - status = (*b) (sp, c.arguments, move (ifd), move (ofd), move (efd)); + try + { + future<uint8_t> f ( + (*b) (sp, c.arguments, move (ifd), move (ofd), move (efd))); + + status = f.get (); + } + catch (const system_error& e) + { + fail (ll) << "unable to execute " << c.program << " builtin: " + << e.what (); + } } else { diff --git a/build2/types b/build2/types index 9b3dbc7..783aff0 100644 --- a/build2/types +++ b/build2/types @@ -8,6 +8,7 @@ #include <tuple> #include <vector> #include <string> +#include <future> #include <memory> // unique_ptr, shared_ptr #include <utility> // pair, move() #include <cstddef> // size_t, nullptr_t @@ -63,6 +64,8 @@ namespace build2 using std::istream; using std::ostream; + using std::future; + // Exceptions. While <exception> is included, there is no using for // std::exception -- use qualified. // diff --git a/build2/utility b/build2/utility index e08ae69..deefc43 100644 --- a/build2/utility +++ b/build2/utility @@ -5,12 +5,13 @@ #ifndef BUILD2_UTILITY #define BUILD2_UTILITY -#include <tuple> // make_tuple() -#include <memory> // make_shared() -#include <string> // to_string() -#include <utility> // move(), forward(), declval(), make_pair() -#include <cassert> // assert() -#include <iterator> // make_move_iterator() +#include <tuple> // make_tuple() +#include <memory> // make_shared() +#include <string> // to_string() +#include <utility> // move(), forward(), declval(), make_pair() +#include <cassert> // assert() +#include <iterator> // make_move_iterator() +#include <functional> // ref(), cref() #include <butl/utility> // combine_hash(), reverse_iterate(), casecmp(), // lcase() @@ -27,6 +28,9 @@ namespace build2 using std::forward; using std::declval; + using std::ref; + using std::cref; + using std::make_pair; using std::make_tuple; using std::make_shared; diff --git a/tests/test/script/buildfile b/tests/test/script/buildfile index e613013..82af4dd 100644 --- a/tests/test/script/buildfile +++ b/tests/test/script/buildfile @@ -2,6 +2,6 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = runner/ integration/ +d = builtin/ runner/ integration/ ./: $d include $d diff --git a/tests/test/script/builtin/buildfile b/tests/test/script/builtin/buildfile new file mode 100644 index 0000000..baa4996 --- /dev/null +++ b/tests/test/script/builtin/buildfile @@ -0,0 +1,5 @@ +# file : tests/test/script/builtin/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: test{cat echo mkdir touch} diff --git a/tests/test/script/builtin/cat.test b/tests/test/script/builtin/cat.test new file mode 100644 index 0000000..7797906 --- /dev/null +++ b/tests/test/script/builtin/cat.test @@ -0,0 +1,57 @@ +# file : tests/test/script/runner/cat.test +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +: in +: +cat <<EOI >>EOO +foo +bar +EOI +foo +bar +EOO + +: dash +: +cat - <<EOI >>EOO +foo +bar +EOI +foo +bar +EOO + +: file +: +cat <<EOI >>>out; +foo +bar +EOI +cat out >>EOO +foo +bar +EOO + +: in-repeat +: +cat - <<EOI >>EOO +foo +bar +EOI +foo +bar +EOO + +: non-existent +: +cat in 2>- != 0 # @@ REGEX + +: empty-path +: +: Cat an empty path. +: +cat '' 2>"cat: invalid path ''" == 1 + +# @@ When piping is ready test cat on a big file to test it is asynchronous. +# diff --git a/tests/test/script/builtin/echo.test b/tests/test/script/builtin/echo.test new file mode 100644 index 0000000..7f43aac --- /dev/null +++ b/tests/test/script/builtin/echo.test @@ -0,0 +1,11 @@ +# file : tests/test/script/runner/echo.test +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +: string +: +echo foo >foo + +: strings +: +echo foo bar >"foo bar" diff --git a/tests/test/script/runner/mkdir.test b/tests/test/script/builtin/mkdir.test index 6b7b5c9..6b7b5c9 100644 --- a/tests/test/script/runner/mkdir.test +++ b/tests/test/script/builtin/mkdir.test diff --git a/tests/test/script/runner/touch.test b/tests/test/script/builtin/touch.test index e9d9f68..4d2ff57 100644 --- a/tests/test/script/runner/touch.test +++ b/tests/test/script/builtin/touch.test @@ -15,12 +15,13 @@ rm a : file-update : -: Test that existing file touch doesn't register cleanup. If it did then it -: would be left dangling after 'rm' call and so test would fail. +: Test that existing file touch doesn't fail. : -$* -f a; -touch a; -rm a +cat <"" >>>a; +touch a + +# @@ How we can test that touch of an existing file doesn't register a cleanup? +# : no-args : diff --git a/tests/test/script/runner/buildfile b/tests/test/script/runner/buildfile index b9c0e69..e5f2761 100644 --- a/tests/test/script/runner/buildfile +++ b/tests/test/script/runner/buildfile @@ -4,9 +4,6 @@ import libs = libbutl%lib{butl} -exe{driver}: cxx{driver} $libs test{cleanup mkdir redirect status touch} - -if ($cxx.target.class == "windows") # @@ TMP - test{*}: ext = ".exe" +exe{driver}: cxx{driver} $libs test{cleanup redirect status} include ../../../../../build2/ diff --git a/tests/test/script/runner/redirect.test b/tests/test/script/runner/redirect.test index 68cc9aa..16c17d8 100644 --- a/tests/test/script/runner/redirect.test +++ b/tests/test/script/runner/redirect.test @@ -166,6 +166,6 @@ EOO echo <foo 1>- : in-str echo "foo" >foo : out-str echo "foo" 2>foo 1>&2 : err-str - cat <foo >foo : inout-str # @@ cat is not a builtin yet. + cat <foo >foo : inout-str cat <foo 2>foo 1>&2 : inerr-str } diff --git a/tests/test/script/runner/status.test b/tests/test/script/runner/status.test index f1ad5bf..00d7257 100644 --- a/tests/test/script/runner/status.test +++ b/tests/test/script/runner/status.test @@ -15,6 +15,10 @@ b = $build.driver -q --no-column --buildfile - <"./: test{testscript}" \ c = cat >>>testscript test = \'$test\' ++if ($cxx.target.class == "windows") + ext = ".exe" +end + # Successfull tests. # : eq-true |