aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2020-02-10 22:09:59 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2020-02-12 16:49:44 +0300
commitd712f69f5f769c5eaada3c1f76419abfb7b6f7ed (patch)
treea7b36e3e87ce5ef71610f864645dbd7bc339a997
parent5524dcb9a9b10a65e450458eb76c84451a283ccc (diff)
Add builtins support for $process.run*() functions
-rw-r--r--libbuild2/functions-process.cxx386
-rw-r--r--libbuild2/test/script/runner.cxx53
-rw-r--r--libbuild2/types.hxx1
-rw-r--r--libbuild2/utility.cxx25
-rw-r--r--libbuild2/utility.hxx8
-rw-r--r--tests/function/process/testscript144
6 files changed, 468 insertions, 149 deletions
diff --git a/libbuild2/functions-process.cxx b/libbuild2/functions-process.cxx
index ce12782..3fa7819 100644
--- a/libbuild2/functions-process.cxx
+++ b/libbuild2/functions-process.cxx
@@ -2,6 +2,7 @@
// license : MIT; see accompanying LICENSE file
#include <libbutl/regex.mxx>
+#include <libbutl/builtin.mxx>
#include <libbuild2/function.hxx>
#include <libbuild2/variable.hxx>
@@ -17,7 +18,190 @@ namespace build2
// - Mode to ignore error/suppress diagnostics and return NULL?
// - Similar regex flags to regex.* functions (icase, etc)?
- // Process arguments.
+ // Convert the program (builtin or process) arguments from names to strings.
+ // The function name is only used for diagnostics.
+ //
+ static inline strings
+ program_args (names&& args, const char* fn)
+ {
+ try
+ {
+ return convert<strings> (move (args));
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "invalid process." << fn << "() argument: " << e << endf;
+ }
+ }
+
+ // Read text from a stream, trim it and return as a value. Throw io_error on
+ // the stream reading error.
+ //
+ static value
+ read (auto_fd&& fd)
+ {
+ string v;
+ ifdstream is (move (fd));
+
+ // Note that getline() will fail if there is no data.
+ //
+ if (is.peek () != ifdstream::traits_type::eof ())
+ getline (is, v, '\0');
+
+ is.close (); // Detect errors.
+
+ names r;
+ r.push_back (to_name (move (trim (v))));
+ return value (move (r));
+ }
+
+ regex
+ parse_regex (const string&, regex::flag_type); // functions-regex.cxx
+
+ // Read lines from a stream, match them against a regular expression, and
+ // return the list of matched lines or their replacements, if the format is
+ // specified. Throw invalid_argument on the regex parsing error and io_error
+ // on the stream reading error.
+ //
+ static value
+ read_regex (auto_fd&& fd, const string& pat, const optional<string>& fmt)
+ {
+ names r;
+ ifdstream is (move (fd), fdstream_mode::skip, ifdstream::badbit);
+
+ // Note that the stream is read out (and is silently closed) if
+ // invalid_argument is thrown, which is probably ok since this is not a
+ // common case.
+ //
+ regex re (parse_regex (pat, regex::ECMAScript));
+
+ for (string l; !eof (getline (is, l)); )
+ {
+ if (fmt)
+ {
+ pair<string, bool> p (regex_replace_match (l, re, *fmt));
+
+ if (p.second)
+ r.push_back (to_name (move (p.first)));
+ }
+ else if (regex_match (l, re))
+ r.push_back (to_name (move (l)));
+ }
+
+ is.close (); // Detect errors.
+
+ return value (move (r));
+ }
+
+ // Return the builtin function pointer if this is a call to a builtin and
+ // NULL otherwise.
+ //
+ static builtin_function*
+ builtin (const names& args)
+ {
+ if (args.empty ())
+ return nullptr;
+
+ const name& nm (args[0]);
+ if (!nm.simple () || nm.pair)
+ return nullptr;
+
+ return builtins.find (nm.value);
+ }
+
+ // Return the builtin name and its arguments. The builtin function is only
+ // used to make sure that args have been checked with the builtin()
+ // predicate.
+ //
+ static pair<string, strings>
+ builtin_args (builtin_function*, names&& args, const char* fn)
+ {
+ string bn (move (args[0].value));
+ args.erase (args.begin (), args.begin () + 1);
+ return pair<string, strings> (move (bn), program_args (move (args), fn));
+ }
+
+ // Read data from a stream, optionally processing it and returning the
+ // result as a value.
+ //
+ using read_function = function<value (auto_fd&&)>;
+
+ // Run a builtin. The builtin name is only used for diagnostics.
+ //
+ static value
+ run_builtin_impl (builtin_function* bf,
+ const strings& args,
+ const string& bn,
+ const read_function& read)
+ {
+ try
+ {
+ dir_path cwd;
+ builtin_callbacks cb;
+ fdpipe ofd (open_pipe ());
+
+ uint8_t rs; // Storage.
+ butl::builtin b (bf (rs,
+ args,
+ nullfd /* stdin */,
+ move (ofd.out) /* stdout */,
+ nullfd /* stderr */,
+ cwd,
+ cb));
+
+ try
+ {
+ value r (read (move (ofd.in)));
+
+ if (b.wait () == 0)
+ return r;
+
+ // Fall through.
+ //
+ }
+ catch (const io_error& e)
+ {
+ // If the builtin has failed then assume the io error was caused by
+ // that and so fall through.
+ //
+ if (b.wait () == 0)
+ fail << "io error reading " << bn << " builtin output: " << e;
+ }
+
+ // While assuming that the builtin has issued the diagnostics on failure
+ // we still print the error message (see process_finish() for details).
+ //
+ fail << bn << " builtin " << process_exit (rs) << endf;
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to execute " << bn << " builtin: " << e << endf;
+ }
+ }
+
+ static inline value
+ run_builtin (builtin_function* bf, const strings& args, const string& bn)
+ {
+ return run_builtin_impl (bf, args, bn, read);
+ }
+
+ static inline value
+ run_builtin_regex (builtin_function* bf,
+ const strings& args,
+ const string& bn,
+ const string& pat,
+ const optional<string>& fmt)
+ {
+ // Note that we rely on the "small function object" optimization here.
+ //
+ return run_builtin_impl (bf, args, bn,
+ [&pat, &fmt] (auto_fd&& fd)
+ {
+ return read_regex (move (fd), pat, fmt);
+ });
+ }
+
+ // Return the process path and its arguments.
//
static pair<process_path, strings>
process_args (names&& args, const char* fn)
@@ -39,7 +223,26 @@ namespace build2
}
else
{
- pp = run_search (convert<path> (move (args[0])));
+ // Strip the builtin-escaping '^' character, if present.
+ //
+ path p (convert<path> (move (args[0])));
+
+ if (p.simple ())
+ try
+ {
+ const string& s (p.string ());
+
+ // Don't end up with an empty path.
+ //
+ if (s.size () > 1 && s[0] == '^')
+ p = path (s, 1, s.size () - 1);
+ }
+ catch (const invalid_path& e)
+ {
+ throw invalid_argument (e.path);
+ }
+
+ pp = run_search (p);
erase = 1;
}
@@ -47,27 +250,18 @@ namespace build2
}
catch (const invalid_argument& e)
{
- fail << "invalid process." << fn << "() executable path: " << e.what ();
+ fail << "invalid process." << fn << "() executable path: " << e;
}
- strings sargs;
- try
- {
- sargs = convert<strings> (move (args));
- }
- catch (const invalid_argument& e)
- {
- fail << "invalid process." << fn << "() argument: " << e.what ();
- }
-
- return pair<process_path, strings> (move (pp), move (sargs));
+ return pair<process_path, strings> (move (pp),
+ program_args (move (args), fn));
}
static process
- run_start (const scope*,
- const process_path& pp,
- const strings& args,
- cstrings& cargs)
+ process_start (const scope*,
+ const process_path& pp,
+ const strings& args,
+ cstrings& cargs)
{
cargs.reserve (args.size () + 2);
cargs.push_back (pp.recall_string ());
@@ -84,77 +278,41 @@ namespace build2
-1 /* stdout */);
}
- static value
- run (const scope* s, const process_path& pp, const strings& args)
+ // Always issue diagnostics on process failure, regardless if the process
+ // exited abnormally or normally with non-zero exit code.
+ //
+ // Note that the diagnostics stack is only printed if a diagnostics record
+ // is created, which is not always the case for run_finish().
+ //
+ void
+ process_finish (const scope*, const cstrings& args, process& pr)
{
- cstrings cargs;
- process pr (run_start (s, pp, args, cargs));
-
- string v;
try
{
- ifdstream is (move (pr.in_ofd));
-
- // Note that getline() will fail if there is no output.
- //
- if (is.peek () != ifdstream::traits_type::eof ())
- getline (is, v, '\0');
-
- is.close (); // Detect errors.
+ if (!pr.wait ())
+ fail << "process " << args[0] << " " << *pr.exit;
}
- catch (const io_error& e)
+ catch (const process_error& e)
{
- if (run_wait (cargs, pr))
- fail << "io error reading " << cargs[0] << " output: " << e;
-
- // If the child process has failed then assume the io error was
- // caused by that and let run_finish() deal with it.
+ fail << "unable to execute " << args[0] << ": " << e;
}
-
- run_finish (cargs, pr);
-
- names r;
- r.push_back (to_name (move (trim (v))));
- return value (move (r));
}
- regex
- parse_regex (const string&, regex::flag_type); // functions-regex.cxx
-
+ // Run a process.
+ //
static value
- run_regex (const scope* s,
- const process_path& pp,
- const strings& args,
- const string& pat,
- const optional<string>& fmt)
+ run_process_impl (const scope* s,
+ const process_path& pp,
+ const strings& args,
+ const read_function& read)
{
- regex re (parse_regex (pat, regex::ECMAScript));
-
cstrings cargs;
- process pr (run_start (s, pp, args, cargs));
+ process pr (process_start (s, pp, args, cargs));
- names r;
+ value r;
try
{
- ifdstream is (move (pr.in_ofd), ifdstream::badbit);
-
- for (string l; !eof (getline (is, l)); )
- {
- if (fmt)
- {
- pair<string, bool> p (regex_replace_match (l, re, *fmt));
-
- if (p.second)
- r.push_back (to_name (move (p.first)));
- }
- else
- {
- if (regex_match (l, re))
- r.push_back (to_name (move (l)));
- }
- }
-
- is.close (); // Detect errors.
+ r = read (move (pr.in_ofd));
}
catch (const io_error& e)
{
@@ -162,12 +320,48 @@ namespace build2
fail << "io error reading " << cargs[0] << " output: " << e;
// If the child process has failed then assume the io error was
- // caused by that and let run_finish() deal with it.
+ // caused by that and let process_finish() deal with it.
}
- run_finish (cargs, pr);
+ process_finish (s, cargs, pr);
+ return r;
+ }
- return value (move (r));
+ static inline value
+ run_process (const scope* s, const process_path& pp, const strings& args)
+ {
+ return run_process_impl (s, pp, args, read);
+ }
+
+ static inline value
+ run_process_regex (const scope* s,
+ const process_path& pp,
+ const strings& args,
+ const string& pat,
+ const optional<string>& fmt)
+ {
+ // Note that we rely on the "small function object" optimization here.
+ //
+ return run_process_impl (s, pp, args,
+ [&pat, &fmt] (auto_fd&& fd)
+ {
+ return read_regex (move (fd), pat, fmt);
+ });
+ }
+
+ static inline value
+ run (const scope* s, names&& args)
+ {
+ if (builtin_function* bf = builtin (args))
+ {
+ pair<string, strings> ba (builtin_args (bf, move (args), "run"));
+ return run_builtin (bf, ba.second, ba.first);
+ }
+ else
+ {
+ pair<process_path, strings> pa (process_args (move (args), "run"));
+ return run_process (s, pa.first, pa.second);
+ }
}
static inline value
@@ -176,8 +370,18 @@ namespace build2
const string& pat,
const optional<string>& fmt)
{
- pair<process_path, strings> pa (process_args (move (args), "run_regex"));
- return run_regex (s, pa.first, pa.second, pat, fmt);
+ if (builtin_function* bf = builtin (args))
+ {
+ pair<string, strings> ba (builtin_args (bf, move (args), "run_regex"));
+ return run_builtin_regex (bf, ba.second, ba.first, pat, fmt);
+ }
+ else
+ {
+ pair<process_path, strings> pa (process_args (move (args),
+ "run_regex"));
+
+ return run_process_regex (s, pa.first, pa.second, pat, fmt);
+ }
}
void
@@ -187,22 +391,22 @@ namespace build2
// $process.run(<prog>[ <args>...])
//
- // Return trimmed stdout.
+ // Run builtin or external program and return trimmed stdout.
//
f[".run"] = [](const scope* s, names args)
{
- pair<process_path, strings> pa (process_args (move (args), "run"));
- return run (s, pa.first, pa.second);
+ return run (s, move (args));
};
f["run"] = [](const scope* s, process_path pp)
{
- return run (s, pp, strings ());
+ return run_process (s, pp, strings ());
};
// $process.run_regex(<prog>[ <args>...], <pat> [, <fmt>])
//
- // Return stdout lines matched and optionally processed with regex.
+ // Run builtin or external program and return stdout lines matched and
+ // optionally processed with regex.
//
// Each line of stdout (including the customary trailing blank) is matched
// (as a whole) against <pat> and, if successful, returned, optionally
@@ -226,7 +430,7 @@ namespace build2
string p,
optional<string> f)
{
- return run_regex (s, pp, strings (), p, f);
+ return run_process_regex (s, pp, strings (), p, f);
};
f["run_regex"] = [](const scope* s,
@@ -234,10 +438,12 @@ namespace build2
names p,
optional<names> f)
{
- return run_regex (s,
- pp, strings (),
- convert<string> (move (p)),
- f ? convert<string> (move (*f)) : nullopt_string);
+ return run_process_regex (s,
+ pp, strings (),
+ convert<string> (move (p)),
+ (f
+ ? convert<string> (move (*f))
+ : nullopt_string));
};
}
}
diff --git a/libbuild2/test/script/runner.cxx b/libbuild2/test/script/runner.cxx
index 535919c..b006f3e 100644
--- a/libbuild2/test/script/runner.cxx
+++ b/libbuild2/test/script/runner.cxx
@@ -7,7 +7,7 @@
#include <libbutl/regex.mxx>
#include <libbutl/builtin.mxx>
-#include <libbutl/fdstream.mxx> // fdopen_mode, fdnull(), fddup()
+#include <libbutl/fdstream.mxx> // fdopen_mode, fddup()
#include <libbutl/filesystem.mxx> // path_search()
#include <libbutl/path-pattern.mxx>
@@ -1352,15 +1352,7 @@ namespace build2
//
case redirect_type::null:
{
- try
- {
- ifd = fdnull ();
- }
- catch (const io_error& e)
- {
- fail (ll) << "unable to write to null device: " << e;
- }
-
+ ifd = open_dev_null ();
break;
}
@@ -1388,7 +1380,6 @@ namespace build2
open_stdin ();
break;
}
-
case redirect_type::trace:
case redirect_type::merge:
case redirect_type::here_str_regex:
@@ -1445,7 +1436,6 @@ namespace build2
fdopen_mode m (fdopen_mode::out | fdopen_mode::create);
- auto_fd fd;
redirect_type rt (r.type != redirect_type::trace
? r.type
: verb < 2
@@ -1457,36 +1447,19 @@ namespace build2
{
try
{
- fd = fddup (dfd);
+ return fddup (dfd);
}
catch (const io_error& e)
{
fail (ll) << "unable to duplicate " << what << ": " << e;
}
-
- return fd;
}
- case redirect_type::null:
- {
- try
- {
- fd = fdnull ();
- }
- catch (const io_error& e)
- {
- fail (ll) << "unable to write to null device: " << e;
- }
-
- return fd;
- }
+ case redirect_type::null: return open_dev_null ();
- case redirect_type::merge:
- {
- // Duplicate the paired file descriptor later.
- //
- return fd; // nullfd
- }
+ // Duplicate the paired file descriptor later.
+ //
+ case redirect_type::merge: return nullfd;
case redirect_type::file:
{
@@ -1521,6 +1494,8 @@ namespace build2
case redirect_type::here_doc_ref: assert (false); break;
}
+ auto_fd fd;
+
try
{
fd = fdopen (p, m);
@@ -1562,15 +1537,7 @@ namespace build2
else
{
assert (out.type == redirect_type::none); // No redirect expected.
-
- try
- {
- ofd = fdopen_pipe ();
- }
- catch (const io_error& e)
- {
- fail (ll) << "unable to open pipe: " << e;
- }
+ ofd = open_pipe ();
}
path esp;
diff --git a/libbuild2/types.hxx b/libbuild2/types.hxx
index 2fefad3..e24c9a6 100644
--- a/libbuild2/types.hxx
+++ b/libbuild2/types.hxx
@@ -284,6 +284,7 @@ namespace build2
// <libbutl/fdstream.mxx>
//
using butl::auto_fd;
+ using butl::fdpipe;
using butl::ifdstream;
using butl::ofdstream;
using butl::fdopen_mode;
diff --git a/libbuild2/utility.cxx b/libbuild2/utility.cxx
index 7fa0ef1..faffd66 100644
--- a/libbuild2/utility.cxx
+++ b/libbuild2/utility.cxx
@@ -313,6 +313,31 @@ namespace build2
fail << "io error reading " << args[0] << " output: " << e << endf;
}
+ fdpipe
+ open_pipe ()
+ {
+ try
+ {
+ return fdopen_pipe ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to open pipe: " << e << endf;
+ }
+ }
+
+ auto_fd
+ open_dev_null ()
+ {
+ try
+ {
+ return fdnull ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to open null device: " << e << endf;
+ }
+ }
const string empty_string;
const path empty_path;
diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx
index 18122e0..dd4b78b 100644
--- a/libbuild2/utility.hxx
+++ b/libbuild2/utility.hxx
@@ -516,6 +516,14 @@ namespace build2
verbosity, pe, args, forward<F> (f), error, ignore_exit, checksum);
}
+ // File descriptor streams.
+ //
+ fdpipe
+ open_pipe ();
+
+ auto_fd
+ open_dev_null ();
+
// Empty/nullopt string, path, and project name.
//
LIBBUILD2_SYMEXPORT extern const string empty_string;
diff --git a/tests/function/process/testscript b/tests/function/process/testscript
index eaa8e2e..1da733d 100644
--- a/tests/function/process/testscript
+++ b/tests/function/process/testscript
@@ -5,25 +5,137 @@
: run
:
-$* <<EOI >>~/EOO/
-print $process.run($build.path --version)
-EOI
-/build2 .+/
-/.+/*
-EOO
+{
+ : process
+ :
+ {
+ $* <<EOI >>~/EOO/
+ print $process.run($build.path --version)
+ EOI
+ /build2 .+/
+ /.+/*
+ EOO
+ }
+
+ : bultin
+ :
+ {
+ echo 'abc' >=f;
+
+ $* <<EOI >>EOO
+ print $process.run(sed -e 's/abc/xyz/' f)
+ EOI
+ xyz
+ EOO
+ }
+
+ : escape-bultin
+ :
+ if ($cxx.target.class == 'linux')
+ {
+ cat <<EOI >=f;
+ a
+
+
+ b
+ EOI
+
+ $* <<EOI >>EOO
+ print $process.run(^cat --squeeze-blank f)
+ EOI
+ a
+
+ b
+ EOO
+ }
+}
: run-regex-match
:
-$* <<EOI >>~/EOO/
-print $process.run_regex($build.path --version, 'build2 .+')
-EOI
-/build2 .+/
-EOO
+{
+ : process
+ :
+ {
+ : success
+ :
+ {
+ $* <<EOI >>~/EOO/
+ print $process.run_regex($build.path --version, 'build2 .+')
+ EOI
+ /build2 .+/
+ EOO
+ }
+
+ : failure
+ :
+ {
+ $* <<EOI 2>>~/EOE/ != 0
+ print $process.run_regex($build.path --version, 'build2 (.+')
+ EOI
+ /error: invalid argument: invalid regex .+/
+ /.+
+ EOE
+ }
+ }
+
+ : builtin
+ :
+ {
+ : success
+ :
+ {
+ cat <<EOI >=f;
+ 123
+ abc
+ EOI
+
+ $* <<EOI >>EOO
+ print $process.run_regex(cat f, 'a.+')
+ EOI
+ abc
+ EOO
+ }
+
+ : failure
+ :
+ {
+ echo 'a' >=f;
+
+ $* <<EOI 2>>~/EOE/ != 0
+ print $process.run_regex(cat f, 'a(.+')
+ EOI
+ /error: invalid argument: invalid regex .+/
+ /.+
+ EOE
+ }
+ }
+}
: run-regex-replace
:
-$* <<EOI >>~/EOO/
-print $process.run_regex($build.path --version, 'build2 ([0-9.]+).*', '\1')
-EOI
-/[0-9]+.[0-9]+.[0-9]+/d
-EOO
+{
+ : process
+ :
+ {
+ $* <<EOI >>~/EOO/
+ print $process.run_regex($build.path --version, 'build2 ([0-9.]+).*', '\1')
+ EOI
+ /[0-9]+.[0-9]+.[0-9]+/d
+ EOO
+ }
+
+ : builtin
+ :
+ {
+ cat <<EOI >=f;
+ 123
+ abc
+ EOI
+
+ $* <<EOI >>EOO
+ print $process.run_regex(cat f, 'a(.+)', 'x\1')
+ EOI
+ xbc
+ EOO
+ }
+}