From ed93e07b1b7a9e0ba99609a9223e43247ff4224e Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 18 Apr 2017 10:40:18 +0200 Subject: Implement curl process --- butl/buildfile | 2 + butl/curl | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++ butl/curl.cxx | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ butl/curl.ixx | 29 +++++++++ butl/curl.txx | 99 +++++++++++++++++++++++++++++ butl/fdstream | 7 ++ butl/process | 69 ++++++++++++++++++-- butl/process-io | 21 ++++++ butl/process-run.txx | 17 ++--- butl/sendmail | 5 +- tests/buildfile | 2 +- tests/curl/buildfile | 7 ++ tests/curl/driver.cxx | 121 +++++++++++++++++++++++++++++++++++ tests/curl/testscript | 57 +++++++++++++++++ 14 files changed, 759 insertions(+), 16 deletions(-) create mode 100644 butl/curl create mode 100644 butl/curl.cxx create mode 100644 butl/curl.ixx create mode 100644 butl/curl.txx create mode 100644 butl/process-io create mode 100644 tests/curl/buildfile create mode 100644 tests/curl/driver.cxx create mode 100644 tests/curl/testscript diff --git a/butl/buildfile b/butl/buildfile index 7d33535..328afc0 100644 --- a/butl/buildfile +++ b/butl/buildfile @@ -6,6 +6,7 @@ lib{butl}: \ {hxx cxx}{ base64 } \ {hxx cxx}{ char-scanner } \ {hxx }{ const-ptr } \ + {hxx ixx txx cxx}{ curl } \ {hxx cxx}{ diagnostics } \ {hxx }{ export } \ {hxx ixx cxx}{ fdstream } \ @@ -22,6 +23,7 @@ lib{butl}: \ {hxx txx }{ prefix-map } \ {hxx ixx cxx}{ process } \ {hxx }{ process-details } \ + {hxx }{ process-io } \ { txx cxx}{ process-run } \ {hxx ixx cxx}{ sendmail } \ {hxx cxx}{ sha256 } \ diff --git a/butl/curl b/butl/curl new file mode 100644 index 0000000..c8285ea --- /dev/null +++ b/butl/curl @@ -0,0 +1,173 @@ +// file : butl/curl -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUTL_CURL +#define BUTL_CURL + +#include +#include + +#include + +#include +#include +#include + +namespace butl +{ + // Perform a method (GET, POST, PUT) on a URL using the curl(1) program. + // Throw process_error and io_error (both derive from system_error) in case + // of errors. + // + // The I (in) and O (out) can be of the following types/values: + // + // nullfd Signal that no input/output is expected. + // + // path Read input/write output from/to a file. If the special "-" + // value is used, then instead input is connected to the curl::out + // ofdstream member and output -- to the curl::in ifdstream member. + // Note that the argument type should be path, not string (i.e., + // pass path("-")). + // + // other Forwarded as is to process_start(). Normally either int or + // auto_fd. + // + // For example: + // + // curl (nullfd, // No input expected for GET. + // path ("-"), // Write response to curl::in. + // 2, + // curl::get, + // "http://example.org"); + // + // curl (path ("-"), // Read request from curl::out. + // path::temp_path (), // Write result to a file. + // 2, + // curl::post, + // "http://example.org"); + // + // curl (nullfd, + // fdnull (), // Write result to /dev/null. + // 2, + // curl::get, + // "tftp://localhost/foo"); + // + // Typical usage: + // + // try + // { + // curl c (nullfd, // No input expected. + // path ("-"), // Output to curl::in. + // 2, // Diagnostics to stderr. + // curl::get, // GET method. + // "https://example.org", + // "-A", "foobot/1.2.3"); // Additional curl(1) options. + // + // for (string s; getline (c.in, s); ) + // cout << s << endl; + // + // c.in.close (); + // + // if (!c.wait ()) + // ... // curl returned non-zero status. + // } + // catch (const std::system_error& e) + // { + // cerr << "curl error: " << e << endl; + // } + // + // Notes: + // + // 1. If opened, in/out streams are in the binary mode. + // + // 2. If opened, in/out must be explicitly closed before calling wait(). + // + // 3. Only binary data HTTP POST is currently supported (the --data-binary + // curl option). + // + class LIBBUTL_EXPORT curl: public process + { + public: + enum method_type {get, put, post}; + + ifdstream in; + ofdstream out; + + template + curl (I&& in, + O&& out, + E&& err, + method_type, + const std::string& url, + A&&... options); + + // Version with the command line callback (see process_run() for details). + // + template + curl (const C&, + I&& in, + O&& out, + E&& err, + method_type, + const std::string& url, + A&&... options); + + private: + enum method_proto {ftp_get, ftp_put, http_get, http_post}; + using method_proto_options = small_vector; + + method_proto + translate (method_type, const std::string& url, method_proto_options&); + + private: + template + struct is_other + { + using type = typename std::remove_reference< + typename std::remove_cv::type>::type; + + static const bool value = !(std::is_same::value || + std::is_same::value); + }; + + struct io_data + { + fdpipe pipe; + method_proto_options options; + std::string storage; + }; + + int + map_in (nullfd_t, method_proto, io_data&); + + int + map_in (const path&, method_proto, io_data&); + + template + typename std::enable_if::value, I>::type + map_in (I&&, method_proto, io_data&); + + int + map_out (nullfd_t, method_proto, io_data&); + + int + map_out (const path&, method_proto, io_data&); + + template + typename std::enable_if::value, O>::type + map_out (O&&, method_proto, io_data&); + }; +} + +#include +#include + +#endif // BUTL_CURL diff --git a/butl/curl.cxx b/butl/curl.cxx new file mode 100644 index 0000000..4951b52 --- /dev/null +++ b/butl/curl.cxx @@ -0,0 +1,166 @@ +// file : butl/curl.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include // move(), forward() +#include // invalid_argument + +#include // casecmp() + +using namespace std; + +namespace butl +{ + int curl:: + map_in (nullfd_t, method_proto mp, io_data& d) + { + switch (mp) + { + case ftp_put: + throw invalid_argument ("no input specified for PUT method"); + case http_post: + throw invalid_argument ("no input specified for POST method"); + case ftp_get: + case http_get: + { + d.pipe.in.reset (fdnull ()); // /dev/null + return d.pipe.in.get (); + } + } + + return -1; + } + + int curl:: + map_in (const path& f, method_proto mp, io_data& d) + { + switch (mp) + { + case ftp_put: + case http_post: + { + if (mp == ftp_put) + { + d.options.push_back ("--upload-file"); + d.options.push_back (f.string ().c_str ()); + } + else + { + d.storage = '@' + f.string (); + + d.options.push_back ("--data-binary"); + d.options.push_back (d.storage.c_str ()); + } + + if (f.string () == "-") + { + d.pipe = fdopen_pipe (fdopen_mode::binary); + out.open (move (d.pipe.out)); + } + else + d.pipe.in.reset (fdnull ()); // /dev/null + + return d.pipe.in.get (); + } + case ftp_get: + case http_get: + { + throw invalid_argument ("file input specified for GET method"); + } + } + + return -1; + } + + int curl:: + map_out (nullfd_t, method_proto mp, io_data& d) + { + switch (mp) + { + case ftp_get: + case http_get: + throw invalid_argument ("no output specified for GET method"); + case ftp_put: + case http_post: // May or may not produce output. + { + d.pipe.out.reset (fdnull ()); + return d.pipe.out.get (); // /dev/null + } + } + + return -1; + } + + int curl:: + map_out (const path& f, method_proto mp, io_data& d) + { + switch (mp) + { + case ftp_get: + case http_get: + case http_post: + { + if (f.string () == "-") + { + // Note: no need for any options, curl writes to stdout by default. + // + d.pipe = fdopen_pipe (fdopen_mode::binary); + in.open (move (d.pipe.in)); + } + else + { + d.options.push_back ("-o"); + d.options.push_back (f.string ().c_str ()); + d.pipe.out.reset (fdnull ()); // /dev/null + } + + return d.pipe.out.get (); + } + case ftp_put: + { + throw invalid_argument ("file output specified for PUT method"); + } + } + + return -1; + } + + curl::method_proto curl:: + translate (method_type m, const string& u, method_proto_options& o) + { + size_t n (u.find ("://")); + + if (n == string::npos) + throw invalid_argument ("no protocol in URL"); + + if (casecmp (u, "ftp", n) == 0 || + casecmp (u, "tftp", n) == 0) + { + switch (m) + { + case method_type::get: return method_proto::ftp_get; + case method_type::put: return method_proto::ftp_put; + case method_type::post: + throw invalid_argument ("POST method with FTP protocol"); + } + } + else if (casecmp (u, "http", n) == 0 || + casecmp (u, "https", n) == 0) + { + o.push_back ("--fail"); // Fail on HTTP errors (e.g., 404). + o.push_back ("--location"); // Follow redirects. + + switch (m) + { + case method_type::get: return method_proto::http_get; + case method_type::post: return method_proto::http_post; + case method_type::put: + throw invalid_argument ("PUT method with HTTP protocol"); + } + } + + throw invalid_argument ("unsupported protocol"); + } +} diff --git a/butl/curl.ixx b/butl/curl.ixx new file mode 100644 index 0000000..83b388b --- /dev/null +++ b/butl/curl.ixx @@ -0,0 +1,29 @@ +// file : butl/curl.ixx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include // move(), forward() + +namespace butl +{ + template + inline curl:: + curl (I&& in, + O&& out, + E&& err, + method_type m, + const std::string& url, + A&&... options) + : curl ([] (const char* [], std::size_t) {}, + std::forward (in), + std::forward (out), + std::forward (err), + m, + url, + std::forward (options)...) + { + } +} diff --git a/butl/curl.txx b/butl/curl.txx new file mode 100644 index 0000000..fe8a25f --- /dev/null +++ b/butl/curl.txx @@ -0,0 +1,99 @@ +// file : butl/curl.txx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include // move(), forward() +#include // invalid_argument + +namespace butl +{ + template + typename std::enable_if::value, I>::type curl:: + map_in (I&& in, method_proto mp, io_data& d) + { + switch (mp) + { + case ftp_put: + { + d.options.push_back ("--upload-file"); + d.options.push_back ("-"); + break; + } + case http_post: + { + d.options.push_back ("--data-binary"); + d.options.push_back ("@-"); + break; + } + case ftp_get: + case http_get: + { + throw std::invalid_argument ("input specified for GET method"); + } + } + + return std::forward (in); + } + + template + typename std::enable_if::value, O>::type curl:: + map_out (O&& out, method_proto mp, io_data&) + { + switch (mp) + { + case ftp_get: + case http_get: + case http_post: + { + // Note: no need for any options, curl writes to stdout by default. + // + break; + } + case ftp_put: + { + throw std::invalid_argument ("output specified for PUT method"); + } + } + + return std::forward (out); + } + + template + curl:: + curl (const C& cmdc, + I&& in, + O&& out, + E&& err, + method_type m, + const std::string& url, + A&&... options) + { + method_proto_options mpo; + method_proto mp (translate (m, url, mpo)); + + io_data in_data; + io_data out_data; + + process& p (*this); + p = process_start ( + cmdc, + map_in (std::forward (in), mp, in_data), + map_out (std::forward (out), mp, out_data), + std::forward (err), + dir_path (), + "curl", + "-s", // Silent. + "-S", // But do show diagnostics. + mpo, + in_data.options, + out_data.options, + std::forward (options)..., + url); + + // Note: leaving this scope closes any open ends of the pipes in io_data. + } +} diff --git a/butl/fdstream b/butl/fdstream index 8f49990..df6b3f1 100644 --- a/butl/fdstream +++ b/butl/fdstream @@ -612,6 +612,13 @@ namespace butl { auto_fd in; auto_fd out; + + void + close () + { + in.close (); + out.close (); + } }; // Create a pipe. Throw ios::failure on the underlying OS error. By default diff --git a/butl/process b/butl/process index c138edc..282a994 100644 --- a/butl/process +++ b/butl/process @@ -9,6 +9,7 @@ # include // pid_t #endif +#include #include #include #include // uint32_t @@ -17,7 +18,9 @@ #include #include #include -#include // auto_fd, fdpipe +#include // auto_fd, fdpipe +#include +#include namespace butl { @@ -389,9 +392,8 @@ namespace butl // // The A arguments can be anything convertible to const char* via the // overloaded process_arg_as() (see below). Out of the box you can use const - // char*, std::string, path/dir_path, and numeric types. - // - // + // char*, std::string, path/dir_path, (as well as [small_]vector[_view] of + // these), and numeric types. // template inline const char* process_arg_as (const char (&s)[N], std::string&) {return s;} + + template + inline void + process_args_as (V& v, const T& x, std::string& storage) + { + v.push_back (process_arg_as (x, storage)); + } + + // [small_]vector[_view]<> + // + template + inline void + process_args_as (V& v, const std::vector& vs, std::string&) + { + for (const std::string& s: vs) + v.push_back (s.c_str ()); + } + + template + inline void + process_args_as (V& v, const small_vector& vs, std::string&) + { + for (const std::string& s: vs) + v.push_back (s.c_str ()); + } + + template + inline void + process_args_as (V& v, const vector_view& vs, std::string&) + { + for (const std::string& s: vs) + v.push_back (s.c_str ()); + } + + template + inline void + process_args_as (V& v, const std::vector& vs, std::string&) + { + for (const char* s: vs) + v.push_back (s); + } + + template + inline void + process_args_as (V& v, const small_vector& vs, std::string&) + { + for (const char* s: vs) + v.push_back (s); + } + + template + inline void + process_args_as (V& v, const vector_view& vs, std::string&) + { + for (const char* s: vs) + v.push_back (s); + } } #include diff --git a/butl/process-io b/butl/process-io new file mode 100644 index 0000000..42a720a --- /dev/null +++ b/butl/process-io @@ -0,0 +1,21 @@ +// file : butl/process-io -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUTL_PROCESS_IO +#define BUTL_PROCESS_IO + +#include + +#include + +namespace butl +{ + inline std::ostream& + operator<< (std::ostream& o, const process_path& p) + { + return o << p.recall_string (); + } +} + +#endif // BUTL_PROCESS_IO diff --git a/butl/process-run.txx b/butl/process-run.txx index 8368890..4437488 100644 --- a/butl/process-run.txx +++ b/butl/process-run.txx @@ -55,20 +55,21 @@ namespace butl // const std::size_t args_size (sizeof... (args)); + small_vector cmd; + cmd.push_back (pp.recall_string ()); + std::string storage[args_size != 0 ? args_size : 1]; - const char* cmd[args_size + 2] = { - pp.recall_string (), - process_arg_as (args, storage[index])..., - nullptr}; + const char* dummy[] = { + nullptr, (process_args_as (cmd, args, storage[index]), nullptr)... }; - // The last argument can be NULL (used to handle zero A... pack). - // - cmdc (cmd, args_size + 2); + cmd.push_back (dummy[0]); // NULL (and get rid of unused warning). + + cmdc (cmd.data (), cmd.size ()); // @@ Do we need to make sure certain fd's are closed before calling // wait()? Is this only the case with pipes? Needs thinking. - return process_start (cwd, pp, cmd, in_i, out_i, err_i); + return process_start (cwd, pp, cmd.data (), in_i, out_i, err_i); } template sendmail (const C&, diff --git a/tests/buildfile b/tests/buildfile index efc54b8..6761a42 100644 --- a/tests/buildfile +++ b/tests/buildfile @@ -2,5 +2,5 @@ # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: {*/ -sendmail/} +./: {*/ -curl/ -sendmail/} diff --git a/tests/curl/buildfile b/tests/curl/buildfile new file mode 100644 index 0000000..6b19bea --- /dev/null +++ b/tests/curl/buildfile @@ -0,0 +1,7 @@ +# file : tests/curl/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +exe{driver}: cxx{driver} ../../butl/lib{butl} test{testscript} + +include ../../butl/ diff --git a/tests/curl/driver.cxx b/tests/curl/driver.cxx new file mode 100644 index 0000000..3711e71 --- /dev/null +++ b/tests/curl/driver.cxx @@ -0,0 +1,121 @@ +// file : tests/curl/driver.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include +#include + +#include +#include // operator<<(ostream, exception) +#include + +using namespace std; +using namespace butl; + +// Usage: argv[0] tftp|http +// + +static void +print_cmd (const char* c[], std::size_t n) +{ + cerr << endl; + process::print (cerr, c, n); + cerr << endl; +} + +static void +tftp () +{ + string u ("tftp://localhost:55123/test-driver/tftp"); + + auto p = print_cmd; + + // GET non-existent. + // + { + curl c (p, nullfd, fdnull (), 2, curl::get, u + "/foo"); + assert (!c.wait ()); + } + + // PUT from file. + // + { + curl c (p, path ("foo-src"), nullfd, 2, curl::put, u + "/foo"); + assert (c.wait ()); + } + + // PUT from stream. + // + { + curl c (p, path ("-"), nullfd, 2, curl::put, u + "/bar"); + c.out << "bar" << endl; + c.out.close (); + assert (c.wait ()); + } + + // GET to stream. + // + { + curl c (p, nullfd, path ("-"), 2, curl::get, u + "/foo"); + string s; + getline (c.in, s); + c.in.close (); + assert (c.wait () && s == "foo"); + } + + // GET to /dev/null. + // + { + curl c (p, nullfd, fdnull (), 2, curl::get, u + "/foo"); + assert (c.wait ()); + } +} + +static void +http () +{ + string u ("https://build2.org"); + + auto p = print_cmd; + + // GET non-existent. + // + { + curl c (p, nullfd, fdnull (), 2, curl::get, u + "/bogus"); + assert (!c.wait ()); + } + + // GET to /dev/null. + // + { + curl c (p, nullfd, fdnull (), 2, curl::get, u); + assert (c.wait ()); + } + + // POST from stream. + // + { + curl c (p, path ("-"), 1, 2, curl::post, u + "/bogus"); + c.out << "bogus" << endl; + c.out.close (); + assert (!c.wait ()); + } +} + +int +main (int argc, const char* argv[]) +try +{ + assert (argc == 2); + + string a (argv[1]); + + if (a == "tftp") tftp (); + else if (a == "http") http (); + else assert (false); +} +catch (const system_error& e) +{ + cerr << argv[0] << ':' << argv[1] << ": " << e << endl; + return 1; +} diff --git a/tests/curl/testscript b/tests/curl/testscript new file mode 100644 index 0000000..e41aeb6 --- /dev/null +++ b/tests/curl/testscript @@ -0,0 +1,57 @@ +# file : tests/curl/testscript +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +#\ + +TFTP server (tftp-hpa) setup: from the test out_base, run (sudo is required +for --secure/chroot): + +sudo /usr/sbin/in.tftpd \ + --foreground \ + --address 127.0.0.1:55123 \ + --user "$(whoami)" \ + --permissive \ + --create \ + --secure \ + "$(pwd)" + +#\ + +: tftp +: +{ + echo 'foo' >=foo-src; + + $* 'tftp' &foo &bar 2>>EOE; + + curl -s -S tftp://localhost:55123/test-driver/tftp/foo + curl: (68) TFTP: File Not Found + + curl -s -S --upload-file foo-src tftp://localhost:55123/test-driver/tftp/foo + + curl -s -S --upload-file - tftp://localhost:55123/test-driver/tftp/bar + + curl -s -S tftp://localhost:55123/test-driver/tftp/foo + + curl -s -S tftp://localhost:55123/test-driver/tftp/foo + EOE + + diff -u foo-src foo; + diff -u - bar <'bar' +} + +: http +: +{ + $* 'http' 2>>EOE + + curl -s -S --fail --location https://build2.org/bogus + curl: (22) The requested URL returned error: 404 Not Found + + curl -s -S --fail --location https://build2.org + + curl -s -S --fail --location --data-binary @- https://build2.org/bogus + curl: (22) The requested URL returned error: 404 Not Found + EOE +} -- cgit v1.1