aboutsummaryrefslogtreecommitdiff
path: root/butl
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2017-04-18 10:40:18 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2017-04-18 10:40:18 +0200
commited93e07b1b7a9e0ba99609a9223e43247ff4224e (patch)
treeaa203bdab5a5fc4f5fd8af16baf6903a7ee3dde0 /butl
parent4408607c51a7c6e293adae41403b21d4a2c9a429 (diff)
Implement curl process
Diffstat (limited to 'butl')
-rw-r--r--butl/buildfile2
-rw-r--r--butl/curl173
-rw-r--r--butl/curl.cxx166
-rw-r--r--butl/curl.ixx29
-rw-r--r--butl/curl.txx99
-rw-r--r--butl/fdstream7
-rw-r--r--butl/process69
-rw-r--r--butl/process-io21
-rw-r--r--butl/process-run.txx17
-rw-r--r--butl/sendmail5
10 files changed, 573 insertions, 15 deletions
diff --git a/butl/buildfile b/butl/buildfile
index 7d33535..328afc0 100644
--- a/butl/buildfile
+++ b/butl/buildfile
@@ -6,6 +6,7 @@ lib{butl}: \
{hxx cxx}{ base64 } \
{hxx cxx}{ char-scanner } \
{hxx }{ const-ptr } \
+ {hxx ixx txx cxx}{ curl } \
{hxx cxx}{ diagnostics } \
{hxx }{ export } \
{hxx ixx cxx}{ fdstream } \
@@ -22,6 +23,7 @@ lib{butl}: \
{hxx txx }{ prefix-map } \
{hxx ixx cxx}{ process } \
{hxx }{ process-details } \
+ {hxx }{ process-io } \
{ txx cxx}{ process-run } \
{hxx ixx cxx}{ sendmail } \
{hxx cxx}{ sha256 } \
diff --git a/butl/curl b/butl/curl
new file mode 100644
index 0000000..c8285ea
--- /dev/null
+++ b/butl/curl
@@ -0,0 +1,173 @@
+// file : butl/curl -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BUTL_CURL
+#define BUTL_CURL
+
+#include <string>
+#include <type_traits>
+
+#include <butl/export>
+
+#include <butl/process>
+#include <butl/fdstream>
+#include <butl/small-vector>
+
+namespace butl
+{
+ // Perform a method (GET, POST, PUT) on a URL using the curl(1) program.
+ // Throw process_error and io_error (both derive from system_error) in case
+ // of errors.
+ //
+ // The I (in) and O (out) can be of the following types/values:
+ //
+ // nullfd Signal that no input/output is expected.
+ //
+ // path Read input/write output from/to a file. If the special "-"
+ // value is used, then instead input is connected to the curl::out
+ // ofdstream member and output -- to the curl::in ifdstream member.
+ // Note that the argument type should be path, not string (i.e.,
+ // pass path("-")).
+ //
+ // other Forwarded as is to process_start(). Normally either int or
+ // auto_fd.
+ //
+ // For example:
+ //
+ // curl (nullfd, // No input expected for GET.
+ // path ("-"), // Write response to curl::in.
+ // 2,
+ // curl::get,
+ // "http://example.org");
+ //
+ // curl (path ("-"), // Read request from curl::out.
+ // path::temp_path (), // Write result to a file.
+ // 2,
+ // curl::post,
+ // "http://example.org");
+ //
+ // curl (nullfd,
+ // fdnull (), // Write result to /dev/null.
+ // 2,
+ // curl::get,
+ // "tftp://localhost/foo");
+ //
+ // Typical usage:
+ //
+ // try
+ // {
+ // curl c (nullfd, // No input expected.
+ // path ("-"), // Output to curl::in.
+ // 2, // Diagnostics to stderr.
+ // curl::get, // GET method.
+ // "https://example.org",
+ // "-A", "foobot/1.2.3"); // Additional curl(1) options.
+ //
+ // for (string s; getline (c.in, s); )
+ // cout << s << endl;
+ //
+ // c.in.close ();
+ //
+ // if (!c.wait ())
+ // ... // curl returned non-zero status.
+ // }
+ // catch (const std::system_error& e)
+ // {
+ // cerr << "curl error: " << e << endl;
+ // }
+ //
+ // Notes:
+ //
+ // 1. If opened, in/out streams are in the binary mode.
+ //
+ // 2. If opened, in/out must be explicitly closed before calling wait().
+ //
+ // 3. Only binary data HTTP POST is currently supported (the --data-binary
+ // curl option).
+ //
+ class LIBBUTL_EXPORT curl: public process
+ {
+ public:
+ enum method_type {get, put, post};
+
+ ifdstream in;
+ ofdstream out;
+
+ template <typename I,
+ typename O,
+ typename E,
+ typename... A>
+ curl (I&& in,
+ O&& out,
+ E&& err,
+ method_type,
+ const std::string& url,
+ A&&... options);
+
+ // Version with the command line callback (see process_run() for details).
+ //
+ template <typename C,
+ typename I,
+ typename O,
+ typename E,
+ typename... A>
+ curl (const C&,
+ I&& in,
+ O&& out,
+ E&& err,
+ method_type,
+ const std::string& url,
+ A&&... options);
+
+ private:
+ enum method_proto {ftp_get, ftp_put, http_get, http_post};
+ using method_proto_options = small_vector<const char*, 2>;
+
+ method_proto
+ translate (method_type, const std::string& url, method_proto_options&);
+
+ private:
+ template <typename T>
+ struct is_other
+ {
+ using type = typename std::remove_reference<
+ typename std::remove_cv<T>::type>::type;
+
+ static const bool value = !(std::is_same<type, nullfd_t>::value ||
+ std::is_same<type, path>::value);
+ };
+
+ struct io_data
+ {
+ fdpipe pipe;
+ method_proto_options options;
+ std::string storage;
+ };
+
+ int
+ map_in (nullfd_t, method_proto, io_data&);
+
+ int
+ map_in (const path&, method_proto, io_data&);
+
+ template <typename I>
+ typename std::enable_if<is_other<I>::value, I>::type
+ map_in (I&&, method_proto, io_data&);
+
+ int
+ map_out (nullfd_t, method_proto, io_data&);
+
+ int
+ map_out (const path&, method_proto, io_data&);
+
+ template <typename O>
+ typename std::enable_if<is_other<O>::value, O>::type
+ map_out (O&&, method_proto, io_data&);
+ };
+}
+
+#include <butl/curl.ixx>
+#include <butl/curl.txx>
+
+#endif // BUTL_CURL
diff --git a/butl/curl.cxx b/butl/curl.cxx
new file mode 100644
index 0000000..4951b52
--- /dev/null
+++ b/butl/curl.cxx
@@ -0,0 +1,166 @@
+// file : butl/curl.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <butl/curl>
+
+#include <utility> // move(), forward()
+#include <exception> // invalid_argument
+
+#include <butl/utility> // casecmp()
+
+using namespace std;
+
+namespace butl
+{
+ int curl::
+ map_in (nullfd_t, method_proto mp, io_data& d)
+ {
+ switch (mp)
+ {
+ case ftp_put:
+ throw invalid_argument ("no input specified for PUT method");
+ case http_post:
+ throw invalid_argument ("no input specified for POST method");
+ case ftp_get:
+ case http_get:
+ {
+ d.pipe.in.reset (fdnull ()); // /dev/null
+ return d.pipe.in.get ();
+ }
+ }
+
+ return -1;
+ }
+
+ int curl::
+ map_in (const path& f, method_proto mp, io_data& d)
+ {
+ switch (mp)
+ {
+ case ftp_put:
+ case http_post:
+ {
+ if (mp == ftp_put)
+ {
+ d.options.push_back ("--upload-file");
+ d.options.push_back (f.string ().c_str ());
+ }
+ else
+ {
+ d.storage = '@' + f.string ();
+
+ d.options.push_back ("--data-binary");
+ d.options.push_back (d.storage.c_str ());
+ }
+
+ if (f.string () == "-")
+ {
+ d.pipe = fdopen_pipe (fdopen_mode::binary);
+ out.open (move (d.pipe.out));
+ }
+ else
+ d.pipe.in.reset (fdnull ()); // /dev/null
+
+ return d.pipe.in.get ();
+ }
+ case ftp_get:
+ case http_get:
+ {
+ throw invalid_argument ("file input specified for GET method");
+ }
+ }
+
+ return -1;
+ }
+
+ int curl::
+ map_out (nullfd_t, method_proto mp, io_data& d)
+ {
+ switch (mp)
+ {
+ case ftp_get:
+ case http_get:
+ throw invalid_argument ("no output specified for GET method");
+ case ftp_put:
+ case http_post: // May or may not produce output.
+ {
+ d.pipe.out.reset (fdnull ());
+ return d.pipe.out.get (); // /dev/null
+ }
+ }
+
+ return -1;
+ }
+
+ int curl::
+ map_out (const path& f, method_proto mp, io_data& d)
+ {
+ switch (mp)
+ {
+ case ftp_get:
+ case http_get:
+ case http_post:
+ {
+ if (f.string () == "-")
+ {
+ // Note: no need for any options, curl writes to stdout by default.
+ //
+ d.pipe = fdopen_pipe (fdopen_mode::binary);
+ in.open (move (d.pipe.in));
+ }
+ else
+ {
+ d.options.push_back ("-o");
+ d.options.push_back (f.string ().c_str ());
+ d.pipe.out.reset (fdnull ()); // /dev/null
+ }
+
+ return d.pipe.out.get ();
+ }
+ case ftp_put:
+ {
+ throw invalid_argument ("file output specified for PUT method");
+ }
+ }
+
+ return -1;
+ }
+
+ curl::method_proto curl::
+ translate (method_type m, const string& u, method_proto_options& o)
+ {
+ size_t n (u.find ("://"));
+
+ if (n == string::npos)
+ throw invalid_argument ("no protocol in URL");
+
+ if (casecmp (u, "ftp", n) == 0 ||
+ casecmp (u, "tftp", n) == 0)
+ {
+ switch (m)
+ {
+ case method_type::get: return method_proto::ftp_get;
+ case method_type::put: return method_proto::ftp_put;
+ case method_type::post:
+ throw invalid_argument ("POST method with FTP protocol");
+ }
+ }
+ else if (casecmp (u, "http", n) == 0 ||
+ casecmp (u, "https", n) == 0)
+ {
+ o.push_back ("--fail"); // Fail on HTTP errors (e.g., 404).
+ o.push_back ("--location"); // Follow redirects.
+
+ switch (m)
+ {
+ case method_type::get: return method_proto::http_get;
+ case method_type::post: return method_proto::http_post;
+ case method_type::put:
+ throw invalid_argument ("PUT method with HTTP protocol");
+ }
+ }
+
+ throw invalid_argument ("unsupported protocol");
+ }
+}
diff --git a/butl/curl.ixx b/butl/curl.ixx
new file mode 100644
index 0000000..83b388b
--- /dev/null
+++ b/butl/curl.ixx
@@ -0,0 +1,29 @@
+// file : butl/curl.ixx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <utility> // move(), forward()
+
+namespace butl
+{
+ template <typename I,
+ typename O,
+ typename E,
+ typename... A>
+ inline curl::
+ curl (I&& in,
+ O&& out,
+ E&& err,
+ method_type m,
+ const std::string& url,
+ A&&... options)
+ : curl ([] (const char* [], std::size_t) {},
+ std::forward<I> (in),
+ std::forward<O> (out),
+ std::forward<E> (err),
+ m,
+ url,
+ std::forward<A> (options)...)
+ {
+ }
+}
diff --git a/butl/curl.txx b/butl/curl.txx
new file mode 100644
index 0000000..fe8a25f
--- /dev/null
+++ b/butl/curl.txx
@@ -0,0 +1,99 @@
+// file : butl/curl.txx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <utility> // move(), forward()
+#include <exception> // invalid_argument
+
+namespace butl
+{
+ template <typename I>
+ typename std::enable_if<curl::is_other<I>::value, I>::type curl::
+ map_in (I&& in, method_proto mp, io_data& d)
+ {
+ switch (mp)
+ {
+ case ftp_put:
+ {
+ d.options.push_back ("--upload-file");
+ d.options.push_back ("-");
+ break;
+ }
+ case http_post:
+ {
+ d.options.push_back ("--data-binary");
+ d.options.push_back ("@-");
+ break;
+ }
+ case ftp_get:
+ case http_get:
+ {
+ throw std::invalid_argument ("input specified for GET method");
+ }
+ }
+
+ return std::forward<I> (in);
+ }
+
+ template <typename O>
+ typename std::enable_if<curl::is_other<O>::value, O>::type curl::
+ map_out (O&& out, method_proto mp, io_data&)
+ {
+ switch (mp)
+ {
+ case ftp_get:
+ case http_get:
+ case http_post:
+ {
+ // Note: no need for any options, curl writes to stdout by default.
+ //
+ break;
+ }
+ case ftp_put:
+ {
+ throw std::invalid_argument ("output specified for PUT method");
+ }
+ }
+
+ return std::forward<O> (out);
+ }
+
+ template <typename C,
+ typename I,
+ typename O,
+ typename E,
+ typename... A>
+ curl::
+ curl (const C& cmdc,
+ I&& in,
+ O&& out,
+ E&& err,
+ method_type m,
+ const std::string& url,
+ A&&... options)
+ {
+ method_proto_options mpo;
+ method_proto mp (translate (m, url, mpo));
+
+ io_data in_data;
+ io_data out_data;
+
+ process& p (*this);
+ p = process_start (
+ cmdc,
+ map_in (std::forward<I> (in), mp, in_data),
+ map_out (std::forward<O> (out), mp, out_data),
+ std::forward<E> (err),
+ dir_path (),
+ "curl",
+ "-s", // Silent.
+ "-S", // But do show diagnostics.
+ mpo,
+ in_data.options,
+ out_data.options,
+ std::forward<A> (options)...,
+ url);
+
+ // Note: leaving this scope closes any open ends of the pipes in io_data.
+ }
+}
diff --git a/butl/fdstream b/butl/fdstream
index 8f49990..df6b3f1 100644
--- a/butl/fdstream
+++ b/butl/fdstream
@@ -612,6 +612,13 @@ namespace butl
{
auto_fd in;
auto_fd out;
+
+ void
+ close ()
+ {
+ in.close ();
+ out.close ();
+ }
};
// Create a pipe. Throw ios::failure on the underlying OS error. By default
diff --git a/butl/process b/butl/process
index c138edc..282a994 100644
--- a/butl/process
+++ b/butl/process
@@ -9,6 +9,7 @@
# include <sys/types.h> // pid_t
#endif
+#include <vector>
#include <iosfwd>
#include <cassert>
#include <cstdint> // uint32_t
@@ -17,7 +18,9 @@
#include <butl/path>
#include <butl/export>
#include <butl/optional>
-#include <butl/fdstream> // auto_fd, fdpipe
+#include <butl/fdstream> // auto_fd, fdpipe
+#include <butl/vector-view>
+#include <butl/small-vector>
namespace butl
{
@@ -389,9 +392,8 @@ namespace butl
//
// The A arguments can be anything convertible to const char* via the
// overloaded process_arg_as() (see below). Out of the box you can use const
- // char*, std::string, path/dir_path, and numeric types.
- //
- //
+ // char*, std::string, path/dir_path, (as well as [small_]vector[_view] of
+ // these), and numeric types.
//
template <typename I,
typename O,
@@ -480,6 +482,8 @@ namespace butl
return p.string ().c_str ();
}
+ // char[N]
+ //
inline const char*
process_arg_as (const char* s, std::string&) {return s;}
@@ -490,6 +494,63 @@ namespace butl
template <std::size_t N>
inline const char*
process_arg_as (const char (&s)[N], std::string&) {return s;}
+
+ template <typename V, typename T>
+ inline void
+ process_args_as (V& v, const T& x, std::string& storage)
+ {
+ v.push_back (process_arg_as (x, storage));
+ }
+
+ // [small_]vector[_view]<>
+ //
+ template <typename V>
+ inline void
+ process_args_as (V& v, const std::vector<std::string>& vs, std::string&)
+ {
+ for (const std::string& s: vs)
+ v.push_back (s.c_str ());
+ }
+
+ template <typename V, std::size_t N>
+ inline void
+ process_args_as (V& v, const small_vector<std::string, N>& vs, std::string&)
+ {
+ for (const std::string& s: vs)
+ v.push_back (s.c_str ());
+ }
+
+ template <typename V>
+ inline void
+ process_args_as (V& v, const vector_view<std::string>& vs, std::string&)
+ {
+ for (const std::string& s: vs)
+ v.push_back (s.c_str ());
+ }
+
+ template <typename V>
+ inline void
+ process_args_as (V& v, const std::vector<const char*>& vs, std::string&)
+ {
+ for (const char* s: vs)
+ v.push_back (s);
+ }
+
+ template <typename V, std::size_t N>
+ inline void
+ process_args_as (V& v, const small_vector<const char*, N>& vs, std::string&)
+ {
+ for (const char* s: vs)
+ v.push_back (s);
+ }
+
+ template <typename V>
+ inline void
+ process_args_as (V& v, const vector_view<const char*>& vs, std::string&)
+ {
+ for (const char* s: vs)
+ v.push_back (s);
+ }
}
#include <butl/process.ixx>
diff --git a/butl/process-io b/butl/process-io
new file mode 100644
index 0000000..42a720a
--- /dev/null
+++ b/butl/process-io
@@ -0,0 +1,21 @@
+// file : butl/process-io -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BUTL_PROCESS_IO
+#define BUTL_PROCESS_IO
+
+#include <ostream>
+
+#include <butl/process>
+
+namespace butl
+{
+ inline std::ostream&
+ operator<< (std::ostream& o, const process_path& p)
+ {
+ return o << p.recall_string ();
+ }
+}
+
+#endif // BUTL_PROCESS_IO
diff --git a/butl/process-run.txx b/butl/process-run.txx
index 8368890..4437488 100644
--- a/butl/process-run.txx
+++ b/butl/process-run.txx
@@ -55,20 +55,21 @@ namespace butl
//
const std::size_t args_size (sizeof... (args));
+ small_vector<const char*, args_size + 2> cmd;
+ cmd.push_back (pp.recall_string ());
+
std::string storage[args_size != 0 ? args_size : 1];
- const char* cmd[args_size + 2] = {
- pp.recall_string (),
- process_arg_as (args, storage[index])...,
- nullptr};
+ const char* dummy[] = {
+ nullptr, (process_args_as (cmd, args, storage[index]), nullptr)... };
- // The last argument can be NULL (used to handle zero A... pack).
- //
- cmdc (cmd, args_size + 2);
+ cmd.push_back (dummy[0]); // NULL (and get rid of unused warning).
+
+ cmdc (cmd.data (), cmd.size ());
// @@ Do we need to make sure certain fd's are closed before calling
// wait()? Is this only the case with pipes? Needs thinking.
- return process_start (cwd, pp, cmd, in_i, out_i, err_i);
+ return process_start (cwd, pp, cmd.data (), in_i, out_i, err_i);
}
template <typename C,
diff --git a/butl/sendmail b/butl/sendmail
index 706f110..fe86285 100644
--- a/butl/sendmail
+++ b/butl/sendmail
@@ -15,7 +15,7 @@
namespace butl
{
- // Send email using the sendmail program.
+ // Send email using the sendmail(1) program.
//
// Write the body of the email to out. Note that you must explicitly close
// it before calling wait(). Throw process_error and io_error (both derive
@@ -64,8 +64,7 @@ namespace butl
const recipients_type& bcc = recipients_type (),
O&&... options);
- // Version with the command line callback (see process_run() for
- // details).
+ // Version with the command line callback (see process_run() for details).
//
template <typename C, typename E, typename... O>
sendmail (const C&,