aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2020-04-29 22:20:21 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2020-05-01 18:10:37 +0300
commit4476d9d02ac7af21a7bc6ef92490491d308237f4 (patch)
tree78fe722595c6303c244a9dc421ce6c15d2165fb6
parentef30db48d9a910fa06a94a07d231f32a8fce0e46 (diff)
Add --pkg-proxy common option
-rw-r--r--bpkg/archive.cxx3
-rw-r--r--bpkg/archive.hxx2
-rw-r--r--bpkg/auth.cxx1
-rw-r--r--bpkg/checksum.cxx4
-rw-r--r--bpkg/common.cli22
-rw-r--r--bpkg/diagnostics.cxx29
-rw-r--r--bpkg/diagnostics.hxx22
-rw-r--r--bpkg/fetch-git.cxx1
-rw-r--r--bpkg/fetch-pkg.cxx12
-rw-r--r--bpkg/fetch.cxx138
-rw-r--r--bpkg/fetch.hxx15
-rw-r--r--bpkg/manifest-utility.cxx1
-rw-r--r--bpkg/pkg-build.cxx3
-rw-r--r--bpkg/pkg-unpack.cxx3
-rw-r--r--bpkg/pkg-verify.cxx1
-rw-r--r--bpkg/satisfaction.cxx3
-rw-r--r--bpkg/types-parsers.cxx22
-rw-r--r--bpkg/types-parsers.hxx10
-rw-r--r--bpkg/types.hxx7
-rw-r--r--bpkg/utility.cxx1
-rw-r--r--bpkg/utility.hxx5
21 files changed, 250 insertions, 55 deletions
diff --git a/bpkg/archive.cxx b/bpkg/archive.cxx
index 1471029..c096701 100644
--- a/bpkg/archive.cxx
+++ b/bpkg/archive.cxx
@@ -3,13 +3,10 @@
#include <bpkg/archive.hxx>
-#include <libbutl/process.mxx>
-
#include <bpkg/utility.hxx>
#include <bpkg/diagnostics.hxx>
using namespace std;
-using namespace butl;
namespace bpkg
{
diff --git a/bpkg/archive.hxx b/bpkg/archive.hxx
index f46812f..cfc4cdb 100644
--- a/bpkg/archive.hxx
+++ b/bpkg/archive.hxx
@@ -4,8 +4,6 @@
#ifndef BPKG_ARCHIVE_HXX
#define BPKG_ARCHIVE_HXX
-#include <libbutl/process.mxx>
-
#include <bpkg/types.hxx>
#include <bpkg/utility.hxx>
diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx
index e29fdb9..e661ad0 100644
--- a/bpkg/auth.cxx
+++ b/bpkg/auth.cxx
@@ -9,7 +9,6 @@
#include <libbutl/sha256.mxx>
#include <libbutl/base64.mxx>
-#include <libbutl/process.mxx>
#include <libbutl/openssl.mxx>
#include <libbutl/timestamp.mxx>
#include <libbutl/filesystem.mxx>
diff --git a/bpkg/checksum.cxx b/bpkg/checksum.cxx
index 228e0c5..efc86dd 100644
--- a/bpkg/checksum.cxx
+++ b/bpkg/checksum.cxx
@@ -7,13 +7,9 @@
# include <algorithm> // replace()
#endif
-#include <libbutl/process.mxx>
-#include <libbutl/filesystem.mxx>
-
#include <bpkg/diagnostics.hxx>
using namespace std;
-using namespace butl;
namespace bpkg
{
diff --git a/bpkg/common.cli b/bpkg/common.cli
index c94f250..4537961 100644
--- a/bpkg/common.cli
+++ b/bpkg/common.cli
@@ -181,12 +181,30 @@ namespace bpkg
programs."
}
+ butl::url --pkg-proxy
+ {
+ "<url>",
+ "HTTP proxy server to use when fetching package manifests and archives
+ from remote \cb{pkg} repositories. If specified, the proxy <url> must
+ be in the \c{\b{http://}\i{host}[\b{:}\i{port}]} form. If \ci{port} is
+ omitted, 80 is used by default.
+
+ Note that to allow caching, the proxied \cb{https://} URLs are
+ converted to \cb{http://} in order to prevent the fetch program from
+ tunneling (which is the standard approach for proxying HTTPS). If both
+ HTTP and HTTPS repositories are used, it is assumed that the proxy
+ server can figure out which URLs need to be converted back to
+ \cb{https://} based on the request information (for example, host
+ name). For security, this mechanism should only be used with signed
+ repositories or when the proxy is located inside a trusted network."
+ }
+
path --git = "git"
{
"<path>",
"The git program to be used to fetch git repositories. You can also
- specify additional options that should be passed to the git program with
- \cb{--git-option}.
+ specify additional options that should be passed to the git program
+ with \cb{--git-option}.
If the git program is not explicitly specified, then \cb{bpkg} will use
\cb{git} by default."
diff --git a/bpkg/diagnostics.cxx b/bpkg/diagnostics.cxx
index 5b77e65..a471d07 100644
--- a/bpkg/diagnostics.cxx
+++ b/bpkg/diagnostics.cxx
@@ -7,8 +7,8 @@
#include <odb/statement.hxx>
-#include <libbutl/process.mxx>
-#include <libbutl/process-io.mxx> // operator<<(ostream, process_arg)
+#include <libbutl/process.mxx> // process_args
+#include <libbutl/process-io.mxx> // operator<<(ostream, process_*)
#include <bpkg/utility.hxx>
@@ -22,14 +22,31 @@ namespace bpkg
void
print_process (const char* const args[], size_t n)
{
- diag_record r (text);
- print_process (r, args, n);
+ diag_record dr (text);
+ print_process (dr, args, n);
}
void
- print_process (diag_record& r, const char* const args[], size_t n)
+ print_process (diag_record& dr, const char* const args[], size_t n)
{
- r << process_args {args, n};
+ dr << process_args {args, n};
+ }
+
+ void
+ print_process (const process_env& pe, const char* const args[], size_t n)
+ {
+ diag_record dr (text);
+ print_process (dr, pe, args, n);
+ }
+
+ void
+ print_process (diag_record& dr,
+ const process_env& pe, const char* const args[], size_t n)
+ {
+ if (pe.env ())
+ dr << pe << ' ';
+
+ dr << process_args {args, n};
}
// Diagnostics verbosity level.
diff --git a/bpkg/diagnostics.hxx b/bpkg/diagnostics.hxx
index ac19a1b..d11ab0b 100644
--- a/bpkg/diagnostics.hxx
+++ b/bpkg/diagnostics.hxx
@@ -49,6 +49,28 @@ namespace bpkg
print_process (args.data (), args.size ());
}
+ // As above but with process_env.
+ //
+ void
+ print_process (diag_record&,
+ const process_env&, const char* const args[], size_t n = 0);
+
+ void
+ print_process (const process_env&, const char* const args[], size_t n = 0);
+
+ inline void
+ print_process (diag_record& dr,
+ const process_env& pe, const cstrings& args)
+ {
+ print_process (dr, pe, args.data (), args.size ());
+ }
+
+ inline void
+ print_process (const process_env& pe, const cstrings& args)
+ {
+ print_process (pe, args.data (), args.size ());
+ }
+
// Verbosity level. Update documentation for --verbose if changing.
//
// 0 - disabled
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index 274108f..df59dde 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -8,7 +8,6 @@
#include <libbutl/git.mxx>
#include <libbutl/utility.mxx> // digit(), xdigit()
-#include <libbutl/process.mxx>
#include <libbutl/filesystem.mxx> // path_entry
#include <libbutl/path-pattern.mxx>
#include <libbutl/semantic-version.mxx>
diff --git a/bpkg/fetch-pkg.cxx b/bpkg/fetch-pkg.cxx
index 67ee3a9..81d4131 100644
--- a/bpkg/fetch-pkg.cxx
+++ b/bpkg/fetch-pkg.cxx
@@ -24,7 +24,11 @@ namespace bpkg
bool ignore_unknown)
{
string url (u.string ());
- process pr (start_fetch (o, url));
+ process pr (start_fetch (o,
+ url,
+ path () /* out */,
+ string () /* user_agent */,
+ o.pkg_proxy ()));
try
{
@@ -94,7 +98,11 @@ namespace bpkg
fail << "file " << df << " already exists";
auto_rmfile arm (df);
- process pr (start_fetch (o, u.string (), df));
+ process pr (start_fetch (o,
+ u.string (),
+ df,
+ string () /* user_agent */,
+ o.pkg_proxy ()));
if (!pr.wait ())
{
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 63a459d..f565f26 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -3,12 +3,9 @@
#include <bpkg/fetch.hxx>
-#include <libbutl/process.mxx>
-
#include <bpkg/diagnostics.hxx>
using namespace std;
-using namespace butl;
namespace bpkg
{
@@ -94,7 +91,8 @@ namespace bpkg
const strings& ops,
const string& url,
const path& out,
- const string& user_agent)
+ const string& user_agent,
+ const string& http_proxy)
{
bool fo (!out.empty ()); // Output to file.
@@ -178,10 +176,23 @@ namespace bpkg
args.push_back (url.c_str ());
args.push_back (nullptr);
- process_path pp (process::path_search (args[0]));
+ process_path pp (process::path_search (args[0]));
+ process_env env (pp);
+
+ // HTTP proxy.
+ //
+ string evar;
+ const char* evars[] = {nullptr, nullptr};
+
+ if (!http_proxy.empty ())
+ {
+ evar = "http_proxy=" + http_proxy;
+ evars[0] = evar.c_str ();
+ env.vars = evars;
+ }
if (verb >= 2)
- print_process (args);
+ print_process (env, args);
// If we are fetching into a file, change the wget's directory to
// that of the output file. We do it this way so that we end up with
@@ -191,8 +202,9 @@ namespace bpkg
return fo
? process (pp, args.data (),
0, 1, 2,
- out.directory ().string ().c_str ())
- : process (pp, args.data (), 0, -1);
+ out.directory ().string ().c_str (),
+ env.vars)
+ : process (pp, args.data (), 0, -1, 2, nullptr /* cwd */, env.vars);
}
// curl
@@ -247,7 +259,8 @@ namespace bpkg
const strings& ops,
const string& url,
const path& out,
- const string& user_agent)
+ const string& user_agent,
+ const string& http_proxy)
{
bool fo (!out.empty ()); // Output to file.
@@ -320,6 +333,14 @@ namespace bpkg
args.push_back (out.string ().c_str ());
}
+ // HTTP proxy.
+ //
+ if (!http_proxy.empty ())
+ {
+ args.push_back ("--proxy");
+ args.push_back (http_proxy.c_str ());
+ }
+
args.push_back (url.c_str ());
args.push_back (nullptr);
@@ -396,7 +417,8 @@ namespace bpkg
const strings& ops,
const string& url,
const path& out,
- const string& user_agent)
+ const string& user_agent,
+ const string& http_proxy)
{
bool fo (!out.empty ()); // Output to file.
@@ -464,10 +486,23 @@ namespace bpkg
args.push_back (url.c_str ());
args.push_back (nullptr);
- process_path pp (process::path_search (args[0]));
+ process_path pp (process::path_search (args[0]));
+ process_env env (pp);
+
+ // HTTP proxy.
+ //
+ string evar;
+ const char* evars[] = {nullptr, nullptr};
+
+ if (!http_proxy.empty ())
+ {
+ evar = "HTTP_PROXY=" + http_proxy;
+ evars[0] = evar.c_str ();
+ env.vars = evars;
+ }
if (verb >= 2)
- print_process (args);
+ print_process (env, args);
// If we are fetching into a file, change the fetch's directory to
// that of the output file. We do it this way so that we end up with
@@ -477,8 +512,9 @@ namespace bpkg
return fo
? process (pp, args.data (),
0, 1, 2,
- out.directory ().string ().c_str ())
- : process (pp, args.data (), 0, -1);
+ out.directory ().string ().c_str (),
+ env.vars)
+ : process (pp, args.data (), 0, -1, 2, nullptr /* cwd */, env.vars);
}
// The dispatcher.
@@ -572,9 +608,10 @@ namespace bpkg
process
start_fetch (const common_options& o,
- const string& url,
+ const string& src,
const path& out,
- const string& user_agent)
+ const string& user_agent,
+ const url& proxy)
{
process (*f) (const path&,
const optional<size_t>&,
@@ -582,6 +619,7 @@ namespace bpkg
const strings&,
const string&,
const path&,
+ const string&,
const string&) = nullptr;
switch (check (o))
@@ -595,15 +633,79 @@ namespace bpkg
if (o.fetch_timeout_specified ())
timeout = o.fetch_timeout ();
+ // If the HTTP proxy is specified and the URL is HTTP(S), then fetch
+ // through the proxy, converting the https URL scheme to http.
+ //
try
{
+ string http_url;
+ string http_proxy;
+
+ if (!proxy.empty ())
+ {
+ auto bad_proxy = [&src, &proxy] (const char* d)
+ {
+ fail << "unable to fetch '" << src << "' using '" << proxy
+ << "' as proxy: " << d;
+ };
+
+ if (icasecmp (proxy.scheme, "http") != 0)
+ bad_proxy ("only HTTP proxy is supported");
+
+ if (!proxy.authority || proxy.authority->host.empty ())
+ bad_proxy ("invalid host name in proxy URL");
+
+ if (!proxy.authority->user.empty ())
+ bad_proxy ("unexpected user in proxy URL");
+
+ if (proxy.path)
+ bad_proxy ("unexpected path in proxy URL");
+
+ if (proxy.query)
+ bad_proxy ("unexpected query in proxy URL");
+
+ if (proxy.fragment)
+ bad_proxy ("unexpected fragment in proxy URL");
+
+ if (proxy.rootless)
+ bad_proxy ("proxy URL cannot be rootless");
+
+ url u;
+ try
+ {
+ u = url (src);
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "unable to fetch '" << src << "': invalid URL: " << e;
+ }
+
+ bool http (icasecmp (u.scheme, "http") == 0);
+ bool https (icasecmp (u.scheme, "https") == 0);
+
+ if (http || https)
+ {
+ http_proxy = proxy.string ();
+
+ if (proxy.authority->port == 0)
+ http_proxy += ":80";
+
+ if (https)
+ {
+ u.scheme = "http";
+ http_url = u.string ();
+ }
+ }
+ }
+
return f (fetch_path,
timeout,
o.no_progress (),
o.fetch_option (),
- url,
+ !http_url.empty () ? http_url : src,
out,
- user_agent);
+ user_agent,
+ http_proxy);
}
catch (const process_error& e)
{
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index ed1fd50..d57dcf3 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -6,8 +6,6 @@
#include <ctime> // time_t
-#include <libbutl/process.mxx>
-
#include <libbpkg/manifest.hxx>
#include <bpkg/types.hxx>
@@ -20,6 +18,9 @@ namespace bpkg
// Repository type pkg (fetch-pkg.cxx).
//
+ // If HTTP proxy is specified via the --pkg-proxy option, then use it for
+ // fetching manifests and archives from the remote pkg repository.
+ //
pkg_repository_manifests
pkg_fetch_repositories (const dir_path&, bool ignore_unknown);
@@ -120,13 +121,17 @@ namespace bpkg
// Start the process of fetching the specified URL. If out is empty, then
// fetch to stdout. In this case also don't show any progress unless we are
// running verbose. If user_agent is empty, then send the default (fetch
- // program specific) User-Agent header value.
+ // program specific) User-Agent header value. If the HTTP proxy URL is not
+ // empty and the URL to fetch is HTTP(S), then fetch it via the specified
+ // proxy server converting the https URL scheme to http (see the --pkg-proxy
+ // option for details).
//
- butl::process
+ process
start_fetch (const common_options& o,
const string& url,
const path& out = {},
- const string& user_agent = {});
+ const string& user_agent = {},
+ const butl::url& proxy = {});
}
#endif // BPKG_FETCH_HXX
diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 697f920..159b2c3 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -6,7 +6,6 @@
#include <cstring> // strcspn()
#include <libbutl/b.mxx>
-#include <libbutl/url.mxx>
#include <libbutl/sha256.mxx>
#include <bpkg/package.hxx> // wildcard_version
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index 6b68a65..106666a 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -10,7 +10,6 @@
#include <iostream> // cout
#include <algorithm> // find_if()
-#include <libbutl/url.mxx>
#include <libbutl/standard-version.mxx>
#include <bpkg/package.hxx>
@@ -2645,7 +2644,7 @@ namespace bpkg
//
size_t p (0);
- using url_traits = butl::url::traits_type;
+ using url_traits = url::traits_type;
// Skip leading ':' that are not part of a URL.
//
diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx
index 02d61b6..10f8920 100644
--- a/bpkg/pkg-unpack.cxx
+++ b/bpkg/pkg-unpack.cxx
@@ -7,8 +7,6 @@
# include <algorithm> // replace()
#endif
-#include <libbutl/process.mxx>
-
#include <libbpkg/manifest.hxx>
#include <bpkg/archive.hxx>
@@ -23,7 +21,6 @@
#include <bpkg/pkg-verify.hxx>
using namespace std;
-using namespace butl;
namespace bpkg
{
diff --git a/bpkg/pkg-verify.cxx b/bpkg/pkg-verify.cxx
index 5c23b69..38b4d68 100644
--- a/bpkg/pkg-verify.cxx
+++ b/bpkg/pkg-verify.cxx
@@ -5,7 +5,6 @@
#include <iostream> // cout
-#include <libbutl/process.mxx>
#include <libbutl/manifest-parser.mxx>
#include <libbutl/manifest-serializer.mxx>
diff --git a/bpkg/satisfaction.cxx b/bpkg/satisfaction.cxx
index c2a4601..52def32 100644
--- a/bpkg/satisfaction.cxx
+++ b/bpkg/satisfaction.cxx
@@ -3,13 +3,10 @@
#include <bpkg/satisfaction.hxx>
-#include <libbutl/process.mxx>
-
#include <bpkg/package-odb.hxx>
#include <bpkg/diagnostics.hxx>
using namespace std;
-using namespace butl;
namespace bpkg
{
diff --git a/bpkg/types-parsers.cxx b/bpkg/types-parsers.cxx
index ce3cd0c..be95219 100644
--- a/bpkg/types-parsers.cxx
+++ b/bpkg/types-parsers.cxx
@@ -7,6 +7,28 @@ namespace bpkg
{
namespace cli
{
+ void parser<url>::
+ parse (url& x, bool& xs, scanner& s)
+ {
+ const char* o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ const char* v (s.next ());
+
+ try
+ {
+ x = url (v);
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_value (o, v, e.what ());
+ }
+
+ xs = true;
+ }
+
template <typename T>
static void
parse_path (T& x, scanner& s)
diff --git a/bpkg/types-parsers.hxx b/bpkg/types-parsers.hxx
index c812e7b..38b7cee 100644
--- a/bpkg/types-parsers.hxx
+++ b/bpkg/types-parsers.hxx
@@ -19,6 +19,16 @@ namespace bpkg
namespace cli
{
template <>
+ struct parser<url>
+ {
+ static void
+ parse (url&, bool&, scanner&);
+
+ static void
+ merge (url& b, const url& a) {b = a;}
+ };
+
+ template <>
struct parser<path>
{
static void
diff --git a/bpkg/types.hxx b/bpkg/types.hxx
index d4847b8..4f0cc98 100644
--- a/bpkg/types.hxx
+++ b/bpkg/types.hxx
@@ -21,6 +21,7 @@
#include <odb/lazy-ptr.hxx>
+#include <libbutl/url.mxx>
#include <libbutl/path.mxx>
#include <libbutl/process.mxx>
#include <libbutl/utility.mxx> // compare_reference_target
@@ -94,9 +95,14 @@ namespace bpkg
using paths = std::vector<path>;
using dir_paths = std::vector<dir_path>;
+ // <libbutl/url.mxx>
+ //
+ using butl::url;
+
// <libbutl/process.mxx>
//
using butl::process;
+ using butl::process_env;
using butl::process_path;
using butl::process_exit;
using butl::process_error;
@@ -104,6 +110,7 @@ namespace bpkg
// <libbutl/fdstream.mxx>
//
using butl::auto_fd;
+ using butl::nullfd;
using butl::fdpipe;
using butl::ifdstream;
using butl::ofdstream;
diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx
index 5ec8159..9c9c991 100644
--- a/bpkg/utility.cxx
+++ b/bpkg/utility.cxx
@@ -4,7 +4,6 @@
#include <bpkg/utility.hxx>
#include <libbutl/prompt.mxx>
-#include <libbutl/process.mxx>
#include <libbutl/fdstream.mxx>
#include <bpkg/diagnostics.hxx>
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index 1763af0..e891f21 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -14,6 +14,7 @@
#include <libbutl/ft/lang.hxx>
#include <libbutl/utility.mxx> // icasecmp(), reverse_iterate(), etc
+#include <libbutl/process.mxx>
#include <libbutl/filesystem.mxx>
#include <libbutl/default-options.mxx>
@@ -47,6 +48,10 @@ namespace bpkg
using butl::setenv;
using butl::unsetenv;
+ // <libbutl/process.mxx>
+ //
+ using butl::process_start_callback;
+
// <libbutl/filesystem.mxx>
//
using butl::auto_rmfile;