From 180fdc20372d6501b8fcabb66e1d3cbda02b35c9 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 5 Dec 2022 12:33:59 +0200 Subject: Use curl instead of wget as default fetch program We used to prefer wget 1.16 because it has --show-progress which results in nicer progress. But experience shows that wget is quite unreliable plus with bdep always using curl, it would be strange to use both curl and wget (and expecting the user to setup proxy, authentication, etc., for both). Also add internal --curl* options to be used by bdep. --- bpkg/buildfile | 8 ++++-- bpkg/common.cli | 10 +++++-- bpkg/fetch.cxx | 83 ++++++++++++++++++++++++++++++++++++++++++++++++--------- doc/cli.sh | 7 +++-- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/bpkg/buildfile b/bpkg/buildfile index cc9e7b5..b3b2ba7 100644 --- a/bpkg/buildfile +++ b/bpkg/buildfile @@ -187,20 +187,24 @@ if $cli.configured # Option length must be the same to get commands/topics/options aligned. # + # Need global --suppress-undocumented because of few undocumented options + # in common.cli. + # + # cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix bpkg --guard-prefix BPKG \ --cxx-prologue "#include " --cli-namespace bpkg::cli \ --generate-vector-scanner --generate-file-scanner --generate-group-scanner \ --keep-separator --generate-specifier --generate-parse --generate-merge \ --page-usage 'bpkg::print_$name$_' --ansi-color --ascii-tree \ ---include-base-last --option-length 24 +--include-base-last --suppress-undocumented --option-length 24 # Both --*-usage options. # cli.cxx{common-options}: cli.options += --short-usage --long-usage \ --generate-modifier - cli.cxx{bpkg-options}: cli.options += --short-usage --suppress-undocumented + cli.cxx{bpkg-options}: cli.options += --short-usage cli.options += --long-usage # All other pages -- long usage. diff --git a/bpkg/common.cli b/bpkg/common.cli index d62633a..ac045d0 100644 --- a/bpkg/common.cli +++ b/bpkg/common.cli @@ -167,8 +167,7 @@ namespace bpkg If the fetch program is not specified, then \cb{bpkg} will try to discover if one of the above programs is available and use that. - Currently, \cb{bpkg} has the following preference order: \cb{wget} - 1.16 or higher (supports \cb{--show-progress}), \cb{curl}, + Currently, \cb{bpkg} has the following preference order: \cb{curl}, \cb{wget}, and \cb{fetch}." } @@ -180,6 +179,13 @@ namespace bpkg specify multiple fetch options." } + // Undocumented equivalents to bdep's --curl* options. We "merge" them + // into --fetch/--fetch-option in an ad hoc manner (see fetch.cxx for + // details). + // + path --curl; + strings --curl-option; + size_t --fetch-timeout { "", diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx index 5ecddbb..44f1ba4 100644 --- a/bpkg/fetch.cxx +++ b/bpkg/fetch.cxx @@ -555,7 +555,7 @@ namespace bpkg // Cache the result of finding/testing the fetch program. Sometimes a simple // global variable is really the right solution... // - enum class fetch_kind {wget, curl, fetch}; + enum class fetch_kind {curl, wget, fetch}; static path path_; static fetch_kind kind_; @@ -575,20 +575,20 @@ namespace bpkg const path& n (p.leaf ()); const string& s (n.string ()); - if (s.find ("wget") != string::npos) - { - if (!check_wget (p)) - fail << p << " does not appear to be the 'wget' program"; - - kind_ = fetch_kind::wget; - } - else if (s.find ("curl") != string::npos) + if (s.find ("curl") != string::npos) { if (!check_curl (p)) fail << p << " does not appear to be the 'curl' program"; kind_ = fetch_kind::curl; } + else if (s.find ("wget") != string::npos) + { + if (!check_wget (p)) + fail << p << " does not appear to be the 'wget' program"; + + kind_ = fetch_kind::wget; + } else if (s.find ("fetch") != string::npos) { if (!check_fetch (p)) @@ -599,15 +599,45 @@ namespace bpkg else fail << "unknown fetch program " << p; } + else if (o.curl_specified ()) + { + const path& p (path_ = o.fetch ()); + + if (!check_curl (p)) + fail << p << " does not appear to be the 'curl' program"; + + kind_ = fetch_kind::curl; + } else { // See if any is available. The preference order is: // + // curl + // wget + // fetch +#if 1 + if (check_curl (path_ = path ("curl"))) + { + kind_ = fetch_kind::curl; + } + else if (check_wget (path_ = path ("wget"))) + { + kind_ = fetch_kind::wget; + } +#else + // Old preference order: + // // wget 1.16 or up // curl // wget // fetch // + // We used to prefer wget 1.16 because it has --show-progress which + // results in nicer progress. But experience shows that wget is quite + // unreliable plus with bdep always using curl, it would be strange + // to use both curl and wget (and expecting the user to setup proxy, + // authentication, etc., for both). + // bool wg (check_wget (path_ = path ("wget"))); if (wg && (wget_major > 1 || (wget_major == 1 && wget_minor >= 16))) @@ -623,12 +653,13 @@ namespace bpkg path_ = path ("wget"); kind_ = fetch_kind::wget; } +#endif else if (check_fetch (path_ = path ("fetch"))) { kind_ = fetch_kind::fetch; } else - fail << "unable to find 'wget', 'curl', or 'fetch'" << + fail << "unable to find 'curl', 'wget', or 'fetch'" << info << "use --fetch to specify the fetch program location"; if (verb >= 3) @@ -656,10 +687,11 @@ namespace bpkg const string&, const string&) = nullptr; - switch (check (o)) + fetch_kind fk (check (o)); + switch (fk) { - case fetch_kind::wget: f = &start_wget; break; case fetch_kind::curl: f = &start_curl; break; + case fetch_kind::wget: f = &start_wget; break; case fetch_kind::fetch: f = &start_fetch; break; } @@ -732,11 +764,36 @@ namespace bpkg } } + // Note that the merge semantics here is not 100% accurate since we may + // override "later" --fetch-option with "earlier" --curl-option. + // However, this should be close enough for our use-case, which is + // bdep's --curl-option values overriding --fetch-option specified in + // the default options file. The situation that we will mis-handle is + // when both are specified on the command line, for example, + // --curl-option --max-time=2 --bpkg-option --fetch-option=--max-time=1, + // but that feel quite far fetched to complicate things here. + // + const strings& fos (o.fetch_option ()); + const strings& cos (o.curl_option ()); + + const strings& os ( + fk != fetch_kind::curl || cos.empty () + ? fos + : (fos.empty () + ? cos + : [&fos, &cos] () + { + strings r (fos.begin (), fos.end ()); + r.insert (r.end (), cos.begin (), cos.end ()); + return r; + } ())); + + return f (path_, timeout, o.progress (), o.no_progress (), - o.fetch_option (), + os, !http_url.empty () ? http_url : src, out, user_agent, diff --git a/doc/cli.sh b/doc/cli.sh index fc81528..30ed7b2 100755 --- a/doc/cli.sh +++ b/doc/cli.sh @@ -66,12 +66,15 @@ function compile () ../bpkg/$n.cli } -o="--output-prefix bpkg- --class-doc bpkg::common_options=short" +# Need global --suppress-undocumented because of few undocumented options +# in common.cli. +# +o="--suppress-undocumented --output-prefix bpkg- --class-doc bpkg::common_options=short" # A few special cases. # compile "common" $o --output-suffix "-options" --class-doc bpkg::common_options=long -compile "bpkg" $o --output-prefix "" --suppress-undocumented --class-doc bpkg::commands=short --class-doc bpkg::topics=short +compile "bpkg" $o --output-prefix "" --class-doc bpkg::commands=short --class-doc bpkg::topics=short compile "pkg-build" $o --class-doc bpkg::pkg_build_pkg_options=exclude-base -- cgit v1.1