Use curl instead of wget as default fetch program

We used to prefer wget 1.16 because it has --show-progress which results in
nicer progress. But experience shows that wget is quite unreliable plus with
bdep always using curl, it would be strange to use both curl and wget (and
expecting the user to setup proxy, authentication, etc., for both).

Also add internal --curl* options to be used by bdep.

4 files changed, 89 insertions, 19 deletions

diff --git a/bpkg/buildfile b/bpkg/buildfile
index cc9e7b5..b3b2ba7 100644
--- a/bpkg/buildfile
+++ b/bpkg/buildfile
@@ -187,20 +187,24 @@ if $cli.configured
 
   # Option length must be the same to get commands/topics/options aligned.
   #
+  # Need global --suppress-undocumented because of few undocumented options
+  # in common.cli.
+  #
+  #
   cli.options += --std c++11 -I $src_root --include-with-brackets \
 --include-prefix bpkg --guard-prefix BPKG \
 --cxx-prologue "#include <bpkg/types-parsers.hxx>" --cli-namespace bpkg::cli \
 --generate-vector-scanner --generate-file-scanner --generate-group-scanner \
 --keep-separator --generate-specifier --generate-parse --generate-merge \
 --page-usage 'bpkg::print_$name$_' --ansi-color --ascii-tree \
---include-base-last --option-length 24
+--include-base-last --suppress-undocumented --option-length 24
 
   # Both --*-usage options.
   #
   cli.cxx{common-options}: cli.options += --short-usage --long-usage \
 --generate-modifier
 
-  cli.cxx{bpkg-options}: cli.options += --short-usage --suppress-undocumented
+  cli.cxx{bpkg-options}: cli.options += --short-usage
 
   cli.options += --long-usage # All other pages -- long usage.
 
diff --git a/bpkg/common.cli b/bpkg/common.cli
index d62633a..ac045d0 100644
--- a/bpkg/common.cli
+++ b/bpkg/common.cli
@@ -167,8 +167,7 @@ namespace bpkg
 
        If the fetch program is not specified, then \cb{bpkg} will try to
        discover if one of the above programs is available and use that.
-       Currently, \cb{bpkg} has the following preference order: \cb{wget}
-       1.16 or higher (supports \cb{--show-progress}), \cb{curl},
+       Currently, \cb{bpkg} has the following preference order: \cb{curl},
        \cb{wget}, and \cb{fetch}."
     }
 
@@ -180,6 +179,13 @@ namespace bpkg
        specify multiple fetch options."
     }
 
+    // Undocumented equivalents to bdep's --curl* options. We "merge" them
+    // into --fetch/--fetch-option in an ad hoc manner (see fetch.cxx for
+    // details).
+    //
+    path --curl;
+    strings --curl-option;
+
     size_t --fetch-timeout
     {
       "<sec>",
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 5ecddbb..44f1ba4 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -555,7 +555,7 @@ namespace bpkg
   // Cache the result of finding/testing the fetch program. Sometimes a simple
   // global variable is really the right solution...
   //
-  enum class fetch_kind {wget, curl, fetch};
+  enum class fetch_kind {curl, wget, fetch};
 
   static path       path_;
   static fetch_kind kind_;
@@ -575,20 +575,20 @@ namespace bpkg
       const path& n (p.leaf ());
       const string& s (n.string ());
 
-      if (s.find ("wget") != string::npos)
-      {
-        if (!check_wget (p))
-          fail << p << " does not appear to be the 'wget' program";
-
-        kind_ = fetch_kind::wget;
-      }
-      else if (s.find ("curl") != string::npos)
+      if (s.find ("curl") != string::npos)
       {
         if (!check_curl (p))
           fail << p << " does not appear to be the 'curl' program";
 
         kind_ = fetch_kind::curl;
       }
+      else if (s.find ("wget") != string::npos)
+      {
+        if (!check_wget (p))
+          fail << p << " does not appear to be the 'wget' program";
+
+        kind_ = fetch_kind::wget;
+      }
       else if (s.find ("fetch") != string::npos)
       {
         if (!check_fetch (p))
@@ -599,15 +599,45 @@ namespace bpkg
       else
         fail << "unknown fetch program " << p;
     }
+    else if (o.curl_specified ())
+    {
+      const path& p (path_ = o.fetch ());
+
+      if (!check_curl (p))
+        fail << p << " does not appear to be the 'curl' program";
+
+      kind_ = fetch_kind::curl;
+    }
     else
     {
       // See if any is available. The preference order is:
       //
+      // curl
+      // wget
+      // fetch
+#if 1
+      if (check_curl (path_ = path ("curl")))
+      {
+        kind_ = fetch_kind::curl;
+      }
+      else if (check_wget (path_ = path ("wget")))
+      {
+        kind_ = fetch_kind::wget;
+      }
+#else
+      // Old preference order:
+      //
       // wget 1.16 or up
       // curl
       // wget
       // fetch
       //
+      // We used to prefer wget 1.16 because it has --show-progress which
+      // results in nicer progress. But experience shows that wget is quite
+      // unreliable plus with bdep always using curl, it would be strange
+      // to use both curl and wget (and expecting the user to setup proxy,
+      // authentication, etc., for both).
+      //
       bool wg (check_wget (path_ = path ("wget")));
 
       if (wg && (wget_major > 1 || (wget_major == 1 && wget_minor >= 16)))
@@ -623,12 +653,13 @@ namespace bpkg
         path_ = path ("wget");
         kind_ = fetch_kind::wget;
       }
+#endif
       else if (check_fetch (path_ = path ("fetch")))
       {
         kind_ = fetch_kind::fetch;
       }
       else
-        fail << "unable to find 'wget', 'curl', or 'fetch'" <<
+        fail << "unable to find 'curl', 'wget', or 'fetch'" <<
           info << "use --fetch to specify the fetch program location";
 
       if (verb >= 3)
@@ -656,10 +687,11 @@ namespace bpkg
                   const string&,
                   const string&) = nullptr;
 
-    switch (check (o))
+    fetch_kind fk (check (o));
+    switch (fk)
     {
-    case fetch_kind::wget:  f = &start_wget;  break;
     case fetch_kind::curl:  f = &start_curl;  break;
+    case fetch_kind::wget:  f = &start_wget;  break;
     case fetch_kind::fetch: f = &start_fetch; break;
     }
 
@@ -732,11 +764,36 @@ namespace bpkg
         }
       }
 
+      // Note that the merge semantics here is not 100% accurate since we may
+      // override "later" --fetch-option with "earlier" --curl-option.
+      // However, this should be close enough for our use-case, which is
+      // bdep's --curl-option values overriding --fetch-option specified in
+      // the default options file. The situation that we will mis-handle is
+      // when both are specified on the command line, for example,
+      // --curl-option --max-time=2 --bpkg-option --fetch-option=--max-time=1,
+      // but that feel quite far fetched to complicate things here.
+      //
+      const strings& fos (o.fetch_option ());
+      const strings& cos (o.curl_option ());
+
+      const strings& os (
+        fk != fetch_kind::curl || cos.empty ()
+        ? fos
+        : (fos.empty ()
+           ? cos
+           : [&fos, &cos] ()
+             {
+               strings r (fos.begin (), fos.end ());
+               r.insert (r.end (), cos.begin (), cos.end ());
+               return r;
+             } ()));
+
+
       return f (path_,
                 timeout,
                 o.progress (),
                 o.no_progress (),
-                o.fetch_option (),
+                os,
                 !http_url.empty () ? http_url : src,
                 out,
                 user_agent,
diff --git a/doc/cli.sh b/doc/cli.sh
index fc81528..30ed7b2 100755
--- a/doc/cli.sh
+++ b/doc/cli.sh
@@ -66,12 +66,15 @@ function compile ()
 ../bpkg/$n.cli
 }
 
-o="--output-prefix bpkg- --class-doc bpkg::common_options=short"
+# Need global --suppress-undocumented because of few undocumented options
+# in common.cli.
+#
+o="--suppress-undocumented --output-prefix bpkg- --class-doc bpkg::common_options=short"
 
 # A few special cases.
 #
 compile "common" $o --output-suffix "-options" --class-doc bpkg::common_options=long
-compile "bpkg" $o --output-prefix "" --suppress-undocumented --class-doc bpkg::commands=short --class-doc bpkg::topics=short
+compile "bpkg" $o --output-prefix "" --class-doc bpkg::commands=short --class-doc bpkg::topics=short
 
 compile "pkg-build" $o --class-doc  bpkg::pkg_build_pkg_options=exclude-base