Assume git repository protocol as smart if HTTP response code is 401

5 files changed, 434 insertions, 80 deletions

diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index 79567e6..dc1e16a 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -285,20 +285,7 @@ namespace bpkg
 
       try
       {
-        ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
-
-        // We could probably write something like this, instead:
-        //
-        // *diag_stream << is.rdbuf () << flush;
-        //
-        // However, it would never throw and we could potentially miss the
-        // reading failure, unless we decide to additionally mess with the
-        // diagnostics stream exception mask.
-        //
-        for (string l; !eof (getline (is, l)); )
-          *diag_stream << l << endl;
-
-        is.close ();
+        dump_stderr (move (pipe.in));
 
         // Fall through.
       }
@@ -549,7 +536,11 @@ namespace bpkg
   // For HTTP(S) sense the protocol type by sending the first HTTP request of
   // the fetch operation handshake and analyzing the first line of the
   // response. Fail if connecting to the server failed, the response code
-  // differs from 200, or reading the response body failed.
+  // differs from 200 and 401, or reading the response body failed. If the
+  // response code is 401 (requires authentication), then consider protocol as
+  // smart. The thinking here is that a git repository with support for
+  // authentication is likely one of the hosting places (like git{hub,lab})
+  // and is unlikely to be dumb.
   //
   // Note that, as a side-effect, this function checks the HTTP(S) server
   // availability and so must be called prior to any git command that involves
@@ -574,12 +565,12 @@ namespace bpkg
 
   static capabilities
   sense_capabilities (const common_options& co,
-                      repository_url url,
+                      const repository_url& repo_url,
                       const semantic_version& git_ver)
   {
-    assert (url.path);
+    assert (repo_url.path);
 
-    switch (url.scheme)
+    switch (repo_url.scheme)
     {
     case repository_protocol::git:
     case repository_protocol::ssh:
@@ -588,6 +579,9 @@ namespace bpkg
     case repository_protocol::https: break; // Ask the server (see below).
     }
 
+    // Craft the URL for sensing the capabilities.
+    //
+    repository_url url (repo_url);
     path& up (*url.path);
 
     if (!up.to_directory ())
@@ -601,19 +595,69 @@ namespace bpkg
       url.query = "service=git-upload-pack";
 
     string u (url.string ());
-    process pr (start_fetch (co,
-                             u,
-                             path () /* out */,
-                             "git/" + git_ver.string ()));
+
+    // Start fetching, also trying to retrieve the HTTP status code.
+    //
+    // We unset failbit to properly handle an empty response (no refs) from
+    // the dumb server.
+    //
+    ifdstream is (ifdstream::badbit);
+
+    pair<process, uint16_t> ps (
+      start_fetch_http (co,
+                        u,
+                        is /* out */,
+                        fdstream_mode::skip | fdstream_mode::binary,
+                        true /* quiet */,
+                        "git/" + git_ver.string ()));
+
+    process& pr (ps.first);
+
+    // If the fetch program stderr is redirected, then read it out and pass
+    // through.
+    //
+    auto dump_stderr = [&pr] ()
+    {
+      if (pr.in_efd != nullfd)
+      try
+      {
+        bpkg::dump_stderr (move (pr.in_efd));
+      }
+      catch (const io_error&)
+      {
+        // Not much we can do here.
+      }
+    };
 
     try
     {
-      // We unset failbit to properly handle an empty response (no refs) from
-      // the dumb server.
+      // If authentication is required (HTTP status code is 401), then
+      // consider the protocol as smart. Drop the diagnostics if that's the
+      // case and dump it otherwise.
       //
-      ifdstream is (move (pr.in_ofd),
-                    fdstream_mode::skip | fdstream_mode::binary,
-                    ifdstream::badbit);
+      if (ps.second == 401)
+      {
+        if (verb >= 2)
+        {
+          info << "smart git protocol assumed for repository " << repo_url
+               << " due to authentication requirement" <<
+            info << "use --git-capabilities to override or suppress this "
+                 << "diagnostics";
+        }
+
+        // Note that we don't care about the process exit code here and just
+        // silently wait for the process completion in the process object
+        // destructor. We, however, close the stream (reading out the
+        // content), so that the process won't get blocked writing to it.
+        //
+        // Also note that we drop the potentially redirected process stderr
+        // stream content. We even don't read it out, since we assume it fully
+        // fits into the pipe buffer.
+        //
+        is.close ();
+
+        return capabilities::smart;
+      }
 
       string l;
       getline (is, l); // Is empty if no refs returned by the dumb server.
@@ -667,6 +711,8 @@ namespace bpkg
 
       is.close ();
 
+      dump_stderr ();
+
       if (pr.wait ())
         return r;
 
@@ -674,6 +720,8 @@ namespace bpkg
     }
     catch (const io_error&)
     {
+      dump_stderr ();
+
       if (pr.wait ())
         fail << "unable to read fetched " << url << endg;
 
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 44f1ba4..a7fe8c9 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -84,13 +84,21 @@ namespace bpkg
     }
   }
 
-  static process
+  // Note that there is no easy way to retrieve the HTTP status code for wget
+  // (there is no reliable way to redirect the status line/headers to stdout)
+  // and thus we always return 0. Due to the status code unavailability there
+  // is no need to redirect stderr and thus we ignore the quiet mode.
+  //
+  static pair<process, uint16_t>
   start_wget (const path& prog,
               const optional<size_t>& timeout,
               bool progress,
               bool no_progress,
+              bool /* quiet */,
               const strings& ops,
               const string& url,
+              ifdstream* out_is,
+              fdstream_mode out_ism,
               const path& out,
               const string& user_agent,
               const string& http_proxy)
@@ -108,8 +116,8 @@ namespace bpkg
     };
 
     // Wget 1.16 introduced the --show-progress option which in the quiet mode
-    // shows a nice and tidy progress bar (if only it also showed errors, then
-    // it would have been perfect).
+    // (-q) shows a nice and tidy progress bar (if only it also showed errors,
+    // then it would have been perfect).
     //
     bool has_show_progress (wget_major > 1 ||
                             (wget_major == 1 && wget_minor >= 16));
@@ -222,12 +230,19 @@ namespace bpkg
     // just the file name (rather than the whole path) in the progress
     // report. Process exceptions must be handled by the caller.
     //
-    return fo
-      ? process (pp, args.data (),
-                 0, 1, 2,
-                 out.directory ().string ().c_str (),
-                 env.vars)
-      : process (pp, args.data (), 0, -1, 2, nullptr /* cwd */, env.vars);
+    process pr (fo
+                ? process (pp, args.data (),
+                           0, 1, 2,
+                           out.directory ().string ().c_str (),
+                           env.vars)
+                : process (pp, args.data (),
+                           0, -1, 2,
+                           nullptr /* cwd */, env.vars));
+
+    if (out_is != nullptr)
+      out_is->open (move (pr.in_ofd), out_ism);
+
+    return make_pair (move (pr), 0);
   }
 
   // curl
@@ -275,13 +290,21 @@ namespace bpkg
     return false;
   }
 
-  static process
+  // If HTTP status code needs to be retrieved, then open the passed stream
+  // and read out the status line(s) extracting the status code and the
+  // headers. Otherwise, return 0 indicating that the status code is not
+  // available. In the former case redirect stderr and respect the quiet mode.
+  //
+  static pair<process, uint16_t>
   start_curl (const path& prog,
               const optional<size_t>& timeout,
               bool progress,
               bool no_progress,
+              bool quiet,
               const strings& ops,
               const string& url,
+              ifdstream* out_is,
+              fdstream_mode out_ism,
               const path& out,
               const string& user_agent,
               const string& http_proxy)
@@ -313,28 +336,31 @@ namespace bpkg
     // Higher than that -- run it verbose. Always show the progress
     // bar if requested explicitly, even in the quiet mode.
     //
-    if (verb < (fo ? 1 : 2))
+    if (!quiet)
     {
-      if (!progress)
+      if (verb < (fo ? 1 : 2))
       {
-        suppress_progress ();
-        no_progress = false;  // Already suppressed.
+        if (!progress)
+        {
+          suppress_progress ();
+          no_progress = false;  // Already suppressed.
+        }
       }
+      else if (fo && verb == 1)
+      {
+        if (!no_progress)
+          args.push_back ("--progress-bar");
+      }
+      else if (verb > 3)
+        args.push_back ("-v");
     }
-    else if (fo && verb == 1)
-    {
-      if (!no_progress)
-        args.push_back ("--progress-bar");
-    }
-    else if (verb > 3)
-      args.push_back ("-v");
 
     // Suppress progress.
     //
     // Note: the `-v -s` options combination is valid and results in a verbose
     // output without progress.
     //
-    if (no_progress)
+    if (no_progress || quiet)
       suppress_progress ();
 
     // Set download timeout if requested.
@@ -369,11 +395,23 @@ namespace bpkg
       args.push_back (http_proxy.c_str ());
     }
 
+    // Status code.
+    //
+    if (out_is != nullptr)
+    {
+      assert (!fo); // Currently unsupported (see start_fetch() for details).
+
+      args.push_back ("-i");
+    }
+
     args.push_back (url.c_str ());
     args.push_back (nullptr);
 
     process_path pp (process::path_search (args[0]));
 
+    // Let's still print the command line in the quiet mode to ease the
+    // troubleshooting.
+    //
     if (verb >= 2)
       print_process (args);
     else if (verb == 1 && fo && !no_progress)
@@ -386,9 +424,143 @@ namespace bpkg
 
     // Process exceptions must be handled by the caller.
     //
-    return fo
-      ? process (pp, args.data ())
-      : process (pp, args.data (), 0, -1);
+    process pr (fo
+                ? process (pp, args.data ())
+                : process (pp, args.data (),
+                           0, -1, out_is != nullptr ? -1 : 2));
+
+    // Close the process stdout stream and read stderr stream out and dump.
+    //
+    // Needs to be called prior to failing, so that the process won't get
+    // blocked writing to stdout and so that stderr get dumped before the
+    // error message we issue.
+    //
+    auto close_streams = [&pr, out_is] ()
+    {
+      try
+      {
+        out_is->close ();
+
+        bpkg::dump_stderr (move (pr.in_efd));
+      }
+      catch (const io_error&)
+      {
+        // Not much we can do here.
+      }
+    };
+
+    // If HTTP status code needs to be retrieved, then open the passed stream
+    // and read out the status line(s) and headers.
+    //
+    // Note that this implementation is inspired by the bdep's
+    // http_service::post() function.
+    //
+    uint16_t sc (0);
+
+    if (out_is != nullptr)
+    try
+    {
+      // At this stage we will read until the empty line (containing just
+      // CRLF). Not being able to reach such a line is an error, which is the
+      // reason for the exception mask choice. When done, we will restore the
+      // original exception mask.
+      //
+      ifdstream::iostate es (out_is->exceptions ());
+
+      out_is->exceptions (
+        ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit);
+
+      out_is->open (move (pr.in_ofd), out_ism);
+
+      // Parse and return the HTTP status code. Return 0 if the argument is
+      // invalid.
+      //
+      auto status_code = [] (const string& s)
+      {
+        char* e (nullptr);
+        unsigned long c (strtoul (s.c_str (), &e, 10)); // Can't throw.
+        assert (e != nullptr);
+
+        return *e == '\0' && c >= 100 && c < 600
+        ? static_cast<uint16_t> (c)
+        : 0;
+      };
+
+      // Read the CRLF-terminated line from the stream stripping the trailing
+      // CRLF.
+      //
+      auto read_line = [out_is] ()
+      {
+        string l;
+        getline (*out_is, l); // Strips the trailing LF (0xA).
+
+        // Note that on POSIX CRLF is not automatically translated into LF, so
+        // we need to strip CR (0xD) manually.
+        //
+        if (!l.empty () && l.back () == '\r')
+          l.pop_back ();
+
+        return l;
+      };
+
+      auto read_status = [&read_line, &status_code, &url, &close_streams] ()
+           -> uint16_t
+      {
+        string l (read_line ());
+
+        for (;;) // Breakout loop.
+        {
+          if (l.compare (0, 5, "HTTP/") != 0)
+            break;
+
+          size_t p (l.find (' ', 5));           // The protocol end.
+          if (p == string::npos)
+            break;
+
+          p = l.find_first_not_of (' ', p + 1); // The code start.
+          if (p == string::npos)
+            break;
+
+          size_t e (l.find (' ', p + 1));       // The code end.
+          if (e == string::npos)
+            break;
+
+          uint16_t c (status_code (string (l, p, e - p)));
+          if (c == 0)
+            break;
+
+          return c;
+        }
+
+        close_streams ();
+
+        fail << "invalid HTTP response status line '" << l
+             << "' while fetching " << url;
+
+        assert (false); // Can't be here.
+        return 0;
+      };
+
+      sc = read_status ();
+
+      if (sc == 100)
+      {
+        while (!read_line ().empty ()) ; // Skips the interim response.
+        sc = read_status ();             // Reads the final status code.
+      }
+
+      while (!read_line ().empty ()) ;   // Skips headers.
+
+      out_is->exceptions (es);
+    }
+    catch (const io_error&)
+    {
+      close_streams ();
+
+      fail << "unable to read HTTP response status line for " << url;
+    }
+
+    return make_pair (move (pr), sc);
   }
 
   // fetch
@@ -438,13 +610,24 @@ namespace bpkg
     return false;
   }
 
-  static process
+  // Note that there is no easy way to retrieve the HTTP status code for the
+  // fetch program and thus we always return 0.
+  //
+  // Also note that in the HTTP status code retrieval mode (out_is != NULL) we
+  // nevertheless redirect stderr to prevent the fetch program from
+  // interactively querying the user for the credentials. Thus, we also
+  // respect the quiet mode in contrast to start_wget().
+  //
+  static pair<process, uint16_t>
   start_fetch (const path& prog,
                const optional<size_t>& timeout,
                bool progress,
                bool no_progress,
+               bool quiet,
                const strings& ops,
                const string& url,
+               ifdstream* out_is,
+               fdstream_mode out_ism,
                const path& out,
                const string& user_agent,
                const string& http_proxy)
@@ -476,23 +659,26 @@ namespace bpkg
     // unless the verbosity level is greater than three, in which case we will
     // run verbose (and with progress). That's the best we can do.
     //
-    if (verb < (fo ? 1 : 2))
+    if (!quiet)
     {
-      if (!progress)
+      if (verb < (fo ? 1 : 2))
       {
-        args.push_back ("-q");
-        no_progress = false;   // Already suppressed with -q.
+        if (!progress)
+        {
+          args.push_back ("-q");
+          no_progress = false;   // Already suppressed with -q.
+        }
+      }
+      else if (verb > 3)
+      {
+        args.push_back ("-v");
+        no_progress = false; // Don't be quiet in the verbose mode (see above).
       }
-    }
-    else if (verb > 3)
-    {
-      args.push_back ("-v");
-      no_progress = false; // Don't be quiet in the verbose mode (see above).
     }
 
     // Suppress progress.
     //
-    if (no_progress)
+    if (no_progress || quiet)
       args.push_back ("-q");
 
     // Set download timeout if requested.
@@ -534,6 +720,9 @@ namespace bpkg
       env.vars = evars;
     }
 
+    // Let's still print the command line in the quiet mode to ease the
+    // troubleshooting.
+    //
     if (verb >= 2)
       print_process (env, args);
 
@@ -542,12 +731,19 @@ namespace bpkg
     // just the file name (rather than the whole path) in the progress
     // report. Process exceptions must be handled by the caller.
     //
-    return fo
-      ? process (pp, args.data (),
-                 0, 1, 2,
-                 out.directory ().string ().c_str (),
-                 env.vars)
-      : process (pp, args.data (), 0, -1, 2, nullptr /* cwd */, env.vars);
+    process pr (fo
+                ? process (pp, args.data (),
+                           0, 1, 2,
+                           out.directory ().string ().c_str (),
+                           env.vars)
+                : process (pp, args.data (),
+                           0, -1, out_is != nullptr ? -1 : 2,
+                           nullptr /* cwd */, env.vars));
+
+    if (out_is != nullptr)
+      out_is->open (move (pr.in_ofd), out_ism);
+
+    return make_pair (move (pr), 0);
   }
 
   // The dispatcher.
@@ -670,22 +866,38 @@ namespace bpkg
     return kind_;
   }
 
-  process
+  static pair<process, uint16_t>
   start_fetch (const common_options& o,
                const string& src,
+               ifdstream* out_is,
+               fdstream_mode out_ism,
+               bool quiet,
                const path& out,
                const string& user_agent,
                const url& proxy)
   {
-    process (*f) (const path&,
-                  const optional<size_t>&,
-                  bool,
-                  bool,
-                  const strings&,
-                  const string&,
-                  const path&,
-                  const string&,
-                  const string&) = nullptr;
+    // Currently, for the sake of simplicity, we don't support retrieving the
+    // HTTP status code if we fetch into a file.
+    //
+    assert (out.empty () || out_is == nullptr);
+
+    // Quiet mode is only meaningful if HTTP status code needs to be
+    // retrieved.
+    //
+    assert (!quiet || out_is != nullptr);
+
+    pair<process, uint16_t> (*f) (const path&,
+                                  const optional<size_t>&,
+                                  bool,
+                                  bool,
+                                  bool,
+                                  const strings&,
+                                  const string&,
+                                  ifdstream*,
+                                  fdstream_mode,
+                                  const path&,
+                                  const string&,
+                                  const string&) = nullptr;
 
     fetch_kind fk (check (o));
     switch (fk)
@@ -793,8 +1005,11 @@ namespace bpkg
                 timeout,
                 o.progress (),
                 o.no_progress (),
+                quiet,
                 os,
                 !http_url.empty () ? http_url : src,
+                out_is,
+                out_ism,
                 out,
                 user_agent,
                 http_proxy);
@@ -809,4 +1024,40 @@ namespace bpkg
       throw failed ();
     }
   }
+
+  process
+  start_fetch (const common_options& o,
+               const string& src,
+               const path& out,
+               const string& user_agent,
+               const url& proxy)
+  {
+    return start_fetch (o,
+                        src,
+                        nullptr /* out_is */,
+                        fdstream_mode::none,
+                        false /* quiet */,
+                        out,
+                        user_agent,
+                        proxy).first;
+  }
+
+  pair<process, uint16_t>
+  start_fetch_http (const common_options& o,
+                    const string& src,
+                    ifdstream& out,
+                    fdstream_mode out_mode,
+                    bool quiet,
+                    const string& user_agent,
+                    const url& proxy)
+  {
+    return start_fetch (o,
+                        src,
+                        &out,
+                        out_mode,
+                        quiet,
+                        path () /* out */,
+                        user_agent,
+                        proxy);
+  }
 }
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index c077079..78e77a7 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -138,11 +138,41 @@ namespace bpkg
   // option for details).
   //
   process
-  start_fetch (const common_options& o,
+  start_fetch (const common_options&,
                const string& url,
                const path& out = {},
                const string& user_agent = {},
                const butl::url& proxy = {});
+
+  // Similar to the above but can only be used for HTTP(S) URLs. Additionally
+  // return the HTTP status code, if the underlying fetch program provides an
+  // easy way to retrieve it, and 0 otherwise.
+  //
+  // In the former case redirect the fetch process stderr to a pipe, so that
+  // depending on the returned status code the caller can either drop or dump
+  // the fetch process diagnostics. May also redirect stderr in the latter
+  // case for some implementation-specific reasons (to prevent the underlying
+  // fetch program from interacting with the user, etc). The caller can detect
+  // whether stderr is redirected by checking process::in_efd.
+  //
+  // In contrast to start_fetch() always fetch to stdout, which can be read by
+  // the caller from the specified stream.
+  //
+  // If the quiet argument is true, then, if stderr is redirected, minimize
+  // the amount of diagnostics printed by the fetch program by only printing
+  // errors. That allows the caller to read stdout and stderr streams
+  // sequentially in the blocking mode by assuming that the diagnostics always
+  // fits into the pipe buffer. If stderr is not redirected, then the quiet
+  // argument is ignored in favor of the more informative diagnostics.
+  //
+  pair<process, uint16_t>
+  start_fetch_http (const common_options&,
+                    const string& url,
+                    ifdstream& out,
+                    fdstream_mode out_mode,
+                    bool quiet,
+                    const string& user_agent = {},
+                    const butl::url& proxy = {});
 }
 
 #endif // BPKG_FETCH_HXX
diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx
index 467e01f..52114df 100644
--- a/bpkg/utility.cxx
+++ b/bpkg/utility.cxx
@@ -366,4 +366,23 @@ namespace bpkg
       ? co.build ().string ().c_str ()
       : BPKG_EXE_PREFIX "b" BPKG_EXE_SUFFIX;
   }
+
+  void
+  dump_stderr (auto_fd&& fd)
+  {
+    ifdstream is (move (fd), fdstream_mode::skip, ifdstream::badbit);
+
+    // We could probably write something like this, instead:
+    //
+    // *diag_stream << is.rdbuf () << flush;
+    //
+    // However, it would never throw and we could potentially miss the reading
+    // failure, unless we decide to additionally mess with the diagnostics
+    // stream exception mask.
+    //
+    for (string l; !eof (getline (is, l)); )
+      *diag_stream << l << endl;
+
+    is.close ();
+  }
 }
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index 1f5c725..8e7260a 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -246,6 +246,12 @@ namespace bpkg
   template <typename... A>
   void
   run_b (const common_options&, verb_b, A&&... args);
+
+  // Read out the data from the specified file descriptor and dump it to
+  // stderr. Throw io_error on the underlying OS errors.
+  //
+  void
+  dump_stderr (auto_fd&&);
 }
 
 #include <bpkg/utility.txx>