diff options
Diffstat (limited to 'libbutl/fdstream.cxx')
-rw-r--r-- | libbutl/fdstream.cxx | 488 |
1 files changed, 382 insertions, 106 deletions
diff --git a/libbutl/fdstream.cxx b/libbutl/fdstream.cxx index 9f54852..07cb9f2 100644 --- a/libbutl/fdstream.cxx +++ b/libbutl/fdstream.cxx @@ -1,9 +1,7 @@ // file : libbutl/fdstream.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file -#ifndef __cpp_modules_ts -#include <libbutl/fdstream.mxx> -#endif +#include <libbutl/fdstream.hxx> #include <errno.h> // errno, E* @@ -12,69 +10,54 @@ # include <unistd.h> // close(), read(), write(), lseek(), dup(), pipe(), // ftruncate(), isatty(), ssize_t, STD*_FILENO # include <sys/uio.h> // writev(), iovec -# include <sys/stat.h> // stat(), S_I* +# include <sys/stat.h> // stat(), fstat(), S_I* # include <sys/time.h> // timeval # include <sys/types.h> // stat, off_t # include <sys/select.h> #else # include <libbutl/win32-utility.hxx> -# include <io.h> // _close(), _read(), _write(), _setmode(), _sopen(), - // _lseek(), _dup(), _pipe(), _chsize_s, - // _get_osfhandle() -# include <share.h> // _SH_DENYNO -# include <stdio.h> // _fileno(), stdin, stdout, stderr, SEEK_* -# include <fcntl.h> // _O_* -# include <sys/stat.h> // S_I* +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x04 +# endif + +# include <io.h> // _close(), _read(), _write(), _setmode(), _sopen(), + // _lseek(), _dup(), _pipe(), _chsize_s, + // _get_osfhandle() +# include <share.h> // _SH_DENYNO +# include <stdio.h> // _fileno(), stdin, stdout, stderr, SEEK_* +# include <fcntl.h> // _O_* +# include <sys/types.h> // _stat +# include <sys/stat.h> // fstat(), S_I* + +# ifdef _MSC_VER // Unlikely to be fixed in newer versions. +# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +# define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) +# endif # include <wchar.h> // wcsncmp(), wcsstr() +# include <thread> // this_thread::yield() # include <algorithm> // count() #endif -#include <cassert> - -#ifndef __cpp_lib_modules_ts -#include <vector> -#include <string> -#include <istream> -#include <ostream> -#include <utility> -#include <cstdint> -#include <cstddef> - #include <ios> // ios_base::openmode, ios_base::failure #include <new> // bad_alloc #include <limits> // numeric_limits -#include <cstring> // memcpy(), memmove() +#include <cassert> +#include <cstring> // memcpy(), memmove(), memchr(), strcmp() +#include <cstdlib> // getenv() #include <iostream> // cin, cout #include <exception> // uncaught_exception[s]() #include <stdexcept> // invalid_argument #include <system_error> -#endif -#include <libbutl/ft/exception.hxx> // uncaught_exceptions +#include <libbutl/ft/exception.hxx> // uncaught_exceptions #include <libbutl/process-details.hxx> -#ifdef __cpp_modules_ts -module butl.fdstream; - -// Only imports additional to interface. -#ifdef __clang__ -#ifdef __cpp_lib_modules_ts -import std.core; -import std.io; -import std.threading; // Clang wants it in purview (see process-details.hxx). -#endif -import butl.path; -import butl.filesystem; -import butl.small_vector; -#endif - -import butl.utility; // throw_*_ios_failure(), function_cast() -#else -#include <libbutl/utility.mxx> -#endif +#include <libbutl/utility.hxx> // throw_*_ios_failure(), function_cast() +#include <libbutl/timestamp.hxx> using namespace std; @@ -164,7 +147,7 @@ namespace butl } #endif - // fdbuf + // fdstreambuf // // Return true if the file descriptor is in the non-blocking mode. Throw // ios::failure on the underlying OS error. @@ -185,7 +168,7 @@ namespace butl #endif } - void fdbuf:: + void fdstreambuf:: open (auto_fd&& fd, uint64_t pos) { close (); @@ -198,7 +181,7 @@ namespace butl fd_ = move (fd); } - bool fdbuf:: + bool fdstreambuf:: blocking (bool m) { // Verify that the file descriptor is open. @@ -222,7 +205,7 @@ namespace butl return !m; } - streamsize fdbuf:: + streamsize fdstreambuf:: showmanyc () { if (!is_open ()) @@ -257,7 +240,7 @@ namespace butl return 0; } - fdbuf::int_type fdbuf:: + fdstreambuf::int_type fdstreambuf:: underflow () { int_type r (traits_type::eof ()); @@ -279,7 +262,7 @@ namespace butl return r; } - bool fdbuf:: + bool fdstreambuf:: load () { // Doesn't handle blocking mode and so should not be called. @@ -296,7 +279,7 @@ namespace butl return n != 0; } - void fdbuf:: + void fdstreambuf:: seekg (uint64_t off) { // In the future we may implement the blocking behavior for a non-blocking @@ -331,7 +314,7 @@ namespace butl setg (buf_, buf_, buf_); } - fdbuf::int_type fdbuf:: + fdstreambuf::int_type fdstreambuf:: overflow (int_type c) { int_type r (traits_type::eof ()); @@ -359,7 +342,7 @@ namespace butl return r; } - int fdbuf:: + int fdstreambuf:: sync () { if (!is_open ()) @@ -376,15 +359,7 @@ namespace butl return save () ? 0 : -1; } -#ifdef _WIN32 - static inline int - write (int fd, const void* buf, size_t n) - { - return _write (fd, buf, static_cast<unsigned int> (n)); - } -#endif - - bool fdbuf:: + bool fdstreambuf:: save () { size_t n (pptr () - pbase ()); @@ -395,7 +370,7 @@ namespace butl // descriptor opened for read-only access (while -1 with errno EBADF is // expected). This is in contrast with VC's _write() and POSIX's write(). // - auto m (write (fd_.get (), buf_, n)); + auto m (fdwrite (fd_.get (), buf_, n)); if (m == -1) throw_generic_ios_failure (errno); @@ -411,7 +386,7 @@ namespace butl return true; } - streamsize fdbuf:: + streamsize fdstreambuf:: xsputn (const char_type* s, streamsize sn) { // The xsputn() function interface doesn't support the non-blocking @@ -510,7 +485,7 @@ namespace butl // Flush the buffer. // size_t wn (bn + an); - int r (wn > 0 ? write (fd_.get (), buf_, wn) : 0); + streamsize r (wn > 0 ? fdwrite (fd_.get (), buf_, wn) : 0); if (r == -1) throw_generic_ios_failure (errno); @@ -553,7 +528,7 @@ namespace butl // The data tail doesn't fit the buffer so write it to the file. // - r = write (fd_.get (), s, n); + r = fdwrite (fd_.get (), s, n); if (r == -1) throw_generic_ios_failure (errno); @@ -568,13 +543,13 @@ namespace butl // // - basic_ostream::seekp(pos) -> // basic_streambuf::pubseekpos(pos, ios::out) -> - // fdbuf::seekpos(pos, ios::out) + // fdstreambuf::seekpos(pos, ios::out) // // - basic_istream::seekg(pos) -> // basic_streambuf::pubseekpos(pos, ios::in) -> - // fdbuf::seekpos(pos, ios::in) + // fdstreambuf::seekpos(pos, ios::in) // - fdbuf::pos_type fdbuf:: + fdstreambuf::pos_type fdstreambuf:: seekpos (pos_type pos, ios_base::openmode which) { // Note that the position type provides an explicit conversion to the @@ -589,21 +564,21 @@ namespace butl // // - basic_ostream::seekp(off, dir) -> // basic_streambuf::pubseekoff(off, dir, ios::out) -> - // fdbuf::seekoff(off, dir, ios::out) + // fdstreambuf::seekoff(off, dir, ios::out) // // - basic_ostream::tellp() -> // basic_streambuf::pubseekoff(0, ios::cur, ios::out) -> - // fdbuf::seekoff(0, ios::cur, ios::out) + // fdstreambuf::seekoff(0, ios::cur, ios::out) // // - basic_istream::seekg(off, dir) -> // basic_streambuf::pubseekoff(off, dir, ios::in) -> - // fdbuf::seekoff(off, dir, ios::in) + // fdstreambuf::seekoff(off, dir, ios::in) // // - basic_istream::tellg() -> // basic_streambuf::pubseekoff(0, ios::cur, ios::in) -> - // fdbuf::seekoff(0, ios::cur, ios::in) + // fdstreambuf::seekoff(0, ios::cur, ios::in) // - fdbuf::pos_type fdbuf:: + fdstreambuf::pos_type fdstreambuf:: seekoff (off_type off, ios_base::seekdir dir, ios_base::openmode which) { // The seekoff() function interface doesn't support the non-blocking @@ -827,9 +802,8 @@ namespace butl catch (const ios_base::failure&) {} } - // Underlying file descriptor is closed by fdbuf dtor with errors (if any) - // being ignored. - // + // Underlying file descriptor is closed by fdstreambuf dtor with errors + // (if any) being ignored. } void ifdstream:: @@ -848,6 +822,13 @@ namespace butl } void ifdstream:: + open (auto_fd&& fd, fdstream_mode m, std::uint64_t pos) + { + open (mode (std::move (fd), m), pos); + skip_ = (m & fdstream_mode::skip) == fdstream_mode::skip; + } + + void ifdstream:: close () { if (skip_ && is_open () && good ()) @@ -863,7 +844,7 @@ namespace butl } ifdstream& - getline (ifdstream& is, string& s, char delim) + getline (ifdstream& is, string& l, char delim) { ifdstream::iostate eb (is.exceptions ()); assert (eb & ifdstream::badbit); @@ -871,16 +852,16 @@ namespace butl // Amend the exception mask to prevent exceptions being thrown by the C++ // IO runtime to avoid incompatibility issues due to ios_base::failure ABI // fiasco (#66145). We will not restore the mask when ios_base::failure is - // thrown by fdbuf since there is no way to "silently" restore it if the - // corresponding bits are in the error state without the exceptions() call - // throwing ios_base::failure. Not restoring exception mask on throwing - // because of badbit should probably be ok since the stream is no longer - // usable. + // thrown by fdstreambuf since there is no way to "silently" restore it if + // the corresponding bits are in the error state without the exceptions() + // call throwing ios_base::failure. Not restoring exception mask on + // throwing because of badbit should probably be ok since the stream is no + // longer usable. // if (eb != ifdstream::badbit) is.exceptions (ifdstream::badbit); - std::getline (is, s, delim); + std::getline (is, l, delim); // Throw if any of the newly set bits are present in the exception mask. // @@ -893,6 +874,58 @@ namespace butl return is; } + bool + getline_non_blocking (ifdstream& is, string& l, char delim) + { + assert (!is.blocking () && (is.exceptions () & ifdstream::badbit) != 0); + + fdstreambuf& sb (*static_cast<fdstreambuf*> (is.rdbuf ())); + + // Read until blocked (0), EOF (-1) or encounter the delimiter. + // + // Note that here we reasonably assume that any failure in in_avail() + // will lead to badbit and thus an exception (see showmanyc()). + // + streamsize s; + while ((s = sb.in_avail ()) > 0) + { + const char* p (sb.gptr ()); + size_t n (sb.egptr () - p); + + const char* e (static_cast<const char*> (memchr (p, delim, n))); + if (e != nullptr) + n = e - p; + + l.append (p, n); + + // Note: consume the delimiter if found. + // + sb.gbump (static_cast<int> (n + (e != nullptr ? 1 : 0))); + + if (e != nullptr) + break; + } + + // Here s can be: + // + // -1 -- EOF. + // 0 -- blocked before encountering delimiter/EOF. + // >0 -- encountered the delimiter. + // + if (s == -1) + { + is.setstate (ifdstream::eofbit); + + // If we couldn't extract anything, not even the delimiter, then this is + // a failure per the getline() interface. + // + if (l.empty ()) + is.setstate (ifdstream::failbit); + } + + return s != 0; + } + // ofdstream // ofdstream:: @@ -1042,10 +1075,11 @@ namespace butl #endif // Unlike other platforms, *BSD allows opening a directory as a file which - // will cause all kinds of problems upstream (e.g., cpfile()). So we detect - // and diagnose this. + // will cause all kinds of problems upstream (e.g., cpfile()). So we + // detect and diagnose this. Note: not certain this is the case for NetBSD + // and OpenBSD. // -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) { struct stat s; if (stat (f, &s) == 0 && S_ISDIR (s.st_mode)) @@ -1123,9 +1157,45 @@ namespace butl // of |= _O_NOINHERIT; - int fd (pass_perm - ? _sopen (f, of, _SH_DENYNO, pf) - : _sopen (f, of, _SH_DENYNO)); + int fd; + + // For reasons unknown an attempt to open an existing file for writing + // sometimes ends up with the EACCES POSIX error which is presumably a + // translation of the ERROR_SHARING_VIOLATION system error returned by the + // underlying CreateFile() function call (see mventry() for details). If + // that's the case, we will keep trying to open the file for two seconds. + // + // Also, it turns out, if someone memory-maps a file, it takes Windows + // some time to realize it's been unmapped and until then any attempt to + // open it results in EINVAL POSIX error, ERROR_USER_MAPPED_FILE system + // error. So we retry those as well. + // + for (size_t i (0); i < 41; ++i) + { + // Sleep 50 milliseconds before the open retry. + // + if (i != 0) + Sleep (50); + + fd = pass_perm + ? _sopen (f, of, _SH_DENYNO, pf) + : _sopen (f, of, _SH_DENYNO); + + // Note that MSVCRT's _sopen() calls CreateFile() underneath, + // translating the system error to POSIX on failure and bailing out + // afterwords. Thus, we can query the original error code on _sopen() + // failure. + // + // Note that MinGW's _sopen() is just a stub forwarding the call to the + // (publicly available) MSVCRT's implementation. + // + if (!(fd == -1 && + out && + (errno == EACCES || errno == EINVAL) && + (GetLastError () == ERROR_SHARING_VIOLATION || + GetLastError () == ERROR_USER_MAPPED_FILE))) + break; + } #endif @@ -1332,6 +1402,28 @@ namespace butl throw_generic_ios_failure (errno); } + entry_stat + fdstat (int fd) + { + struct stat s; + if (fstat (fd, &s) != 0) + throw_generic_error (errno); + + auto m (s.st_mode); + entry_type t (entry_type::unknown); + + // Note: cannot be a symlink. + // + if (S_ISREG (m)) + t = entry_type::regular; + else if (S_ISDIR (m)) + t = entry_type::directory; + else if (S_ISBLK (m) || S_ISCHR (m) || S_ISFIFO (m) || S_ISSOCK (m)) + t = entry_type::other; + + return entry_stat {t, static_cast<uint64_t> (s.st_size)}; + } + bool fdterm (int fd) { @@ -1352,9 +1444,23 @@ namespace butl throw_generic_ios_failure (errno); } - pair<size_t, size_t> - fdselect (fdselect_set& read, fdselect_set& write) + bool + fdterm_color (int, bool) + { + const char* t (std::getenv ("TERM")); + + // This test was lifted from GCC (Emacs shell sets TERM=dumb). + // + return t != nullptr && strcmp (t, "dumb") != 0; + } + + static pair<size_t, size_t> + fdselect (fdselect_set& read, + fdselect_set& write, + const chrono::milliseconds* timeout) { + using namespace chrono; + // Copy fdselect_set into the native fd_set, updating max_fd. Also clear // the ready flag in the source set. // @@ -1366,6 +1472,8 @@ namespace butl for (fdselect_state& s: from) { + s.ready = false; + if (s.fd == nullfd) continue; @@ -1373,7 +1481,6 @@ namespace butl throw invalid_argument ("invalid file descriptor"); FD_SET (s.fd, &to); - s.ready = false; if (max_fd < s.fd) max_fd = s.fd; @@ -1390,29 +1497,64 @@ namespace butl ++max_fd; + // Note that if the timeout is not NULL, then the select timeout needs to + // be recalculated for each select() call (of which we can potentially + // have multiple due to EINTR). So the timeout can be used as bool. + // + timestamp now; + timestamp deadline; + + if (timeout) + { + now = system_clock::now (); + deadline = now + *timeout; + } + // Repeat the select() call while getting the EINTR error and throw on // any other error. // // Note that select() doesn't modify the sets on failure (according to // POSIX standard as well as to the Linux, FreeBSD and MacOS man pages). // - for (;;) + for (timeval tv;;) { + if (timeout) + { + if (now < deadline) + { + microseconds t (duration_cast<microseconds> (deadline - now)); + tv.tv_sec = t.count () / 1000000; + tv.tv_usec = t.count () % 1000000; + } + else + { + tv.tv_sec = 0; + tv.tv_usec = 0; + } + } + int r (select (max_fd, &rds, &wds, nullptr /* exceptfds */, - nullptr /* timeout */)); + timeout ? &tv : nullptr)); if (r == -1) { if (errno == EINTR) + { + if (timeout) + now = system_clock::now (); + continue; + } throw_system_ios_failure (errno); } - assert (r != 0); // We don't expect the timeout to occur. + if (!timeout) + assert (r != 0); // We don't expect the timeout to occur. + break; } @@ -1445,6 +1587,12 @@ namespace butl return read (fd, buf, n); } + streamsize + fdwrite (int fd, const void* buf, size_t n) + { + return write (fd, buf, n); + } + #else auto_fd @@ -1537,11 +1685,19 @@ namespace butl } catch (const system_error& e) { - // Make sure that the error denotes errno portable code. + // Re-throw system_error as ios::failure, preserving the error category + // and description. // - assert (e.code ().category () == generic_category ()); + int v (e.code ().value ()); + const error_category& c (e.code ().category ()); - throw_generic_ios_failure (e.code ().value ()); + if (c == generic_category ()) + throw_generic_ios_failure (v); + else + { + assert (c == system_category ()); + throw_system_ios_failure (v, e.what ()); + } } } @@ -1692,9 +1848,34 @@ namespace butl throw_generic_ios_failure (e); } + entry_stat + fdstat (int fd) + { + // Since symlinks have been taken care of, we can just _fstat(). + // + struct __stat64 s; + if (_fstat64 (fd, &s) != 0) + throw_generic_error (errno); + + auto m (s.st_mode); + entry_type t (entry_type::unknown); + + if (S_ISREG (m)) + t = entry_type::regular; + else if (S_ISDIR (m)) + t = entry_type::directory; + else if (S_ISCHR (m)) + t = entry_type::other; + + return entry_stat {t, static_cast<uint64_t> (s.st_size)}; + } + bool fdterm (int fd) { + // @@ Both GCC and Clang simply call GetConsoleMode() for this check. I + // wonder why we don't do the same? See also fdterm_color() below. + // We don't need to close it (see fd_to_handle()). // HANDLE h (fd_to_handle (fd)); @@ -1780,9 +1961,49 @@ namespace butl return false; } - pair<size_t, size_t> - fdselect (fdselect_set& read, fdselect_set& write) + bool + fdterm_color (int fd, bool enable) + { + // We don't need to close it (see fd_to_handle()). + // + HANDLE h (fd_to_handle (fd)); + + // See GH issue #312 for background on this logic. + // + DWORD m; + if (!GetConsoleMode (h, &m)) + throw_system_ios_failure (GetLastError ()); + + // Some terminals (e.g. Windows Terminal) enable VT processing by default. + // + if ((m & ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0) + return true; + + if (enable) + { + // If SetConsoleMode() fails, assume VT processing is unsupported (it + // is only supported from a certain build of Windows 10). + // + // Note that Wine pretends to support this but doesn't handle the escape + // sequences. See https://bugs.winehq.org/show_bug.cgi?id=49780. + // + if (SetConsoleMode (h, + (m | + ENABLE_PROCESSED_OUTPUT | + ENABLE_VIRTUAL_TERMINAL_PROCESSING))) + return true; + } + + return false; + } + + static pair<size_t, size_t> + fdselect (fdselect_set& read, + fdselect_set& write, + const chrono::milliseconds* timeout) { + using namespace chrono; + if (!write.empty ()) throw invalid_argument ("write file descriptor set is not supported"); @@ -1792,25 +2013,35 @@ namespace butl for (fdselect_state& s: read) { + s.ready = false; + if (s.fd == nullfd) continue; if (s.fd < 0) throw invalid_argument ("invalid file descriptor"); - s.ready = false; ++n; } if (n == 0) throw invalid_argument ("empty file descriptor set"); + // Note that if the timeout is not NULL, then the deadline needs to be + // checked prior to re-probing the pipe for data presence. So the timeout + // can be used as bool. + // + timestamp deadline; + + if (timeout) + deadline = system_clock::now () + *timeout; + // Keep iterating through the set until at least one byte can be read from // any of the pipes or any of them get closed (so can read eof). // size_t r (0); - while (true) + for (size_t i (0);; ++i) { for (fdselect_state& s: read) { @@ -1877,13 +2108,37 @@ namespace butl throw_system_ios_failure (e); } - // Bail out if some descriptors are ready for reading and sleep a bit - // and repeat otherwise. + // Bail out if some descriptors are ready for reading or the deadline + // has been reached, if specified, and sleep a bit and repeat otherwise. // if (r != 0) break; - Sleep (50); + // Use exponential backoff but not too aggressive and with 25ms max. + // + DWORD t ( + static_cast<DWORD> (i <= 1000 ? 0 : + i >= 1000 + 100 ? 25 : 1 + ((i - 1000) / 4))); + + if (timeout) + { + timestamp now (system_clock::now ()); + + if (now < deadline) + { + milliseconds tm (duration_cast<milliseconds> (deadline - now)); + + if (t > tm.count ()) + t = static_cast<DWORD> (tm.count ()); + } + else + break; + } + + if (t == 0) + this_thread::yield (); + else + Sleep (t); } return make_pair (r, 0); @@ -1926,5 +2181,26 @@ namespace butl return r; } + streamsize + fdwrite (int fd, const void* buf, size_t n) + { + return _write (fd, buf, static_cast<unsigned int> (n)); + } + #endif + + pair<size_t, size_t> + fdselect (fdselect_set& read, fdselect_set& write) + { + return fdselect (read, write, nullptr /* timeout */); + } + + template <> + pair<size_t, size_t> + fdselect (fdselect_set& read, + fdselect_set& write, + const chrono::milliseconds& timeout) + { + return fdselect (read, write, &timeout); + } } |