diff options
Diffstat (limited to 'libbutl/fdstream.cxx')
| -rw-r--r-- | libbutl/fdstream.cxx | 348 |
1 files changed, 246 insertions, 102 deletions
diff --git a/libbutl/fdstream.cxx b/libbutl/fdstream.cxx index 4948052..07cb9f2 100644 --- a/libbutl/fdstream.cxx +++ b/libbutl/fdstream.cxx @@ -1,9 +1,7 @@ // file : libbutl/fdstream.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file -#ifndef __cpp_modules_ts -#include <libbutl/fdstream.mxx> -#endif +#include <libbutl/fdstream.hxx> #include <errno.h> // errno, E* @@ -12,72 +10,54 @@ # include <unistd.h> // close(), read(), write(), lseek(), dup(), pipe(), // ftruncate(), isatty(), ssize_t, STD*_FILENO # include <sys/uio.h> // writev(), iovec -# include <sys/stat.h> // stat(), S_I* +# include <sys/stat.h> // stat(), fstat(), S_I* # include <sys/time.h> // timeval # include <sys/types.h> // stat, off_t # include <sys/select.h> #else # include <libbutl/win32-utility.hxx> -# include <io.h> // _close(), _read(), _write(), _setmode(), _sopen(), - // _lseek(), _dup(), _pipe(), _chsize_s, - // _get_osfhandle() -# include <share.h> // _SH_DENYNO -# include <stdio.h> // _fileno(), stdin, stdout, stderr, SEEK_* -# include <fcntl.h> // _O_* -# include <sys/stat.h> // S_I* +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x04 +# endif + +# include <io.h> // _close(), _read(), _write(), _setmode(), _sopen(), + // _lseek(), _dup(), _pipe(), _chsize_s, + // _get_osfhandle() +# include <share.h> // _SH_DENYNO +# include <stdio.h> // _fileno(), stdin, stdout, stderr, SEEK_* +# include <fcntl.h> // _O_* +# include <sys/types.h> // _stat +# include <sys/stat.h> // fstat(), S_I* + +# ifdef _MSC_VER // Unlikely to be fixed in newer versions. +# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +# define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) +# endif # include <wchar.h> // wcsncmp(), wcsstr() +# include <thread> // this_thread::yield() # include <algorithm> // count() #endif -#include <cassert> - -#ifndef __cpp_lib_modules_ts -#include <vector> -#include <string> -#include <chrono> -#include <istream> -#include <ostream> -#include <utility> -#include <cstdint> -#include <cstddef> - #include <ios> // ios_base::openmode, ios_base::failure #include <new> // bad_alloc #include <limits> // numeric_limits -#include <cstring> // memcpy(), memmove() +#include <cassert> +#include <cstring> // memcpy(), memmove(), memchr(), strcmp() +#include <cstdlib> // getenv() #include <iostream> // cin, cout #include <exception> // uncaught_exception[s]() #include <stdexcept> // invalid_argument #include <system_error> -#endif -#include <libbutl/ft/exception.hxx> // uncaught_exceptions +#include <libbutl/ft/exception.hxx> // uncaught_exceptions #include <libbutl/process-details.hxx> -#ifdef __cpp_modules_ts -module butl.fdstream; - -// Only imports additional to interface. -#ifdef __clang__ -#ifdef __cpp_lib_modules_ts -import std.core; -import std.io; -import std.threading; // Clang wants it in purview (see process-details.hxx). -#endif -import butl.path; -import butl.filesystem; -import butl.small_vector; -#endif - -import butl.utility; // throw_*_ios_failure(), function_cast() -import butl.timestamp; -#else -#include <libbutl/utility.mxx> -#include <libbutl/timestamp.mxx> -#endif +#include <libbutl/utility.hxx> // throw_*_ios_failure(), function_cast() +#include <libbutl/timestamp.hxx> using namespace std; @@ -167,7 +147,7 @@ namespace butl } #endif - // fdbuf + // fdstreambuf // // Return true if the file descriptor is in the non-blocking mode. Throw // ios::failure on the underlying OS error. @@ -188,7 +168,7 @@ namespace butl #endif } - void fdbuf:: + void fdstreambuf:: open (auto_fd&& fd, uint64_t pos) { close (); @@ -201,7 +181,7 @@ namespace butl fd_ = move (fd); } - bool fdbuf:: + bool fdstreambuf:: blocking (bool m) { // Verify that the file descriptor is open. @@ -225,7 +205,7 @@ namespace butl return !m; } - streamsize fdbuf:: + streamsize fdstreambuf:: showmanyc () { if (!is_open ()) @@ -260,7 +240,7 @@ namespace butl return 0; } - fdbuf::int_type fdbuf:: + fdstreambuf::int_type fdstreambuf:: underflow () { int_type r (traits_type::eof ()); @@ -282,7 +262,7 @@ namespace butl return r; } - bool fdbuf:: + bool fdstreambuf:: load () { // Doesn't handle blocking mode and so should not be called. @@ -299,7 +279,7 @@ namespace butl return n != 0; } - void fdbuf:: + void fdstreambuf:: seekg (uint64_t off) { // In the future we may implement the blocking behavior for a non-blocking @@ -334,7 +314,7 @@ namespace butl setg (buf_, buf_, buf_); } - fdbuf::int_type fdbuf:: + fdstreambuf::int_type fdstreambuf:: overflow (int_type c) { int_type r (traits_type::eof ()); @@ -362,7 +342,7 @@ namespace butl return r; } - int fdbuf:: + int fdstreambuf:: sync () { if (!is_open ()) @@ -379,15 +359,7 @@ namespace butl return save () ? 0 : -1; } -#ifdef _WIN32 - static inline int - write (int fd, const void* buf, size_t n) - { - return _write (fd, buf, static_cast<unsigned int> (n)); - } -#endif - - bool fdbuf:: + bool fdstreambuf:: save () { size_t n (pptr () - pbase ()); @@ -398,7 +370,7 @@ namespace butl // descriptor opened for read-only access (while -1 with errno EBADF is // expected). This is in contrast with VC's _write() and POSIX's write(). // - auto m (write (fd_.get (), buf_, n)); + auto m (fdwrite (fd_.get (), buf_, n)); if (m == -1) throw_generic_ios_failure (errno); @@ -414,7 +386,7 @@ namespace butl return true; } - streamsize fdbuf:: + streamsize fdstreambuf:: xsputn (const char_type* s, streamsize sn) { // The xsputn() function interface doesn't support the non-blocking @@ -513,7 +485,7 @@ namespace butl // Flush the buffer. // size_t wn (bn + an); - int r (wn > 0 ? write (fd_.get (), buf_, wn) : 0); + streamsize r (wn > 0 ? fdwrite (fd_.get (), buf_, wn) : 0); if (r == -1) throw_generic_ios_failure (errno); @@ -556,7 +528,7 @@ namespace butl // The data tail doesn't fit the buffer so write it to the file. // - r = write (fd_.get (), s, n); + r = fdwrite (fd_.get (), s, n); if (r == -1) throw_generic_ios_failure (errno); @@ -571,13 +543,13 @@ namespace butl // // - basic_ostream::seekp(pos) -> // basic_streambuf::pubseekpos(pos, ios::out) -> - // fdbuf::seekpos(pos, ios::out) + // fdstreambuf::seekpos(pos, ios::out) // // - basic_istream::seekg(pos) -> // basic_streambuf::pubseekpos(pos, ios::in) -> - // fdbuf::seekpos(pos, ios::in) + // fdstreambuf::seekpos(pos, ios::in) // - fdbuf::pos_type fdbuf:: + fdstreambuf::pos_type fdstreambuf:: seekpos (pos_type pos, ios_base::openmode which) { // Note that the position type provides an explicit conversion to the @@ -592,21 +564,21 @@ namespace butl // // - basic_ostream::seekp(off, dir) -> // basic_streambuf::pubseekoff(off, dir, ios::out) -> - // fdbuf::seekoff(off, dir, ios::out) + // fdstreambuf::seekoff(off, dir, ios::out) // // - basic_ostream::tellp() -> // basic_streambuf::pubseekoff(0, ios::cur, ios::out) -> - // fdbuf::seekoff(0, ios::cur, ios::out) + // fdstreambuf::seekoff(0, ios::cur, ios::out) // // - basic_istream::seekg(off, dir) -> // basic_streambuf::pubseekoff(off, dir, ios::in) -> - // fdbuf::seekoff(off, dir, ios::in) + // fdstreambuf::seekoff(off, dir, ios::in) // // - basic_istream::tellg() -> // basic_streambuf::pubseekoff(0, ios::cur, ios::in) -> - // fdbuf::seekoff(0, ios::cur, ios::in) + // fdstreambuf::seekoff(0, ios::cur, ios::in) // - fdbuf::pos_type fdbuf:: + fdstreambuf::pos_type fdstreambuf:: seekoff (off_type off, ios_base::seekdir dir, ios_base::openmode which) { // The seekoff() function interface doesn't support the non-blocking @@ -830,9 +802,8 @@ namespace butl catch (const ios_base::failure&) {} } - // Underlying file descriptor is closed by fdbuf dtor with errors (if any) - // being ignored. - // + // Underlying file descriptor is closed by fdstreambuf dtor with errors + // (if any) being ignored. } void ifdstream:: @@ -873,7 +844,7 @@ namespace butl } ifdstream& - getline (ifdstream& is, string& s, char delim) + getline (ifdstream& is, string& l, char delim) { ifdstream::iostate eb (is.exceptions ()); assert (eb & ifdstream::badbit); @@ -881,16 +852,16 @@ namespace butl // Amend the exception mask to prevent exceptions being thrown by the C++ // IO runtime to avoid incompatibility issues due to ios_base::failure ABI // fiasco (#66145). We will not restore the mask when ios_base::failure is - // thrown by fdbuf since there is no way to "silently" restore it if the - // corresponding bits are in the error state without the exceptions() call - // throwing ios_base::failure. Not restoring exception mask on throwing - // because of badbit should probably be ok since the stream is no longer - // usable. + // thrown by fdstreambuf since there is no way to "silently" restore it if + // the corresponding bits are in the error state without the exceptions() + // call throwing ios_base::failure. Not restoring exception mask on + // throwing because of badbit should probably be ok since the stream is no + // longer usable. // if (eb != ifdstream::badbit) is.exceptions (ifdstream::badbit); - std::getline (is, s, delim); + std::getline (is, l, delim); // Throw if any of the newly set bits are present in the exception mask. // @@ -903,6 +874,58 @@ namespace butl return is; } + bool + getline_non_blocking (ifdstream& is, string& l, char delim) + { + assert (!is.blocking () && (is.exceptions () & ifdstream::badbit) != 0); + + fdstreambuf& sb (*static_cast<fdstreambuf*> (is.rdbuf ())); + + // Read until blocked (0), EOF (-1) or encounter the delimiter. + // + // Note that here we reasonably assume that any failure in in_avail() + // will lead to badbit and thus an exception (see showmanyc()). + // + streamsize s; + while ((s = sb.in_avail ()) > 0) + { + const char* p (sb.gptr ()); + size_t n (sb.egptr () - p); + + const char* e (static_cast<const char*> (memchr (p, delim, n))); + if (e != nullptr) + n = e - p; + + l.append (p, n); + + // Note: consume the delimiter if found. + // + sb.gbump (static_cast<int> (n + (e != nullptr ? 1 : 0))); + + if (e != nullptr) + break; + } + + // Here s can be: + // + // -1 -- EOF. + // 0 -- blocked before encountering delimiter/EOF. + // >0 -- encountered the delimiter. + // + if (s == -1) + { + is.setstate (ifdstream::eofbit); + + // If we couldn't extract anything, not even the delimiter, then this is + // a failure per the getline() interface. + // + if (l.empty ()) + is.setstate (ifdstream::failbit); + } + + return s != 0; + } + // ofdstream // ofdstream:: @@ -1052,10 +1075,11 @@ namespace butl #endif // Unlike other platforms, *BSD allows opening a directory as a file which - // will cause all kinds of problems upstream (e.g., cpfile()). So we detect - // and diagnose this. + // will cause all kinds of problems upstream (e.g., cpfile()). So we + // detect and diagnose this. Note: not certain this is the case for NetBSD + // and OpenBSD. // -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) { struct stat s; if (stat (f, &s) == 0 && S_ISDIR (s.st_mode)) @@ -1141,12 +1165,17 @@ namespace butl // underlying CreateFile() function call (see mventry() for details). If // that's the case, we will keep trying to open the file for two seconds. // - for (size_t i (0); i < 21; ++i) + // Also, it turns out, if someone memory-maps a file, it takes Windows + // some time to realize it's been unmapped and until then any attempt to + // open it results in EINVAL POSIX error, ERROR_USER_MAPPED_FILE system + // error. So we retry those as well. + // + for (size_t i (0); i < 41; ++i) { - // Sleep 100 milliseconds before the open retry. + // Sleep 50 milliseconds before the open retry. // if (i != 0) - Sleep (100); + Sleep (50); fd = pass_perm ? _sopen (f, of, _SH_DENYNO, pf) @@ -1160,10 +1189,11 @@ namespace butl // Note that MinGW's _sopen() is just a stub forwarding the call to the // (publicly available) MSVCRT's implementation. // - if (!(fd == -1 && - out && - errno == EACCES && - GetLastError () == ERROR_SHARING_VIOLATION)) + if (!(fd == -1 && + out && + (errno == EACCES || errno == EINVAL) && + (GetLastError () == ERROR_SHARING_VIOLATION || + GetLastError () == ERROR_USER_MAPPED_FILE))) break; } @@ -1372,6 +1402,28 @@ namespace butl throw_generic_ios_failure (errno); } + entry_stat + fdstat (int fd) + { + struct stat s; + if (fstat (fd, &s) != 0) + throw_generic_error (errno); + + auto m (s.st_mode); + entry_type t (entry_type::unknown); + + // Note: cannot be a symlink. + // + if (S_ISREG (m)) + t = entry_type::regular; + else if (S_ISDIR (m)) + t = entry_type::directory; + else if (S_ISBLK (m) || S_ISCHR (m) || S_ISFIFO (m) || S_ISSOCK (m)) + t = entry_type::other; + + return entry_stat {t, static_cast<uint64_t> (s.st_size)}; + } + bool fdterm (int fd) { @@ -1392,6 +1444,16 @@ namespace butl throw_generic_ios_failure (errno); } + bool + fdterm_color (int, bool) + { + const char* t (std::getenv ("TERM")); + + // This test was lifted from GCC (Emacs shell sets TERM=dumb). + // + return t != nullptr && strcmp (t, "dumb") != 0; + } + static pair<size_t, size_t> fdselect (fdselect_set& read, fdselect_set& write, @@ -1410,6 +1472,8 @@ namespace butl for (fdselect_state& s: from) { + s.ready = false; + if (s.fd == nullfd) continue; @@ -1417,7 +1481,6 @@ namespace butl throw invalid_argument ("invalid file descriptor"); FD_SET (s.fd, &to); - s.ready = false; if (max_fd < s.fd) max_fd = s.fd; @@ -1524,6 +1587,12 @@ namespace butl return read (fd, buf, n); } + streamsize + fdwrite (int fd, const void* buf, size_t n) + { + return write (fd, buf, n); + } + #else auto_fd @@ -1779,9 +1848,34 @@ namespace butl throw_generic_ios_failure (e); } + entry_stat + fdstat (int fd) + { + // Since symlinks have been taken care of, we can just _fstat(). + // + struct __stat64 s; + if (_fstat64 (fd, &s) != 0) + throw_generic_error (errno); + + auto m (s.st_mode); + entry_type t (entry_type::unknown); + + if (S_ISREG (m)) + t = entry_type::regular; + else if (S_ISDIR (m)) + t = entry_type::directory; + else if (S_ISCHR (m)) + t = entry_type::other; + + return entry_stat {t, static_cast<uint64_t> (s.st_size)}; + } + bool fdterm (int fd) { + // @@ Both GCC and Clang simply call GetConsoleMode() for this check. I + // wonder why we don't do the same? See also fdterm_color() below. + // We don't need to close it (see fd_to_handle()). // HANDLE h (fd_to_handle (fd)); @@ -1867,6 +1961,42 @@ namespace butl return false; } + bool + fdterm_color (int fd, bool enable) + { + // We don't need to close it (see fd_to_handle()). + // + HANDLE h (fd_to_handle (fd)); + + // See GH issue #312 for background on this logic. + // + DWORD m; + if (!GetConsoleMode (h, &m)) + throw_system_ios_failure (GetLastError ()); + + // Some terminals (e.g. Windows Terminal) enable VT processing by default. + // + if ((m & ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0) + return true; + + if (enable) + { + // If SetConsoleMode() fails, assume VT processing is unsupported (it + // is only supported from a certain build of Windows 10). + // + // Note that Wine pretends to support this but doesn't handle the escape + // sequences. See https://bugs.winehq.org/show_bug.cgi?id=49780. + // + if (SetConsoleMode (h, + (m | + ENABLE_PROCESSED_OUTPUT | + ENABLE_VIRTUAL_TERMINAL_PROCESSING))) + return true; + } + + return false; + } + static pair<size_t, size_t> fdselect (fdselect_set& read, fdselect_set& write, @@ -1883,13 +2013,14 @@ namespace butl for (fdselect_state& s: read) { + s.ready = false; + if (s.fd == nullfd) continue; if (s.fd < 0) throw invalid_argument ("invalid file descriptor"); - s.ready = false; ++n; } @@ -1910,7 +2041,7 @@ namespace butl // size_t r (0); - while (true) + for (size_t i (0);; ++i) { for (fdselect_state& s: read) { @@ -1983,7 +2114,11 @@ namespace butl if (r != 0) break; - DWORD t (50); + // Use exponential backoff but not too aggressive and with 25ms max. + // + DWORD t ( + static_cast<DWORD> (i <= 1000 ? 0 : + i >= 1000 + 100 ? 25 : 1 + ((i - 1000) / 4))); if (timeout) { @@ -2000,7 +2135,10 @@ namespace butl break; } - Sleep (t); + if (t == 0) + this_thread::yield (); + else + Sleep (t); } return make_pair (r, 0); @@ -2043,6 +2181,12 @@ namespace butl return r; } + streamsize + fdwrite (int fd, const void* buf, size_t n) + { + return _write (fd, buf, static_cast<unsigned int> (n)); + } + #endif pair<size_t, size_t> |