aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2016-09-14 22:17:29 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2016-09-16 12:56:58 +0300
commit60dfe51f11d247f8490f54b4441a4fda14d6c66a (patch)
tree7392216579201e3be49a0498ade221c981fcf59b
parent985f1dde6afa6a42149118e6085325eacb1bd53e (diff)
Optimize fdstream so performance is on par with fstream
-rw-r--r--butl/fdstream5
-rw-r--r--butl/fdstream.cxx126
-rw-r--r--tests/fdstream/driver.cxx137
3 files changed, 263 insertions, 5 deletions
diff --git a/butl/fdstream b/butl/fdstream
index bb5364b..49b3d4d 100644
--- a/butl/fdstream
+++ b/butl/fdstream
@@ -86,13 +86,16 @@ namespace butl
virtual int
sync ();
+ virtual std::streamsize
+ xsputn (const char_type*, std::streamsize);
+
private:
bool
save ();
private:
int fd_ = -1;
- char buf_[4096];
+ char buf_[8192];
};
// File stream mode.
diff --git a/butl/fdstream.cxx b/butl/fdstream.cxx
index cff9b59..ac5aa79 100644
--- a/butl/fdstream.cxx
+++ b/butl/fdstream.cxx
@@ -6,7 +6,8 @@
#ifndef _WIN32
# include <fcntl.h> // open(), O_*
-# include <unistd.h> // close(), read(), write(), lseek()
+# include <unistd.h> // close(), read(), write(), lseek(), ssize_t
+# include <sys/uio.h> // writev(), iovec
# include <sys/stat.h> // S_I*
# include <sys/types.h> // off_t
#else
@@ -24,6 +25,7 @@
#include <new> // bad_alloc
#include <limits> // numeric_limits
#include <cassert>
+#include <cstring> // memcpy(), memmove()
#include <exception> // uncaught_exception()
#include <stdexcept> // invalid_argument
#include <type_traits>
@@ -178,6 +180,124 @@ namespace butl
return true;
}
+ streamsize fdbuf::
+ xsputn (const char_type* s, streamsize sn)
+ {
+ // To avoid futher 'signed/unsigned comparison' compiler warnings.
+ //
+ size_t n (static_cast<size_t> (sn));
+
+ // Buffer the data if there is enough space.
+ //
+ size_t an (epptr () - pptr ()); // Amount of free space in the buffer.
+ if (n <= an)
+ {
+ memcpy (pptr (), s, n);
+ pbump (n);
+ return n;
+ }
+
+ size_t bn (pptr () - pbase ()); // Buffered data size.
+
+#ifndef _WIN32
+
+ ssize_t r;
+ if (bn > 0)
+ {
+ // Write both buffered and new data with a single system call.
+ //
+ iovec iov[2] = {{pbase (), bn}, {const_cast<char*> (s), n}};
+ r = writev (fd_, iov, 2);
+ }
+ else
+ r = write (fd_, s, n);
+
+ if (r == -1)
+ throw_ios_failure (errno);
+
+ size_t m (static_cast<size_t> (r));
+
+ // If the buffered data wasn't fully written then move the unwritten part
+ // to the beginning of the buffer.
+ //
+ if (m < bn)
+ {
+ memmove (pbase (), pbase () + m, bn - m);
+ pbump (-m); // Note that pbump() accepts negatives.
+ return 0;
+ }
+
+ setp (buf_, buf_ + sizeof (buf_) - 1);
+ return m - bn;
+
+#else
+
+ // On Windows there is no writev() available so sometimes we will make two
+ // system calls. Fill and flush the buffer, then try to fit the data tail
+ // into the empty buffer. If the data tail is too long then just write it
+ // to the file and keep the buffer empty.
+ //
+ // We will end up with two _write() calls if the total data size to be
+ // written exceeds double the buffer size. In this case the buffer filling
+ // is redundant so let's pretend there is no free space in the buffer, and
+ // so buffered and new data will be written separatelly.
+ //
+ if (bn + n > 2 * (bn + an))
+ an = 0;
+ else
+ {
+ memcpy (pptr (), s, an);
+ pbump (an);
+ }
+
+ // Flush the buffer.
+ //
+ size_t wn (bn + an);
+ int r (wn > 0 ? _write (fd_, buf_, wn) : 0);
+
+ if (r == -1)
+ throw_ios_failure (errno);
+
+ size_t m (static_cast<size_t> (r));
+
+ // If the buffered data wasn't fully written then move the unwritten part
+ // to the beginning of the buffer.
+ //
+ if (m < wn)
+ {
+ memmove (pbase (), pbase () + m, wn - m);
+ pbump (-m); // Note that pbump() accepts negatives.
+ return m < bn ? 0 : m - bn;
+ }
+
+ setp (buf_, buf_ + sizeof (buf_) - 1);
+
+ // Now 'an' holds the size of the data portion written as a part of the
+ // buffer flush.
+ //
+ s += an;
+ n -= an;
+
+ // Buffer the data tail if it fits the buffer.
+ //
+ if (n <= static_cast<size_t> (epptr () - pbase ()))
+ {
+ memcpy (pbase (), s, n);
+ pbump (n);
+ return sn;
+ }
+
+ // The data tail doesn't fit the buffer so write it to the file.
+ //
+ r = _write (fd_, s, n);
+
+ if (r == -1)
+ throw_ios_failure (errno);
+
+ return an + r;
+#endif
+ }
+
// fdstream_base
//
fdstream_base::
@@ -477,9 +597,9 @@ namespace butl
if (mode (fdopen_mode::at_end))
{
#ifndef _WIN32
- bool r (lseek(fd, 0, SEEK_END) != static_cast<off_t>(-1));
+ bool r (lseek (fd, 0, SEEK_END) != static_cast<off_t> (-1));
#else
- bool r (_lseek(fd, 0, SEEK_END) != -1);
+ bool r (_lseek (fd, 0, SEEK_END) != -1);
#endif
// Note that in the case of an error we don't delete the newly created
diff --git a/tests/fdstream/driver.cxx b/tests/fdstream/driver.cxx
index 03eab80..2e3bac5 100644
--- a/tests/fdstream/driver.cxx
+++ b/tests/fdstream/driver.cxx
@@ -4,12 +4,17 @@
#include <ios>
#include <string>
+#include <vector>
+#include <iomanip>
#include <cassert>
#include <sstream>
+#include <fstream>
+#include <iostream>
#include <exception>
#include <butl/path>
#include <butl/fdstream>
+#include <butl/timestamp>
#include <butl/filesystem>
using namespace std;
@@ -38,9 +43,60 @@ to_file (const path& f, const string& s, fdopen_mode m = fdopen_mode::none)
ofs.close ();
}
+template <typename S, typename T>
+static duration
+write_time (const path& p, const T& s, size_t n)
+{
+ timestamp t (timestamp::clock::now ());
+ S os (p.string (), ofstream::out);
+ os.exceptions (S::failbit | S::badbit);
+
+ for (size_t i (0); i < n; ++i)
+ {
+ if (i > 0)
+ os << '\n'; // Important not to use endl as it syncs a stream.
+
+ os << s;
+ }
+
+ os.close ();
+ return timestamp::clock::now () - t;
+}
+
+template <typename S, typename T>
+static duration
+read_time (const path& p, const T& s, size_t n)
+{
+ vector<T> v (n);
+
+ timestamp t (timestamp::clock::now ());
+ S is (p.string (), ofstream::in);
+ is.exceptions (S::failbit | S::badbit);
+
+ for (auto& ve: v)
+ is >> ve;
+
+ assert (is.eof ());
+
+ is.close ();
+ duration d (timestamp::clock::now () - t);
+
+ for (const auto& ve: v)
+ assert (ve == s);
+
+ return d;
+}
+
int
-main ()
+main (int argc, const char* argv[])
{
+ if (!(argc == 1 || (argc == 2 && argv[1] == string ("-v"))))
+ {
+ cerr << "usage: " << argv[0] << " [-v]" << endl;
+ return 1;
+ }
+
+ bool v (argc == 2);
dir_path td (dir_path::temp_directory () / dir_path ("butl-fdstream"));
// Recreate the temporary directory (that possibly exists from the previous
@@ -307,5 +363,84 @@ main ()
#endif
+ // Compare fdstream and fstream operations performance.
+ //
+ duration fwd (0);
+ duration dwd (0);
+ duration frd (0);
+ duration drd (0);
+
+ path ff (td / path ("fstream"));
+ path fd (td / path ("fdstream"));
+
+ // Make several measurements with different ordering for each benchmark to
+ // level fluctuations.
+ //
+ // Write/read ~10M-size files by 100, 1000, 10 000, 100 000 byte-length
+ // strings.
+ //
+ size_t sz (100);
+ for (size_t i (0); i < 4; ++i)
+ {
+ string s;
+ s.reserve (sz);
+
+ // Fill string with characters from '0' to 'z'.
+ //
+ for (size_t i (0); i < sz; ++i)
+ s.push_back ('0' + i % (123 - 48));
+
+ size_t n (10 * 1024 * 1024 / sz);
+
+ for (size_t i (0); i < 4; ++i)
+ {
+ if (i % 2 == 0)
+ {
+ fwd += write_time<ofstream> (ff, s, n);
+ dwd += write_time<ofdstream> (fd, s, n);
+ frd += read_time<ifstream> (ff, s, n);
+ drd += read_time<ifdstream> (fd, s, n);
+ }
+ else
+ {
+ dwd += write_time<ofdstream> (fd, s, n);
+ fwd += write_time<ofstream> (ff, s, n);
+ drd += read_time<ifdstream> (fd, s, n);
+ frd += read_time<ifstream> (ff, s, n);
+ }
+ }
+
+ sz *= 10;
+ }
+
+ // Write/read ~10M-size files by 64-bit integers.
+ //
+ uint64_t u (0x1234567890123456);
+ size_t n (10 * 1024 * 1024 / sizeof (u));
+
+ for (size_t i (0); i < 4; ++i)
+ {
+ if (i % 2 == 0)
+ {
+ fwd += write_time<ofstream> (ff, u, n);
+ dwd += write_time<ofdstream> (fd, u, n);
+ frd += read_time<ifstream> (ff, u, n);
+ drd += read_time<ifdstream> (fd, u, n);
+ }
+ else
+ {
+ dwd += write_time<ofdstream> (fd, u, n);
+ fwd += write_time<ofstream> (ff, u, n);
+ drd += read_time<ifdstream> (fd, u, n);
+ frd += read_time<ifstream> (ff, u, n);
+ }
+ }
+
+ if (v)
+ cerr << "fdstream/fstream write and read duration ratios are "
+ << fixed << setprecision (2)
+ << static_cast<double> (dwd.count ()) / fwd.count () << " and "
+ << static_cast<double> (drd.count ()) / frd.count () << endl;
+
rmdir_r (td);
}