aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bbot/agent/agent.cxx4
-rw-r--r--bbot/types.hxx13
-rw-r--r--bbot/utility.hxx8
-rw-r--r--bbot/utility.txx21
-rw-r--r--bbot/worker/worker.cxx16
5 files changed, 46 insertions, 16 deletions
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 31becd3..60f7271 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -836,8 +836,8 @@ try
try_mkdir_p (gd);
try_mkdir_p (pd);
- path tf (gd / "task.manifest"); // Task manifest file.
- path rf (pd / "result.manifest"); // Result manifest file.
+ path tf (gd / "task.manifest"); // Task manifest file.
+ path rf (pd / "result.manifest.lz4"); // Result manifest file.
serialize_manifest (tm, tf, "task");
diff --git a/bbot/types.hxx b/bbot/types.hxx
index 7e485f7..a39abe2 100644
--- a/bbot/types.hxx
+++ b/bbot/types.hxx
@@ -21,6 +21,8 @@
#include <libbutl/path.mxx>
#include <libbutl/optional.mxx>
+#include <libbutl/fdstream.mxx>
+#include <libbutl/lz4-stream.hxx>
#include <libbutl/vector-view.mxx>
#include <libbutl/small-vector.mxx>
#include <libbutl/standard-version.mxx>
@@ -69,6 +71,17 @@ namespace bbot
using butl::optional;
using butl::nullopt;
+ // <libbutl/fdstream.mxx>
+ //
+ using butl::auto_fd;
+ using butl::ifdstream;
+ using butl::ofdstream;
+
+ // <libbutl/lz4-stream.hxx>
+ //
+ using olz4stream = butl::lz4::ostream;
+ using ilz4stream = butl::lz4::istream;
+
// <libbutl/vector-view.mxx>
//
using butl::vector_view;
diff --git a/bbot/utility.hxx b/bbot/utility.hxx
index 4545bd3..b93f8b6 100644
--- a/bbot/utility.hxx
+++ b/bbot/utility.hxx
@@ -16,7 +16,6 @@
#include <libbutl/process.mxx>
#include <libbutl/process-io.mxx>
#include <libbutl/utility.mxx> // icasecmp(), reverse_iterate(), etc
-#include <libbutl/fdstream.mxx>
#include <libbutl/filesystem.mxx>
#include <bbot/types.hxx>
@@ -46,10 +45,6 @@ namespace bbot
using butl::setenv;
using butl::unsetenv;
- // <libbutl/fdstream.mxx>
- //
- using butl::auto_fd;
-
// <libbutl/filesystem.mxx>
//
using butl::auto_rmdir;
@@ -132,7 +127,8 @@ namespace bbot
// Manifest parsing and serialization.
//
- // For parsing, if path is '-', then read from stdin.
+ // For parsing, if path is '-', then read from stdin. If path has the .lz4
+ // extension, then assume the content is compressed.
//
template <typename T>
T
diff --git a/bbot/utility.txx b/bbot/utility.txx
index e485a80..3199a61 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -150,8 +150,25 @@ namespace bbot
if (!file_exists (f))
fail (fh) << what << " manifest file " << f << " does not exist";
- ifdstream ifs (f);
- return parse_manifest<T> (ifs, f.string (), what, fh, iu);
+ bool d (f.extension () == "lz4");
+
+ ifdstream ifs (f, (d
+ ? ifdstream::badbit
+ : ifdstream::badbit | ifdstream::failbit));
+ ilz4stream izs;
+
+ if (d)
+ izs.open (ifs, true /* end */);
+
+ return parse_manifest<T> (d
+ ? static_cast<istream&> (izs)
+ : static_cast<istream&> (ifs),
+ f.string (), what, fh, iu);
+ }
+ catch (const invalid_argument& e) // Invalid compressed content.
+ {
+ fail (fh) << "invalid " << what << " manifest " << f << ": " << e
+ << endf;
}
catch (const system_error& e) // EACCES, etc.
{
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 1f5c946..f9081a2 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -547,9 +547,9 @@ run_b (step_id step,
verbosity, buildspec, forward<A> (a)...);
}
-// Upload manifest to the specified TFTP URL with curl. Issue diagnostics and
-// throw failed on invalid manifest or process management errors and throw
-// io_error for input/output errors or non-zero curl exit.
+// Upload compressed manifest to the specified TFTP URL with curl. Issue
+// diagnostics and throw failed on invalid manifest or process management
+// errors and throw io_error for input/output errors or non-zero curl exit.
//
template <typename T>
static void
@@ -564,9 +564,11 @@ upload_manifest (tracer& trace,
// broken pipe error on the client and partial/truncated upload on the
// server. This happens quite regularly on older Linux distributions
// (e.g., Debian 8, Ubuntu 16.04) but also sometimes on Windows. On the
- // other hand, uploading from a file appears to work reliably.
+ // other hand, uploading from a file appears to work reliably (we still
+ // get an odd error on Windows from time to time with larger uploads).
//
#if 0
+ // Note: need to add compression support if re-enable this.
tftp_curl c (trace,
path ("-"),
nullfd,
@@ -582,10 +584,12 @@ upload_manifest (tracer& trace,
auto_rmfile tmp;
try
{
- tmp = auto_rmfile (path::temp_path (what + "-manifest"));
+ tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4"));
ofdstream ofs (tmp.path);
- manifest_serializer s (ofs, tmp.path.string ());
+ olz4stream ozs (ofs, 5 /* 256KB */, 9, nullopt /* content_size */);
+ manifest_serializer s (ozs, tmp.path.string ());
m.serialize (s);
+ ozs.close ();
ofs.close ();
}
catch (const io_error& e) // In case not derived from system_error.