diff options
| -rw-r--r-- | bbot/agent/agent.cxx | 4 | ||||
| -rw-r--r-- | bbot/types.hxx | 13 | ||||
| -rw-r--r-- | bbot/utility.hxx | 8 | ||||
| -rw-r--r-- | bbot/utility.txx | 21 | ||||
| -rw-r--r-- | bbot/worker/worker.cxx | 16 |
5 files changed, 46 insertions, 16 deletions
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx index 31becd3..60f7271 100644 --- a/bbot/agent/agent.cxx +++ b/bbot/agent/agent.cxx @@ -836,8 +836,8 @@ try try_mkdir_p (gd); try_mkdir_p (pd); - path tf (gd / "task.manifest"); // Task manifest file. - path rf (pd / "result.manifest"); // Result manifest file. + path tf (gd / "task.manifest"); // Task manifest file. + path rf (pd / "result.manifest.lz4"); // Result manifest file. serialize_manifest (tm, tf, "task"); diff --git a/bbot/types.hxx b/bbot/types.hxx index 7e485f7..a39abe2 100644 --- a/bbot/types.hxx +++ b/bbot/types.hxx @@ -21,6 +21,8 @@ #include <libbutl/path.mxx> #include <libbutl/optional.mxx> +#include <libbutl/fdstream.mxx> +#include <libbutl/lz4-stream.hxx> #include <libbutl/vector-view.mxx> #include <libbutl/small-vector.mxx> #include <libbutl/standard-version.mxx> @@ -69,6 +71,17 @@ namespace bbot using butl::optional; using butl::nullopt; + // <libbutl/fdstream.mxx> + // + using butl::auto_fd; + using butl::ifdstream; + using butl::ofdstream; + + // <libbutl/lz4-stream.hxx> + // + using olz4stream = butl::lz4::ostream; + using ilz4stream = butl::lz4::istream; + // <libbutl/vector-view.mxx> // using butl::vector_view; diff --git a/bbot/utility.hxx b/bbot/utility.hxx index 4545bd3..b93f8b6 100644 --- a/bbot/utility.hxx +++ b/bbot/utility.hxx @@ -16,7 +16,6 @@ #include <libbutl/process.mxx> #include <libbutl/process-io.mxx> #include <libbutl/utility.mxx> // icasecmp(), reverse_iterate(), etc -#include <libbutl/fdstream.mxx> #include <libbutl/filesystem.mxx> #include <bbot/types.hxx> @@ -46,10 +45,6 @@ namespace bbot using butl::setenv; using butl::unsetenv; - // <libbutl/fdstream.mxx> - // - using butl::auto_fd; - // <libbutl/filesystem.mxx> // using butl::auto_rmdir; @@ -132,7 +127,8 @@ namespace bbot // Manifest parsing and serialization. // - // For parsing, if path is '-', then read from stdin. + // For parsing, if path is '-', then read from stdin. If path has the .lz4 + // extension, then assume the content is compressed. // template <typename T> T diff --git a/bbot/utility.txx b/bbot/utility.txx index e485a80..3199a61 100644 --- a/bbot/utility.txx +++ b/bbot/utility.txx @@ -150,8 +150,25 @@ namespace bbot if (!file_exists (f)) fail (fh) << what << " manifest file " << f << " does not exist"; - ifdstream ifs (f); - return parse_manifest<T> (ifs, f.string (), what, fh, iu); + bool d (f.extension () == "lz4"); + + ifdstream ifs (f, (d + ? ifdstream::badbit + : ifdstream::badbit | ifdstream::failbit)); + ilz4stream izs; + + if (d) + izs.open (ifs, true /* end */); + + return parse_manifest<T> (d + ? static_cast<istream&> (izs) + : static_cast<istream&> (ifs), + f.string (), what, fh, iu); + } + catch (const invalid_argument& e) // Invalid compressed content. + { + fail (fh) << "invalid " << what << " manifest " << f << ": " << e + << endf; } catch (const system_error& e) // EACCES, etc. { diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 1f5c946..f9081a2 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -547,9 +547,9 @@ run_b (step_id step, verbosity, buildspec, forward<A> (a)...); } -// Upload manifest to the specified TFTP URL with curl. Issue diagnostics and -// throw failed on invalid manifest or process management errors and throw -// io_error for input/output errors or non-zero curl exit. +// Upload compressed manifest to the specified TFTP URL with curl. Issue +// diagnostics and throw failed on invalid manifest or process management +// errors and throw io_error for input/output errors or non-zero curl exit. // template <typename T> static void @@ -564,9 +564,11 @@ upload_manifest (tracer& trace, // broken pipe error on the client and partial/truncated upload on the // server. This happens quite regularly on older Linux distributions // (e.g., Debian 8, Ubuntu 16.04) but also sometimes on Windows. On the - // other hand, uploading from a file appears to work reliably. + // other hand, uploading from a file appears to work reliably (we still + // get an odd error on Windows from time to time with larger uploads). // #if 0 + // Note: need to add compression support if re-enable this. tftp_curl c (trace, path ("-"), nullfd, @@ -582,10 +584,12 @@ upload_manifest (tracer& trace, auto_rmfile tmp; try { - tmp = auto_rmfile (path::temp_path (what + "-manifest")); + tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4")); ofdstream ofs (tmp.path); - manifest_serializer s (ofs, tmp.path.string ()); + olz4stream ozs (ofs, 5 /* 256KB */, 9, nullopt /* content_size */); + manifest_serializer s (ozs, tmp.path.string ()); m.serialize (s); + ozs.close (); ofs.close (); } catch (const io_error& e) // In case not derived from system_error. |