From 6b9bdad3b68b12ff8e2075d54c1f7f005bb2f768 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Thu, 18 Mar 2021 15:02:39 +0200 Subject: Add noop mode to file cache, add --file-cache option to select --- build2/b-options.cxx | 20 +++++++++++++++++++ build2/b-options.hxx | 8 ++++++++ build2/b-options.ixx | 12 ++++++++++++ build2/b.cli | 10 ++++++++++ build2/b.cxx | 13 ++++++++++++- libbuild2/build/script/parser.test.cxx | 2 +- libbuild2/file-cache.cxx | 16 +++++++++++----- libbuild2/file-cache.hxx | 35 +++++++++++++++++++--------------- libbuild2/file-cache.ixx | 19 +++++++++++------- libbuild2/function.test.cxx | 2 +- libbuild2/test/script/parser.test.cxx | 2 +- tests/libbuild2/driver.cxx | 2 +- 12 files changed, 109 insertions(+), 32 deletions(-) diff --git a/build2/b-options.cxx b/build2/b-options.cxx index c061fef..17b791e 100644 --- a/build2/b-options.cxx +++ b/build2/b-options.cxx @@ -709,6 +709,8 @@ namespace build2 max_jobs_specified_ (false), queue_depth_ (4), queue_depth_specified_ (false), + file_cache_ (), + file_cache_specified_ (false), max_stack_ (), max_stack_specified_ (false), serial_stop_ (), @@ -892,6 +894,13 @@ namespace build2 this->queue_depth_specified_ = true; } + if (a.file_cache_specified_) + { + ::build2::cl::parser< string>::merge ( + this->file_cache_, a.file_cache_); + this->file_cache_specified_ = true; + } + if (a.max_stack_specified_) { ::build2::cl::parser< size_t>::merge ( @@ -1105,6 +1114,14 @@ namespace build2 << " the build system scheduler implementation for details." << ::std::endl; os << std::endl + << "\033[1m--file-cache\033[0m \033[4mimpl\033[0m File cache implementation to use for intermediate build" << ::std::endl + << " results. Valid values are \033[1mnoop\033[0m (no caching or" << ::std::endl + << " compression) and \033[1msync-lz4\033[0m (no caching with synchronous" << ::std::endl + << " LZ4 on-disk compression). If this option is not" << ::std::endl + << " specified, then a suitable default implementation is used" << ::std::endl + << " (currently \033[1msync-lz4\033[0m)." << ::std::endl; + + os << std::endl << "\033[1m--max-stack\033[0m \033[4mnum\033[0m The maximum stack size in KBytes to allow for newly" << ::std::endl << " created threads. For \033[4mpthreads\033[0m-based systems the driver" << ::std::endl << " queries the stack size of the main thread and uses the" << ::std::endl @@ -1311,6 +1328,9 @@ namespace build2 _cli_options_map_["-Q"] = &::build2::cl::thunk< options, size_t, &options::queue_depth_, &options::queue_depth_specified_ >; + _cli_options_map_["--file-cache"] = + &::build2::cl::thunk< options, string, &options::file_cache_, + &options::file_cache_specified_ >; _cli_options_map_["--max-stack"] = &::build2::cl::thunk< options, size_t, &options::max_stack_, &options::max_stack_specified_ >; diff --git a/build2/b-options.hxx b/build2/b-options.hxx index fd93aba..4e1b7bd 100644 --- a/build2/b-options.hxx +++ b/build2/b-options.hxx @@ -510,6 +510,12 @@ namespace build2 bool queue_depth_specified () const; + const string& + file_cache () const; + + bool + file_cache_specified () const; + const size_t& max_stack () const; @@ -632,6 +638,8 @@ namespace build2 bool max_jobs_specified_; size_t queue_depth_; bool queue_depth_specified_; + string file_cache_; + bool file_cache_specified_; size_t max_stack_; bool max_stack_specified_; bool serial_stop_; diff --git a/build2/b-options.ixx b/build2/b-options.ixx index 3d5781c..0e90ba1 100644 --- a/build2/b-options.ixx +++ b/build2/b-options.ixx @@ -374,6 +374,18 @@ namespace build2 return this->queue_depth_specified_; } + inline const string& options:: + file_cache () const + { + return this->file_cache_; + } + + inline bool options:: + file_cache_specified () const + { + return this->file_cache_specified_; + } + inline const size_t& options:: max_stack () const { diff --git a/build2/b.cli b/build2/b.cli index b54e9a5..a24837c 100644 --- a/build2/b.cli +++ b/build2/b.cli @@ -494,6 +494,16 @@ namespace build2 details." } + string --file-cache + { + "", + "File cache implementation to use for intermediate build results. Valid + values are \cb{noop} (no caching or compression) and \cb{sync-lz4} (no + caching with synchronous LZ4 on-disk compression). If this option is + not specified, then a suitable default implementation is used + (currently \cb{sync-lz4})." + } + size_t --max-stack { "", diff --git a/build2/b.cxx b/build2/b.cxx index 81520bb..1ba15e2 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -762,7 +762,18 @@ main (int argc, char* argv[]) : nullopt)); global_mutexes mutexes (sched.shard_size ()); - file_cache fcache (sched); + + bool fcache_comp (true); + if (ops.file_cache_specified ()) + { + const string& v (ops.file_cache ()); + if (v == "noop" || v == "none") + fcache_comp = false; + else if (v != "sync-lz4") + fail << "invalid --file-cache value '" << v << "'"; + } + + file_cache fcache (fcache_comp); // Trace some overall environment information. // diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index a277102..29711ef 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -180,7 +180,7 @@ namespace build2 // scheduler sched (1); global_mutexes mutexes (1); - file_cache fcache (sched); + file_cache fcache; context ctx (sched, mutexes, fcache); try diff --git a/libbuild2/file-cache.cxx b/libbuild2/file-cache.cxx index 107bf3f..0c2fcbb 100644 --- a/libbuild2/file-cache.cxx +++ b/libbuild2/file-cache.cxx @@ -26,7 +26,8 @@ namespace build2 // to compressing the new file (for example, if we fail and leave the // uncompressed file behind for troubleshooting). // - try_rmfile_ignore_error (comp_path_); + if (!comp_path_.empty ()) + try_rmfile_ignore_error (comp_path_); pin (); return write (*this); @@ -37,6 +38,8 @@ namespace build2 { assert (state_ == uninit); + bool c (!comp_path_.empty ()); + // Determine the cache state from the filesystem state. // // First check for the uncompressed file. Its presence means that the @@ -45,15 +48,18 @@ namespace build2 // if (exists (path_)) { - try_rmfile_ignore_error (comp_path_); + if (c) + try_rmfile_ignore_error (comp_path_); + state_ = uncomp; } - else if (exists (comp_path_)) + else if (c && exists (comp_path_)) { state_ = comp; } else - fail << path_ << " (or its compressed variant) does not exist" << + fail << path_ << (c ? " (or its compressed variant)" : "") + << " does not exist" << info << "consider cleaning the build state"; } @@ -158,7 +164,7 @@ namespace build2 // file, then we don't attempt to remove the uncompressed file either // since it could be an indicator that the compressed file is invalid. // - if (try_rmfile_ignore_error (comp_path_)) + if (comp_path_.empty () || try_rmfile_ignore_error (comp_path_)) try_rmfile_ignore_error (path_); break; } diff --git a/libbuild2/file-cache.hxx b/libbuild2/file-cache.hxx index 1502fb8..d6904ed 100644 --- a/libbuild2/file-cache.hxx +++ b/libbuild2/file-cache.hxx @@ -74,22 +74,25 @@ namespace build2 // one that simply saves the file on disk) is file_cache::entry that is just // auto_rmfile. - // The synchronous compressed file cache implementation. + // The synchronous LZ4 on-disk compression file cache implementation. // // If the cache entry is no longer pinned, this implementation compresses - // the content and removes the uncompressed file all as part of the call that - // caused the entry to become unpinned. + // the content and removes the uncompressed file all as part of the call + // that caused the entry to become unpinned. // // In order to deal with interruptions during compression, when recreating // the cache entry state from the filesystem state, this implementation // treats the presence of the uncompressed file as an indication that the // compressed file, if any, is invalid. // - class scheduler; - class file_cache { public: + // If compression is disabled, then this implementation becomes equivalent + // to the noop implementation. + // + explicit + file_cache (bool compress = true); class entry; @@ -108,7 +111,6 @@ namespace build2 close (); write (): entry_ (nullptr) {} - ~write (); // Move-to-NULL-only type. // @@ -117,6 +119,8 @@ namespace build2 write& operator= (write&&); write& operator= (const write&) = delete; + ~write (); + private: friend class entry; @@ -133,7 +137,6 @@ namespace build2 { public: read (): entry_ (nullptr) {} - ~read (); // Move-to-NULL-only type. // @@ -142,6 +145,8 @@ namespace build2 read& operator= (read&&); read& operator= (const read&) = delete; + ~read (); + private: friend class entry; @@ -203,11 +208,13 @@ namespace build2 entry& operator= (entry&&); entry& operator= (const entry&) = delete; - // Implementation details. - // - entry (path_type, bool); ~entry (); + private: + friend class file_cache; + + entry (path_type, bool, bool); + void preempt (); @@ -224,7 +231,7 @@ namespace build2 state state_ = null; path_type path_; // Uncompressed path. - path_type comp_path_; // Compressed path. + path_type comp_path_; // Compressed path (empty if disabled). size_t pin_ = 0; // Pin count. }; @@ -254,10 +261,8 @@ namespace build2 string compressed_extension (const char* ext = nullptr); - // Implementation details. - // - explicit - file_cache (scheduler&); + private: + bool compress_; }; } diff --git a/libbuild2/file-cache.ixx b/libbuild2/file-cache.ixx index 2b76fb6..1a01410 100644 --- a/libbuild2/file-cache.ixx +++ b/libbuild2/file-cache.ixx @@ -95,7 +95,9 @@ namespace build2 inline void file_cache::entry:: unpin () { - if (--pin_ == 0 && (state_ == uncomp || state_ == decomp)) + if (--pin_ == 0 && + !comp_path_.empty () && + (state_ == uncomp || state_ == decomp)) preempt (); } @@ -106,11 +108,11 @@ namespace build2 } inline file_cache::entry:: - entry (path_type p, bool t) + entry (path_type p, bool t, bool c) : temporary (t), state_ (uninit), path_ (move (p)), - comp_path_ (path_ + ".lz4"), + comp_path_ (c ? path_ + ".lz4" : path_type ()), pin_ (1) { } @@ -152,13 +154,13 @@ namespace build2 inline file_cache::entry file_cache:: create (path f, optional) { - return entry (move (f), true /* temporary */); + return entry (move (f), true /* temporary */, compress_); } inline file_cache::entry file_cache:: create_existing (path f) { - entry e (move (f), false /* temporary */); + entry e (move (f), false /* temporary */, compress_); e.init_existing (); return e; } @@ -166,11 +168,14 @@ namespace build2 inline string file_cache:: compressed_extension (const char* e) { - return (e != nullptr ? string (e) : string ()) + ".lz4"; + return compress_ + ? (e != nullptr ? string (e) : string ()) + ".lz4" + : string (); } inline file_cache:: - file_cache (scheduler&) + file_cache (bool compress) + : compress_ (compress) { } } diff --git a/libbuild2/function.test.cxx b/libbuild2/function.test.cxx index 7aa1a50..c64bd0a 100644 --- a/libbuild2/function.test.cxx +++ b/libbuild2/function.test.cxx @@ -47,7 +47,7 @@ namespace build2 // scheduler sched (1); global_mutexes mutexes (1); - file_cache fcache (sched); + file_cache fcache; context ctx (sched, mutexes, fcache); auto& functions (ctx.functions); diff --git a/libbuild2/test/script/parser.test.cxx b/libbuild2/test/script/parser.test.cxx index df91586..202f368 100644 --- a/libbuild2/test/script/parser.test.cxx +++ b/libbuild2/test/script/parser.test.cxx @@ -166,7 +166,7 @@ namespace build2 // scheduler sched (1); global_mutexes mutexes (1); - file_cache fcache (sched); + file_cache fcache; context ctx (sched, mutexes, fcache); bool scope (false); diff --git a/tests/libbuild2/driver.cxx b/tests/libbuild2/driver.cxx index 5d18623..1c4554a 100644 --- a/tests/libbuild2/driver.cxx +++ b/tests/libbuild2/driver.cxx @@ -38,7 +38,7 @@ main (int, char* argv[]) // scheduler sched (1); global_mutexes mutexes (1); - file_cache fcache (sched); + file_cache fcache; context ctx (sched, mutexes, fcache); return 0; -- cgit v1.1