Add noop mode to file cache, add --file-cache option to select

12 files changed, 109 insertions, 32 deletions

diff --git a/build2/b-options.cxx b/build2/b-options.cxx
index c061fef..17b791e 100644
--- a/build2/b-options.cxx
+++ b/build2/b-options.cxx
@@ -709,6 +709,8 @@ namespace build2
     max_jobs_specified_ (false),
     queue_depth_ (4),
     queue_depth_specified_ (false),
+    file_cache_ (),
+    file_cache_specified_ (false),
     max_stack_ (),
     max_stack_specified_ (false),
     serial_stop_ (),
@@ -892,6 +894,13 @@ namespace build2
       this->queue_depth_specified_ = true;
     }
 
+    if (a.file_cache_specified_)
+    {
+      ::build2::cl::parser< string>::merge (
+        this->file_cache_, a.file_cache_);
+      this->file_cache_specified_ = true;
+    }
+
     if (a.max_stack_specified_)
     {
       ::build2::cl::parser< size_t>::merge (
@@ -1105,6 +1114,14 @@ namespace build2
        << "                      the build system scheduler implementation for details." << ::std::endl;
 
     os << std::endl
+       << "\033[1m--file-cache\033[0m \033[4mimpl\033[0m     File cache implementation to use for intermediate build" << ::std::endl
+       << "                      results. Valid values are \033[1mnoop\033[0m (no caching or" << ::std::endl
+       << "                      compression) and \033[1msync-lz4\033[0m (no caching with synchronous" << ::std::endl
+       << "                      LZ4 on-disk compression). If this option is not" << ::std::endl
+       << "                      specified, then a suitable default implementation is used" << ::std::endl
+       << "                      (currently \033[1msync-lz4\033[0m)." << ::std::endl;
+
+    os << std::endl
        << "\033[1m--max-stack\033[0m \033[4mnum\033[0m       The maximum stack size in KBytes to allow for newly" << ::std::endl
        << "                      created threads. For \033[4mpthreads\033[0m-based systems the driver" << ::std::endl
        << "                      queries the stack size of the main thread and uses the" << ::std::endl
@@ -1311,6 +1328,9 @@ namespace build2
       _cli_options_map_["-Q"] =
       &::build2::cl::thunk< options, size_t, &options::queue_depth_,
         &options::queue_depth_specified_ >;
+      _cli_options_map_["--file-cache"] =
+      &::build2::cl::thunk< options, string, &options::file_cache_,
+        &options::file_cache_specified_ >;
       _cli_options_map_["--max-stack"] =
       &::build2::cl::thunk< options, size_t, &options::max_stack_,
         &options::max_stack_specified_ >;
diff --git a/build2/b-options.hxx b/build2/b-options.hxx
index fd93aba..4e1b7bd 100644
--- a/build2/b-options.hxx
+++ b/build2/b-options.hxx
@@ -510,6 +510,12 @@ namespace build2
     bool
     queue_depth_specified () const;
 
+    const string&
+    file_cache () const;
+
+    bool
+    file_cache_specified () const;
+
     const size_t&
     max_stack () const;
 
@@ -632,6 +638,8 @@ namespace build2
     bool max_jobs_specified_;
     size_t queue_depth_;
     bool queue_depth_specified_;
+    string file_cache_;
+    bool file_cache_specified_;
     size_t max_stack_;
     bool max_stack_specified_;
     bool serial_stop_;
diff --git a/build2/b-options.ixx b/build2/b-options.ixx
index 3d5781c..0e90ba1 100644
--- a/build2/b-options.ixx
+++ b/build2/b-options.ixx
@@ -374,6 +374,18 @@ namespace build2
     return this->queue_depth_specified_;
   }
 
+  inline const string& options::
+  file_cache () const
+  {
+    return this->file_cache_;
+  }
+
+  inline bool options::
+  file_cache_specified () const
+  {
+    return this->file_cache_specified_;
+  }
+
   inline const size_t& options::
   max_stack () const
   {
diff --git a/build2/b.cli b/build2/b.cli
index b54e9a5..a24837c 100644
--- a/build2/b.cli
+++ b/build2/b.cli
@@ -494,6 +494,16 @@ namespace build2
        details."
     }
 
+    string --file-cache
+    {
+      "<impl>",
+      "File cache implementation to use for intermediate build results. Valid
+       values are \cb{noop} (no caching or compression) and \cb{sync-lz4} (no
+       caching with synchronous LZ4 on-disk compression). If this option is
+       not specified, then a suitable default implementation is used
+       (currently \cb{sync-lz4})."
+    }
+
     size_t --max-stack
     {
       "<num>",
diff --git a/build2/b.cxx b/build2/b.cxx
index 81520bb..1ba15e2 100644
--- a/build2/b.cxx
+++ b/build2/b.cxx
@@ -762,7 +762,18 @@ main (int argc, char* argv[])
                     : nullopt));
 
     global_mutexes mutexes (sched.shard_size ());
-    file_cache fcache (sched);
+
+    bool fcache_comp (true);
+    if (ops.file_cache_specified ())
+    {
+      const string& v (ops.file_cache ());
+      if (v == "noop" || v == "none")
+        fcache_comp = false;
+      else if (v != "sync-lz4")
+        fail << "invalid --file-cache value '" << v << "'";
+    }
+
+    file_cache fcache (fcache_comp);
 
     // Trace some overall environment information.
     //
diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx
index a277102..29711ef 100644
--- a/libbuild2/build/script/parser.test.cxx
+++ b/libbuild2/build/script/parser.test.cxx
@@ -180,7 +180,7 @@ namespace build2
         //
         scheduler sched (1);
         global_mutexes mutexes (1);
-        file_cache fcache (sched);
+        file_cache fcache;
         context ctx (sched, mutexes, fcache);
 
         try
diff --git a/libbuild2/file-cache.cxx b/libbuild2/file-cache.cxx
index 107bf3f..0c2fcbb 100644
--- a/libbuild2/file-cache.cxx
+++ b/libbuild2/file-cache.cxx
@@ -26,7 +26,8 @@ namespace build2
     // to compressing the new file (for example, if we fail and leave the
     // uncompressed file behind for troubleshooting).
     //
-    try_rmfile_ignore_error (comp_path_);
+    if (!comp_path_.empty ())
+      try_rmfile_ignore_error (comp_path_);
 
     pin ();
     return write (*this);
@@ -37,6 +38,8 @@ namespace build2
   {
     assert (state_ == uninit);
 
+    bool c (!comp_path_.empty ());
+
     // Determine the cache state from the filesystem state.
     //
     // First check for the uncompressed file. Its presence means that the
@@ -45,15 +48,18 @@ namespace build2
     //
     if (exists (path_))
     {
-      try_rmfile_ignore_error (comp_path_);
+      if (c)
+        try_rmfile_ignore_error (comp_path_);
+
       state_ = uncomp;
     }
-    else if (exists (comp_path_))
+    else if (c && exists (comp_path_))
     {
       state_ = comp;
     }
     else
-      fail << path_ << " (or its compressed variant) does not exist" <<
+      fail << path_ << (c ? " (or its compressed variant)" : "")
+           << " does not exist" <<
         info << "consider cleaning the build state";
   }
 
@@ -158,7 +164,7 @@ namespace build2
         // file, then we don't attempt to remove the uncompressed file either
         // since it could be an indicator that the compressed file is invalid.
         //
-        if (try_rmfile_ignore_error (comp_path_))
+        if (comp_path_.empty () || try_rmfile_ignore_error (comp_path_))
           try_rmfile_ignore_error (path_);
         break;
       }
diff --git a/libbuild2/file-cache.hxx b/libbuild2/file-cache.hxx
index 1502fb8..d6904ed 100644
--- a/libbuild2/file-cache.hxx
+++ b/libbuild2/file-cache.hxx
@@ -74,22 +74,25 @@ namespace build2
   // one that simply saves the file on disk) is file_cache::entry that is just
   // auto_rmfile.
 
-  // The synchronous compressed file cache implementation.
+  // The synchronous LZ4 on-disk compression file cache implementation.
   //
   // If the cache entry is no longer pinned, this implementation compresses
-  // the content and removes the uncompressed file all as part of the call that
-  // caused the entry to become unpinned.
+  // the content and removes the uncompressed file all as part of the call
+  // that caused the entry to become unpinned.
   //
   // In order to deal with interruptions during compression, when recreating
   // the cache entry state from the filesystem state, this implementation
   // treats the presence of the uncompressed file as an indication that the
   // compressed file, if any, is invalid.
   //
-  class scheduler;
-
   class file_cache
   {
   public:
+    // If compression is disabled, then this implementation becomes equivalent
+    // to the noop implementation.
+    //
+    explicit
+    file_cache (bool compress = true);
 
     class entry;
 
@@ -108,7 +111,6 @@ namespace build2
       close ();
 
       write (): entry_ (nullptr) {}
-      ~write ();
 
       // Move-to-NULL-only type.
       //
@@ -117,6 +119,8 @@ namespace build2
       write& operator= (write&&);
       write& operator= (const write&) = delete;
 
+      ~write ();
+
     private:
       friend class entry;
 
@@ -133,7 +137,6 @@ namespace build2
     {
     public:
       read (): entry_ (nullptr) {}
-      ~read ();
 
       // Move-to-NULL-only type.
       //
@@ -142,6 +145,8 @@ namespace build2
       read& operator= (read&&);
       read& operator= (const read&) = delete;
 
+      ~read ();
+
     private:
       friend class entry;
 
@@ -203,11 +208,13 @@ namespace build2
       entry& operator= (entry&&);
       entry& operator= (const entry&) = delete;
 
-      // Implementation details.
-      //
-      entry (path_type, bool);
       ~entry ();
 
+    private:
+      friend class file_cache;
+
+      entry (path_type, bool, bool);
+
       void
       preempt ();
 
@@ -224,7 +231,7 @@ namespace build2
 
       state     state_ = null;
       path_type path_;           // Uncompressed path.
-      path_type comp_path_;      // Compressed path.
+      path_type comp_path_;      // Compressed path (empty if disabled).
       size_t    pin_ = 0;        // Pin count.
     };
 
@@ -254,10 +261,8 @@ namespace build2
     string
     compressed_extension (const char* ext = nullptr);
 
-    // Implementation details.
-    //
-    explicit
-    file_cache (scheduler&);
+  private:
+    bool compress_;
   };
 }
 
diff --git a/libbuild2/file-cache.ixx b/libbuild2/file-cache.ixx
index 2b76fb6..1a01410 100644
--- a/libbuild2/file-cache.ixx
+++ b/libbuild2/file-cache.ixx
@@ -95,7 +95,9 @@ namespace build2
   inline void file_cache::entry::
   unpin ()
   {
-    if (--pin_ == 0 && (state_ == uncomp || state_ == decomp))
+    if (--pin_ == 0          &&
+        !comp_path_.empty () &&
+        (state_ == uncomp || state_ == decomp))
       preempt ();
   }
 
@@ -106,11 +108,11 @@ namespace build2
   }
 
   inline file_cache::entry::
-  entry (path_type p, bool t)
+  entry (path_type p, bool t, bool c)
       : temporary (t),
         state_ (uninit),
         path_ (move (p)),
-        comp_path_ (path_ + ".lz4"),
+        comp_path_ (c ? path_ + ".lz4" : path_type ()),
         pin_ (1)
   {
   }
@@ -152,13 +154,13 @@ namespace build2
   inline file_cache::entry file_cache::
   create (path f, optional<bool>)
   {
-    return entry (move (f), true /* temporary */);
+    return entry (move (f), true /* temporary */, compress_);
   }
 
   inline file_cache::entry file_cache::
   create_existing (path f)
   {
-    entry e (move (f), false /* temporary */);
+    entry e (move (f), false /* temporary */, compress_);
     e.init_existing ();
     return e;
   }
@@ -166,11 +168,14 @@ namespace build2
   inline string file_cache::
   compressed_extension (const char* e)
   {
-    return (e != nullptr ? string (e) : string ()) + ".lz4";
+    return compress_
+      ? (e != nullptr ? string (e) : string ()) + ".lz4"
+      : string ();
   }
 
   inline file_cache::
-  file_cache (scheduler&)
+  file_cache (bool compress)
+      : compress_ (compress)
   {
   }
 }
diff --git a/libbuild2/function.test.cxx b/libbuild2/function.test.cxx
index 7aa1a50..c64bd0a 100644
--- a/libbuild2/function.test.cxx
+++ b/libbuild2/function.test.cxx
@@ -47,7 +47,7 @@ namespace build2
     //
     scheduler sched (1);
     global_mutexes mutexes (1);
-    file_cache fcache (sched);
+    file_cache fcache;
     context ctx (sched, mutexes, fcache);
 
     auto& functions (ctx.functions);
diff --git a/libbuild2/test/script/parser.test.cxx b/libbuild2/test/script/parser.test.cxx
index df91586..202f368 100644
--- a/libbuild2/test/script/parser.test.cxx
+++ b/libbuild2/test/script/parser.test.cxx
@@ -166,7 +166,7 @@ namespace build2
         //
         scheduler sched (1);
         global_mutexes mutexes (1);
-        file_cache fcache (sched);
+        file_cache fcache;
         context ctx (sched, mutexes, fcache);
 
         bool scope (false);
diff --git a/tests/libbuild2/driver.cxx b/tests/libbuild2/driver.cxx
index 5d18623..1c4554a 100644
--- a/tests/libbuild2/driver.cxx
+++ b/tests/libbuild2/driver.cxx
@@ -38,7 +38,7 @@ main (int, char* argv[])
   //
   scheduler sched (1);
   global_mutexes mutexes (1);
-  file_cache fcache (sched);
+  file_cache fcache;
   context ctx (sched, mutexes, fcache);
 
   return 0;