Add support for checked out repository fragments caching

7 files changed, 265 insertions, 94 deletions

diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index 0c2ac21..8195375 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -2152,9 +2152,13 @@ namespace bpkg
 
   // Noop on POSIX.
   //
-  bool
-  git_fixup_worktree (const common_options&, const dir_path&, bool)
+  optional<bool>
+  git_fixup_worktree (const common_options&,
+                      const dir_path&,
+                      bool revert,
+                      bool)
   {
+    assert (!revert);
     return false;
   }
 
@@ -2545,15 +2549,26 @@ namespace bpkg
     return r;
   }
 
-  bool
+  optional<bool>
   git_fixup_worktree (const common_options& co,
                       const dir_path& dir,
-                      bool revert)
+                      bool revert,
+                      bool ie)
   {
-    optional<bool> r (
-      fixup_worktree (co, dir, revert, dir_path () /* prefix */));
+    try
+    {
+      optional<bool> r (
+        fixup_worktree (co, dir, revert, dir_path () /* prefix */));
 
-    return r ? *r : false;
+      return r ? *r : false;
+    }
+    catch (const failed&)
+    {
+      if (ie)
+        return nullopt;
+
+      throw;
+    }
   }
 
 #endif
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index d57dcf3..33e5d55 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -105,15 +105,19 @@ namespace bpkg
   // Fix up or revert the fixes (including in submodules, recursively) in a
   // working tree previously checked out by git_checkout() or
   // git_checkout_submodules(). Return true if any changes have been made to
-  // the filesystem.
+  // the filesystem. On error issue diagnostics and return nullopt in the
+  // ignore errors mode and throw failed otherwise.
   //
   // Noop on POSIX. On Windows it may replace git's filesystem-agnostic
   // symlinks with hardlinks for the file targets and junctions for the
   // directory targets. Note that it still makes sure the working tree is
   // being treated by git as "clean" despite the changes.
   //
-  bool
-  git_fixup_worktree (const common_options&, const dir_path&, bool revert);
+  optional<bool>
+  git_fixup_worktree (const common_options&,
+                      const dir_path&,
+                      bool revert,
+                      bool ignore_errors = false);
 
   // Low-level fetch API (fetch.cxx).
   //
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index f2af79a..8e50d05 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -5994,6 +5994,7 @@ namespace bpkg
 
     // purge, fetch/unpack|checkout
     //
+    pkg_checkout_cache checkout_cache (o);
     for (build_package& p: reverse_iterate (build_pkgs))
     {
       assert (p.action);
@@ -6125,22 +6126,24 @@ namespace bpkg
             case repository_basis::version_control:
               {
                 sp = p.checkout_root
-                     ? pkg_checkout (o,
-                                     pdb,
-                                     t,
-                                     ap->id.name,
-                                     p.available_version (),
-                                     *p.checkout_root,
-                                     true /* replace */,
-                                     p.checkout_purge,
-                                     simulate)
-                     : pkg_checkout (o,
-                                     pdb,
-                                     t,
-                                     ap->id.name,
-                                     p.available_version (),
-                                     true /* replace */,
-                                     simulate);
+                  ? pkg_checkout (checkout_cache,
+                                  o,
+                                  pdb,
+                                  t,
+                                  ap->id.name,
+                                  p.available_version (),
+                                  *p.checkout_root,
+                                  true /* replace */,
+                                  p.checkout_purge,
+                                  simulate)
+                  : pkg_checkout (checkout_cache,
+                                  o,
+                                  pdb,
+                                  t,
+                                  ap->id.name,
+                                  p.available_version (),
+                                  true /* replace */,
+                                  simulate);
                 break;
               }
             case repository_basis::directory:
@@ -6257,6 +6260,7 @@ namespace bpkg
         break; // Get out from the breakout loop.
       }
     }
+    checkout_cache.clear (); // Detect errors.
 
     // configure
     //
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx
index b184bfd..5b69d40 100644
--- a/bpkg/pkg-checkout.cxx
+++ b/bpkg/pkg-checkout.cxx
@@ -22,6 +22,8 @@ using namespace butl;
 
 namespace bpkg
 {
+  // pkg_checkout()
+  //
   static void
   checkout (const common_options& o,
             const repository_location& rl,
@@ -59,19 +61,20 @@ namespace bpkg
   // For some platforms/repository types the working tree needs to be
   // temporary "fixed up" for the build2 operations to work properly on it.
   //
-  static bool
+  static optional<bool>
   fixup (const common_options& o,
          const repository_location& rl,
          const dir_path& dir,
-         bool revert = false)
+         bool revert = false,
+         bool ie = false)
   {
-    bool r (false);
+    optional<bool> r;
 
     switch (rl.type ())
     {
     case repository_type::git:
       {
-        r = git_fixup_worktree (o, dir, revert);
+        r = git_fixup_worktree (o, dir, revert, ie);
         break;
       }
     case repository_type::pkg:
@@ -84,7 +87,8 @@ namespace bpkg
   // Return the selected package object which may replace the existing one.
   //
   static shared_ptr<selected_package>
-  pkg_checkout (const common_options& o,
+  pkg_checkout (pkg_checkout_cache& cache,
+                const common_options& o,
                 database& db,
                 transaction& t,
                 package_name n,
@@ -177,10 +181,7 @@ namespace bpkg
     // (or interruption) the user will need to run bpkg-rep-fetch to restore
     // the missing repository.
     //
-    bool fs_changed (false);
-
     if (!simulate)
-    try
     {
       if (exists (d))
         fail << "package directory " << d << " already exists";
@@ -191,18 +192,58 @@ namespace bpkg
       dir_path sd (repository_state (rl));
       dir_path rd (mdb.config_orig / repos_dir / sd);
 
-      if (!exists (rd))
-        fail << "missing repository directory for package " << n << " " << v
-             << " in configuration " << c <<
-          info << "run 'bpkg rep-fetch' to repair";
-
-      // The repository temporary directory.
+      // Try to reuse the cached repository (moved to the temporary directory
+      // with some fragment checked out and fixed up).
       //
-      auto_rmdir rmt (temp_dir / sd);
-      const dir_path& td (rmt.path);
+      pkg_checkout_cache::state_map& cm (cache.map_);
+      auto i (cm.find (rd));
+
+      if (i == cm.end () || i->second.rl.fragment () != rl.fragment ())
+      {
+        // Restore the repository if some different fragment is checked out.
+        //
+        if (i != cm.end ())
+          cache.erase (i);
+
+        // Checkout and cache the fragment.
+        //
+        if (!exists (rd))
+          fail << "missing repository directory for package " << n << " " << v
+               << " in configuration " << c <<
+            info << "run 'bpkg rep-fetch' to repair";
+
+        // The repository temporary directory.
+        //
+        auto_rmdir rmt (temp_dir / sd);
+
+        // Move the repository to the temporary directory.
+        //
+        {
+          const dir_path& td (rmt.path);
+
+          if (exists (td))
+            rm_r (td);
+
+          mv (rd, td);
+        }
+
+        // Pre-insert the incomplete repository entry into the cache and
+        // "finalize" it by setting the fixed up value later, after the
+        // repository fragment checkout succeeds. Until then the repository
+        // may not be restored in its permanent place.
+        //
+        using state = pkg_checkout_cache::state;
+
+        i = cm.emplace (rd, state {move (rmt), rl, nullopt}).first;
 
-      if (exists (td))
-        rm_r (td);
+        // Checkout the repository fragment and fix up the working tree.
+        //
+        state& s (i->second);
+        const dir_path& td (s.rmt.path);
+
+        checkout (o, rl, td, ap, db);
+        s.fixedup = fixup (o, rl, td);
+      }
 
       // The temporary out of source directory that is required for the dist
       // meta-operation.
@@ -213,21 +254,10 @@ namespace bpkg
       if (exists (od))
         rm_r (od);
 
-      // Finally, move the repository to the temporary directory and proceed
-      // with the checkout.
-      //
-      mv (rd, td);
-      fs_changed = true;
-
-      // Checkout the repository fragment and fix up the working tree.
-      //
-      checkout (o, rl, td, ap, db);
-      bool fixedup (fixup (o, rl, td));
-
       // Calculate the package path that points into the checked out fragment
       // directory.
       //
-      dir_path pd (td / path_cast<dir_path> (pl->location));
+      dir_path pd (i->second.rmt.path / path_cast<dir_path> (pl->location));
 
       // Form the buildspec.
       //
@@ -271,33 +301,8 @@ namespace bpkg
              "config.dist.root='" + ord.representation () + "'",
              bspec);
 
-      // Revert the fix-ups.
-      //
-      if (fixedup)
-        fixup (o, rl, td, true /* revert */);
-
-      // Manipulations over the repository are now complete, so we can return
-      // it to its permanent location.
-      //
-      mv (td, rd);
-      fs_changed = false;
-
-      rmt.cancel ();
-
       mc = sha256 (o, d / manifest_file);
     }
-    catch (const failed&)
-    {
-      if (fs_changed)
-      {
-        // We assume that the diagnostics has already been issued.
-        //
-        warn << "repository state is now broken" <<
-          info << "run 'bpkg rep-fetch' to repair";
-      }
-
-      throw;
-    }
 
     if (p != nullptr)
     {
@@ -369,7 +374,8 @@ namespace bpkg
   }
 
   shared_ptr<selected_package>
-  pkg_checkout (const common_options& o,
+  pkg_checkout (pkg_checkout_cache& cache,
+                const common_options& o,
                 database& db,
                 transaction& t,
                 package_name n,
@@ -379,7 +385,8 @@ namespace bpkg
                 bool purge,
                 bool simulate)
   {
-    return pkg_checkout (o,
+    return pkg_checkout (cache,
+                         o,
                          db,
                          t,
                          move (n),
@@ -391,7 +398,8 @@ namespace bpkg
   }
 
   shared_ptr<selected_package>
-  pkg_checkout (const common_options& o,
+  pkg_checkout (pkg_checkout_cache& cache,
+                const common_options& o,
                 database& db,
                 transaction& t,
                 package_name n,
@@ -399,7 +407,8 @@ namespace bpkg
                 bool replace,
                 bool simulate)
   {
-    return pkg_checkout (o,
+    return pkg_checkout (cache,
+                         o,
                          db,
                          t,
                          move (n),
@@ -436,10 +445,13 @@ namespace bpkg
       fail << "package version expected" <<
         info << "run 'bpkg help pkg-checkout' for more information";
 
+    pkg_checkout_cache checkout_cache (o);
+
     // Commits the transaction.
     //
     if (o.output_root_specified ())
-      p = pkg_checkout (o,
+      p = pkg_checkout (checkout_cache,
+                        o,
                         db,
                         t,
                         move (n),
@@ -449,7 +461,8 @@ namespace bpkg
                         o.output_purge (),
                         false /* simulate */);
     else
-      p = pkg_checkout (o,
+      p = pkg_checkout (checkout_cache,
+                        o,
                         db,
                         t,
                         move (n),
@@ -457,9 +470,74 @@ namespace bpkg
                         o.replace (),
                         false /* simulate */);
 
+    checkout_cache.clear (); // Detect errors.
+
     if (verb && !o.no_result ())
       text << "checked out " << *p;
 
     return 0;
   }
+
+  // pkg_checkout_cache
+  //
+  pkg_checkout_cache::
+  ~pkg_checkout_cache ()
+  {
+    if (!map_.empty () && !clear (true /* ignore_errors */))
+    {
+      // We assume that the diagnostics has already been issued.
+      //
+      warn << "repository state is now broken" <<
+        info << "run 'bpkg rep-fetch' to repair";
+    }
+  }
+
+  bool pkg_checkout_cache::
+  clear (bool ie)
+  {
+    while (!map_.empty ())
+    {
+      if (!erase (map_.begin (), ie))
+        return false;
+    }
+
+    return true;
+  }
+
+  bool pkg_checkout_cache::
+  erase (state_map::iterator i, bool ie)
+  {
+    state& s (i->second);
+
+    // Bail out if the entry is incomplete.
+    //
+    if (!s.fixedup)
+    {
+      assert (ie); // Only makes sense in the ignore errors mode.
+      return false;
+    }
+
+    // Revert the fix-ups.
+    //
+    // But first make the entry incomplete, so on error we don't try to
+    // restore the partially restored repository later.
+    //
+    bool f (*s.fixedup);
+
+    s.fixedup = nullopt;
+
+    if (f && !fixup (options_, s.rl, s.rmt.path, true /* revert */, ie))
+      return false;
+
+    // Manipulations over the repository are now complete, so we can return it
+    // to the permanent location.
+    //
+    if (!mv (s.rmt.path, i->first, ie))
+      return false;
+
+    s.rmt.cancel ();
+
+    map_.erase (i);
+    return true;
+  }
 }
diff --git a/bpkg/pkg-checkout.hxx b/bpkg/pkg-checkout.hxx
index 3a058b5..d910509 100644
--- a/bpkg/pkg-checkout.hxx
+++ b/bpkg/pkg-checkout.hxx
@@ -4,6 +4,8 @@
 #ifndef BPKG_PKG_CHECKOUT_HXX
 #define BPKG_PKG_CHECKOUT_HXX
 
+#include <map>
+
 #include <libbpkg/manifest.hxx>     // version
 #include <libbpkg/package-name.hxx>
 
@@ -18,13 +20,70 @@ namespace bpkg
   int
   pkg_checkout (const pkg_checkout_options&, cli::scanner& args);
 
+  // Checked out repository fragments cache.
+  //
+  // Needs to be passed to pkg_checkout() calls.
+  //
+  class pkg_checkout_cache
+  {
+  public:
+    // The options reference is assumed to be valid till the end of the cache
+    // object lifetime.
+    //
+    pkg_checkout_cache (const common_options& o): options_ (o) {}
+
+    // Restore the cached repositories in their permanent locations (move back
+    // from the temporary directory, fixup, etc) and erase the entries.
+    //
+    // Note that the destructor will clear the cache but will ignore any
+    // errors. To detect such errors, call clear() explicitly.
+    //
+    bool
+    clear (bool ignore_errors = false);
+
+    // Call clear() in the ignore errors mode and issue the "repository is now
+    // broken" warning on failure.
+    //
+    ~pkg_checkout_cache ();
+
+    // pkg_checkout () implementation details.
+    //
+  public:
+    struct state
+    {
+      auto_rmdir rmt;         // The repository temporary directory.
+      repository_location rl; // The repository location.
+
+      // nullopt if the repository fragment checkout failed in the middle and
+      // the repository cannot be restored in its permanent location (we will
+      // call such entry incomplete). True if the repository directory was
+      // fixed up.
+      //
+      optional<bool> fixedup;
+    };
+
+    using state_map = std::map<dir_path, state>;
+
+    state_map map_;
+    const common_options& options_;
+
+    // Restore the repository in its permanent location and erase the cache
+    // entry. On error issue diagnostics and return false in the ignore errors
+    // mode and throw failed otherwise. Note that erasing an incomplete entry
+    // is an error.
+    //
+    bool
+    erase (state_map::iterator, bool ignore_errors = false);
+  };
+
   // Check out the package from a version control-based repository into a
   // directory other than the configuration directory and commit the
   // transaction. Return the selected package object which may replace the
   // existing one.
   //
   shared_ptr<selected_package>
-  pkg_checkout (const common_options&,
+  pkg_checkout (pkg_checkout_cache&,
+                const common_options&,
                 database&,
                 transaction&,
                 package_name,
@@ -39,7 +98,8 @@ namespace bpkg
   // existing one.
   //
   shared_ptr<selected_package>
-  pkg_checkout (const common_options&,
+  pkg_checkout (pkg_checkout_cache&,
+                const common_options&,
                 database&,
                 transaction&,
                 package_name,
diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx
index 40ae02c..b179f63 100644
--- a/bpkg/utility.cxx
+++ b/bpkg/utility.cxx
@@ -255,8 +255,8 @@ namespace bpkg
     }
   }
 
-  void
-  mv (const dir_path& from, const dir_path& to)
+  bool
+  mv (const dir_path& from, const dir_path& to, bool ie)
   {
     if (verb >= 3)
       text << "mv " << from << ' ' << to; // Prints trailing slashes.
@@ -267,8 +267,16 @@ namespace bpkg
     }
     catch (const system_error& e)
     {
-      fail << "unable to move directory " << from << " to " << to << ": " << e;
+      error << "unable to move directory " << from << " to " << to << ": "
+            << e;
+
+      if (ie)
+        return false;
+
+      throw failed ();
     }
+
+    return true;
   }
 
   dir_path
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index 73dd107..ce6120f 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -171,8 +171,10 @@ namespace bpkg
         uint16_t verbosity = 3,
         rm_error_mode = rm_error_mode::fail);
 
-  void
-  mv (const dir_path& from, const dir_path& to);
+  // Note that if ignore_error is true, the diagnostics is still issued.
+  //
+  bool
+  mv (const dir_path& from, const dir_path& to, bool ignore_errors = false);
 
   // Set (with diagnostics at verbosity level 3 or higher) the new and return
   // the previous working directory.