Update rep-fetch

15 files changed, 578 insertions, 177 deletions

diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index 7ffaaaf..17c56c5 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -292,7 +292,7 @@ namespace bpkg
   // histories we will use the file URL notation for local repositories.
   //
   static string
-  git_url (const repository_url& url)
+  to_git_url (const repository_url& url)
   {
     if (url.scheme != repository_protocol::file)
       return url.string ();
@@ -326,6 +326,23 @@ namespace bpkg
 #endif
   }
 
+  // Create the URL object from a string representation printed by git
+  // commands.
+  //
+  static repository_url
+  from_git_url (string&& u)
+  {
+    // Fix-up the broken Windows file URL notation (see to_git_url() for
+    // details).
+    //
+#ifdef _WIN32
+    if (casecmp (u, "file://", 7) == 0 && u[7] != '/')
+      u.insert (7, 1, '/');
+#endif
+
+    return repository_url (u);
+  }
+
   // Sense the git protocol capabilities for a specified URL.
   //
   // Protocols other than HTTP(S) are considered smart but without the
@@ -477,7 +494,7 @@ namespace bpkg
                            co.git_option (),
                            "ls-remote",
                            "--refs",
-                           git_url (url)));
+                           to_git_url (url)));
 
     pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
 
@@ -798,21 +815,12 @@ namespace bpkg
 
         try
         {
-          string u (git_string (co, "submodule URL",
-                                co.git_option (),
-                                "-C", dir,
-                                "config",
-                                "--get",
-                                "submodule." + name + ".url"));
-
-          // Fix-up the broken Windows file URL notation (see the git_url()
-          // function for details).
-          //
-#ifdef _WIN32
-          if (casecmp (u, "file://", 7) == 0 && u[7] != '/')
-            u.insert (7, 1, '/');
-#endif
-          url = repository_url (u);
+          url = from_git_url (git_string (co, "submodule URL",
+                                          co.git_option (),
+                                          "-C", dir,
+                                          "config",
+                                          "--get",
+                                          "submodule." + name + ".url"));
         }
         catch (const invalid_argument& e)
         {
@@ -859,7 +867,7 @@ namespace bpkg
 
                         "--name", name,
                         "--path", sdir,
-                        "--url", git_url (url),
+                        "--url", to_git_url (url),
                         shallow
                         ? cstrings ({"--depth", "1"})
                         : cstrings (),
@@ -978,7 +986,7 @@ namespace bpkg
           ref.commit    ? opt      ("--no-checkout")          : nullopt,
 
           verb < 1 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt,
-          git_url (url),
+          to_git_url (url),
           d))
       fail << "unable to clone " << url << endg;
 
@@ -1012,12 +1020,55 @@ namespace bpkg
 
     assert (ref.branch);
 
-    capabilities cap (sense_capabilities (co, url));
-    bool shallow (shallow_fetch (co, url, cap, ref));
-
     dir_path d (destdir);
     d /= dir_path (*ref.branch);
 
+    // If the repository location differs from the one that was used to clone
+    // the repository then we re-clone it from the new location.
+    //
+    // Another (more hairy) way of doing this would be fixing up the remote
+    // origin URLs recursively prior to fetching.
+    //
+    try
+    {
+      repository_url u (from_git_url (git_string (co, "remote repository URL",
+                                                  co.git_option (),
+                                                  "-C", d,
+                                                  "config",
+                                                  "--get",
+                                                  "remote.origin.url")));
+      if (u != url)
+      {
+        // Note that the repository canonical name can not change under the
+        // legal scenarios that lead to the location change. Changed canonical
+        // name means that the repository was manually amended. We could
+        // re-clone such repositories as well but want to leave the backdoor
+        // for tests.
+        //
+        u.fragment = rl.url ().fragment;       // Restore the fragment.
+        repository_location l (u, rl.type ());
+
+        if (rl.canonical_name () == l.canonical_name ())
+        {
+          if (verb)
+            info << "re-cloning " << rl.canonical_name ()
+                 << " due to location change" <<
+              info << "new location " << rl.url () <<
+              info << "old location " << u;
+
+          rm_r (d);
+          return git_clone (co, rl, destdir);
+        }
+      }
+    }
+    catch (const invalid_argument& e)
+    {
+      fail << "invalid remote.origin.url configuration value: " << e << endg;
+    }
+
+    capabilities cap (sense_capabilities (co, url));
+    bool shallow (shallow_fetch (co, url, cap, ref));
+
     update_tree (co,
                  d,
                  dir_path (),
diff --git a/bpkg/forward.hxx b/bpkg/forward.hxx
index cb49926..c253e72 100644
--- a/bpkg/forward.hxx
+++ b/bpkg/forward.hxx
@@ -14,6 +14,7 @@ namespace bpkg
 
   // <bpkg/package.hxx>
   //
+  class repository;
   class selected_package;
 }
 
diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 1500cee..8377bab 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -68,7 +68,7 @@ namespace bpkg
   }
 
   repository_location
-  parse_location (const char* s, optional<repository_type> ot)
+  parse_location (const string& s, optional<repository_type> ot)
   try
   {
     repository_url u (s);
diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx
index f0d5cdd..a0966d4 100644
--- a/bpkg/manifest-utility.hxx
+++ b/bpkg/manifest-utility.hxx
@@ -39,7 +39,7 @@ namespace bpkg
   // current working directory. Diagnose invalid locations and throw failed.
   //
   repository_location
-  parse_location (const char*, optional<repository_type>);
+  parse_location (const string&, optional<repository_type>);
 
   // Return the repository state subdirectory for the specified location as it
   // appears under .bpkg/repositories/ in the bpkg configuration. Return empty
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index 54efca1..e864b59 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -244,12 +244,6 @@ namespace bpkg
     complements_type complements;
     prerequisites_type prerequisites;
 
-    // Used to detect recursive fetching. Will probably be replaced
-    // by the 'repositories' file timestamp or hashsum later.
-    //
-    #pragma db transient
-    bool fetched = false;
-
   public:
     explicit
     repository (repository_location l): location (move (l))
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index 384cb91..fc6ff26 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -618,7 +618,9 @@ namespace bpkg
 
             if (!ar->location.empty ())
               dr << info << "repository " << ar->location << " appears to "
-                 << "be broken";
+                 << "be broken" <<
+                info << "or the repository state could be stale" <<
+                info << "run 'bpkg rep-fetch' to update";
           }
 
           // If all that's available is a stub then we need to make sure the
diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx
index 690b584..2dcce9a 100644
--- a/bpkg/rep-add.cxx
+++ b/bpkg/rep-add.cxx
@@ -15,6 +15,42 @@ using namespace butl;
 
 namespace bpkg
 {
+  shared_ptr<repository>
+  rep_add (database& db, const repository_location& rl)
+  {
+    const string& rn (rl.canonical_name ());
+
+    shared_ptr<repository> r (db.find<repository> (rn));
+
+    bool updated (false);
+
+    if (r == nullptr)
+    {
+      r.reset (new repository (rl));
+      db.persist (r);
+    }
+    else if (r->location.url () != rl.url ())
+    {
+      r->location = rl;
+      db.update (r);
+
+      updated = true;
+    }
+
+    shared_ptr<repository> root (db.load<repository> (""));
+
+    bool added (
+      root->complements.insert (lazy_shared_ptr<repository> (db, r)).second);
+
+    if (added)
+      db.update (root);
+
+    if (verb)
+      text << (added ? "added " : updated ? "updated " : "unchanged ") << rn;
+
+    return r;
+  }
+
   int
   rep_add (const rep_add_options& o, cli::scanner& args)
   {
@@ -31,8 +67,6 @@ namespace bpkg
     transaction t (db.begin ());
     session s; // Repository dependencies can have cycles.
 
-    shared_ptr<repository> root (db.load<repository> (""));
-
     while (args.more ())
     {
       repository_location rl (
@@ -41,37 +75,9 @@ namespace bpkg
                         ? optional<repository_type> (o.type ())
                         : nullopt));
 
-      const string& rn (rl.canonical_name ());
-
-      // Create the new repository if it is not in the database yet. Otherwise
-      // update its location. Add it as a complement to the root repository (if
-      // it is not there yet).
-      //
-      shared_ptr<repository> r (db.find<repository> (rn));
-
-      bool updated (false);
-
-      if (r == nullptr)
-      {
-        r.reset (new repository (rl));
-        db.persist (r);
-      }
-      else if (r->location.url () != rl.url ())
-      {
-        r->location = rl;
-        db.update (r);
-
-        updated = true;
-      }
-
-      bool added (
-        root->complements.insert (lazy_shared_ptr<repository> (db, r)).second);
-
-      if (verb)
-        text << (added ? "added " : updated ? "updated " : "unchanged ") << rn;
+      rep_add (db, rl);
     }
 
-    db.update (root);
     t.commit ();
 
     return 0;
diff --git a/bpkg/rep-add.hxx b/bpkg/rep-add.hxx
index 9f91713..192df05 100644
--- a/bpkg/rep-add.hxx
+++ b/bpkg/rep-add.hxx
@@ -5,7 +5,10 @@
 #ifndef BPKG_REP_ADD_HXX
 #define BPKG_REP_ADD_HXX
 
+#include <libbpkg/manifest.hxx>
+
 #include <bpkg/types.hxx>
+#include <bpkg/forward.hxx> // database, repository
 #include <bpkg/utility.hxx>
 
 #include <bpkg/rep-add-options.hxx>
@@ -14,6 +17,13 @@ namespace bpkg
 {
   int
   rep_add (const rep_add_options&, cli::scanner& args);
+
+  // Create the new repository if it is not in the database yet or update its
+  // location if it differs. Then add it as a complement to the root
+  // repository if it is not already.
+  //
+  shared_ptr<repository>
+  rep_add (database&, const repository_location&);
 }
 
 #endif // BPKG_REP_ADD_HXX
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 3956c5a..3bede3c 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -4,15 +4,19 @@
 
 #include <bpkg/rep-fetch.hxx>
 
+#include <set>
+
 #include <libbutl/process.mxx>
 #include <libbutl/process-io.mxx>      // operator<<(ostream, process_path)
 #include <libbutl/manifest-parser.mxx>
 
 #include <bpkg/auth.hxx>
 #include <bpkg/fetch.hxx>
+#include <bpkg/rep-add.hxx>
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
 #include <bpkg/database.hxx>
+#include <bpkg/rep-remove.hxx>
 #include <bpkg/diagnostics.hxx>
 #include <bpkg/manifest-utility.hxx>
 
@@ -21,6 +25,22 @@ using namespace butl;
 
 namespace bpkg
 {
+  // The fetch operation failure may result in mismatch of the (rolled back)
+  // repository database state and the repository filesystem state. Restoring
+  // the filesystem state on failure would require making copies which seems
+  // unnecessarily pessimistic. So instead, we will revert the repository
+  // state to the clean state as if repositories were added but never fetched
+  // (see rep_remove_clean() for more details).
+  //
+  // The following flag is set by the rep_fetch_*() functions when they are
+  // about to change the repository filesystem state. That, in particular,
+  // means that the flag will be set even if the subsequent fetch operation
+  // fails, and so the caller can rely on it while handling the thrown
+  // exception. The flag must be reset by such a caller prior to the
+  // rep_fetch_*() call.
+  //
+  static bool filesystem_state_changed;
+
   static rep_fetch_data
   rep_fetch_bpkg (const common_options& co,
                   const dir_path* conf,
@@ -157,6 +177,14 @@ namespace bpkg
     auto_rmdir rm (temp_dir / sd);
     dir_path& td (rm.path);
 
+    // We are about to modify the repository filesystem state.
+    //
+    // In the future we can probably do something smarter about the flag,
+    // keeping it unset unless the repository state directory is really
+    // changed.
+    //
+    filesystem_state_changed = true;
+
     if (exists (td))
       rm_r (td);
 
@@ -389,21 +417,40 @@ namespace bpkg
     return rep_fetch_data ();
   }
 
+  using repositories = set<shared_ptr<repository>>;
+
   static void
-  rep_fetch (const configuration_options& co,
-             transaction& t,
+  rep_fetch (const common_options& co,
+             const dir_path& conf,
+             database& db,
              const shared_ptr<repository>& r,
-             const shared_ptr<repository>& root,
-             const string& reason)
+             repositories& fetched,
+             repositories& removed,
+             const string& reason = string ())
   {
     tracer trace ("rep_fetch(rep)");
 
-    database& db (t.database ());
     tracer_guard tg (db, trace);
 
+    // Check that the repository is not fetched yet and register it as fetched
+    // otherwise.
+    //
+    // Note that we can end up with a repository dependency cycle via
+    // prerequisites. Thus we register the repository before recursing into its
+    // dependencies.
+    //
+    if (!fetched.insert (r).second) // Is already fetched.
+      return;
+
     const repository_location& rl (r->location);
     l4 ([&]{trace << r->name << " " << rl;});
-    assert (rl.absolute () || rl.remote ());
+
+    // Cancel the repository removal.
+    //
+    // Note that this is an optimization as the rep_remove() function checks
+    // for reachability of the repository being removed.
+    //
+    removed.erase (r);
 
     // The fetch_*() functions below will be quiet at level 1, which
     // can be quite confusing if the download hangs.
@@ -414,23 +461,45 @@ namespace bpkg
 
       dr << "fetching " << r->name;
 
-      const auto& ua (root->complements);
-
-      if (ua.find (lazy_shared_ptr<repository> (db, r)) == ua.end ())
-      {
-        assert (!reason.empty ());
+      if (!reason.empty ())
         dr << " (" << reason << ")";
-      }
     }
 
-    r->fetched = true; // Mark as being fetched.
+    // Register complements and prerequisites for potential removal unless
+    // they are fetched. Clear repository dependency sets afterwards.
+    //
+    auto remove = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp)
+    {
+      shared_ptr<repository> r (rp.load ());
+      if (fetched.find (r) == fetched.end ())
+        removed.insert (move (r));
+    };
 
-    // Load the repositories and packages and use it to populate the
+    for (const lazy_shared_ptr<repository>& cr: r->complements)
+    {
+      // Remove the complement unless it is the root repository (see
+      // rep_fetch() for details).
+      //
+      if (cr.object_id () != "")
+        remove (cr);
+    }
+
+    for (const lazy_weak_ptr<repository>& pr: r->prerequisites)
+      remove (lazy_shared_ptr<repository> (pr));
+
+    r->complements.clear ();
+    r->prerequisites.clear ();
+
+    // Remove this repository from locations of the available packages it
+    // contains.
+    //
+    rep_remove_package_locations (db, r->name);
+
+    // Load the repository and package manifests and use them to populate the
     // prerequisite and complement repository sets as well as available
     // packages.
     //
-    rep_fetch_data rfd (
-      rep_fetch (co, &co.directory (), rl, true /* ignore_unknow */));
+    rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */));
 
     for (repository_manifest& rm: rfd.repositories)
     {
@@ -439,51 +508,53 @@ namespace bpkg
       if (rr == repository_role::base)
         continue; // Entry for this repository.
 
+      repository_location& l (rm.location);
+
       // If the location is relative, complete it using this repository
       // as a base.
       //
-      if (rm.location.relative ())
+      if (l.relative ())
       {
         try
         {
-          rm.location = repository_location (rm.location, rl);
+          l = repository_location (l, rl);
         }
         catch (const invalid_argument& e)
         {
-          fail << "invalid relative repository location '" << rm.location
+          fail << "invalid relative repository location '" << l
                << "': " << e <<
             info << "base repository location is " << rl;
         }
       }
 
-      // We might already have this repository in the database.
+      // Create the new repository if it is not in the database yet. Otherwise
+      // update its location.
       //
-      shared_ptr<repository> pr (
-        db.find<repository> (
-          rm.location.canonical_name ()));
+      shared_ptr<repository> pr (db.find<repository> (l.canonical_name ()));
 
       if (pr == nullptr)
       {
-        pr = make_shared<repository> (move (rm.location));
+        pr = make_shared<repository> (move (l));
         db.persist (pr); // Enter into session, important if recursive.
       }
+      else if (pr->location.url () != l.url ())
+      {
+        pr->location = move (l);
+        db.update (r);
+      }
 
-      // Load the prerequisite repository unless it has already been
-      // (or is already being) fetched.
+      // Load the prerequisite repository.
       //
-      if (!pr->fetched)
+      string reason;
+      switch (rr)
       {
-        string reason;
-        switch (rr)
-        {
-        case repository_role::complement:   reason = "complements ";     break;
-        case repository_role::prerequisite: reason = "prerequisite of "; break;
-        case repository_role::base: assert (false);
-        }
-        reason += r->name;
-
-        rep_fetch (co, t, pr, root, reason);
+      case repository_role::complement:   reason = "complements ";     break;
+      case repository_role::prerequisite: reason = "prerequisite of "; break;
+      case repository_role::base: assert (false);
       }
+      reason += r->name;
+
+      rep_fetch (co, conf, db, pr, fetched, removed, reason);
 
       // @@ What if we have duplicated? Ideally, we would like to check
       //    this once and as early as possible. The original idea was to
@@ -530,7 +601,11 @@ namespace bpkg
     if (rl.type () == repository_type::git &&
         r->complements.empty ()            &&
         r->prerequisites.empty ())
-      r->complements.insert (lazy_shared_ptr<repository> (db, root));
+      r->complements.insert (lazy_shared_ptr<repository> (db, string ()));
+
+    // Save the changes to the repository object.
+    //
+    db.update (r);
 
     // "Suspend" session while persisting packages to reduce memory
     // consumption.
@@ -600,70 +675,156 @@ namespace bpkg
     }
 
     session::current (s); // "Resume".
+  }
 
-    // Save the changes to the repository object.
+  static void
+  rep_fetch (const common_options& o,
+             const dir_path& conf,
+             transaction& t,
+             const vector<lazy_shared_ptr<repository>>& repos)
+  {
+    database& db (t.database ());
+
+    // As a fist step we fetch repositories recursively building the list of
+    // the former prerequisites and complements to be considered for removal.
     //
-    db.update (r);
+    // We delay the actual removal until we fetch all the required repositories
+    // as a dependency dropped by one repository can appear for another one.
+    //
+    try
+    {
+      // If fetch fails and the repository filesystem state is changed, then
+      // the configuration is broken, and we have to take some drastic
+      // measures (see below).
+      //
+      filesystem_state_changed = false;
+
+      repositories fetched;
+      repositories removed;
+
+      for (const lazy_shared_ptr<repository>& r: repos)
+        rep_fetch (o, conf, db, r.load (), fetched, removed);
+
+      // Finally, remove dangling repositories.
+      //
+      for (const shared_ptr<repository>& r: removed)
+        rep_remove (conf, db, r);
+    }
+    catch (const failed&)
+    {
+      t.rollback ();
+
+      if (filesystem_state_changed)
+      {
+        // Warn prior to the cleanup operation that potentially can also fail.
+        // Note that we assume that the diagnostics has already been issued.
+        //
+        warn << "repository state is now broken and will be cleaned up" <<
+          info << "run 'bpkg rep-fetch' to update";
+
+        rep_remove_clean (conf, db);
+      }
+
+      throw;
+    }
+  }
+
+  void
+  rep_fetch (const common_options& o,
+             const dir_path& conf,
+             database& db,
+             const vector<repository_location>& rls)
+  {
+    vector<lazy_shared_ptr<repository>> repos;
+    repos.reserve (rls.size ());
+
+    transaction t (db.begin ());
+
+    shared_ptr<repository> root (db.load<repository> (""));
+    repository::complements_type& ua (root->complements); // User-added repos.
+
+    for (const repository_location& rl: rls)
+    {
+      lazy_shared_ptr<repository> r (db, rl.canonical_name ());
+
+      // Add the repository, unless it is already a top-level one and has the
+      // same location.
+      //
+      if (ua.find (r) == ua.end () || r.load ()->location.url () != rl.url ())
+        rep_add (db, rl);
+
+      repos.emplace_back (r);
+    }
+
+    rep_fetch (o, conf, t, repos);
+
+    t.commit ();
   }
 
   int
-  rep_fetch (const rep_fetch_options& o, cli::scanner&)
+  rep_fetch (const rep_fetch_options& o, cli::scanner& args)
   {
     tracer trace ("rep_fetch");
 
     dir_path c (o.directory ());
     l4 ([&]{trace << "configuration: " << c;});
 
+    // Build the list of repositories the user wants to fetch.
+    //
+    vector<lazy_shared_ptr<repository>> repos;
+
     database db (open (c, trace));
     transaction t (db.begin ());
     session s; // Repository dependencies can have cycles.
 
     shared_ptr<repository> root (db.load<repository> (""));
-    const auto& ua (root->complements); // User-added repositories.
+    repository::complements_type& ua (root->complements); // User-added repos.
 
-    if (ua.empty ())
-      fail << "configuration " << c << " has no repositories" <<
-        info << "use 'bpkg rep-add' to add a repository";
-
-    // Clean repositories and available packages. At the end only
-    // repositories that were explicitly added by the user and the
-    // special root repository should remain.
-    //
-    db.erase_query<available_package> ();
+    if (!args.more ())
+    {
+      if (ua.empty ())
+        fail << "configuration " << c << " has no repositories" <<
+          info << "use 'bpkg rep-add' to add a repository";
 
-    for (shared_ptr<repository> r: pointer_result (db.query<repository> ()))
+      for (const lazy_shared_ptr<repository>& r: ua)
+        repos.push_back (r);
+    }
+    else
     {
-      if (r == root)
-      {
-        l5 ([&]{trace << "skipping root";});
-      }
-      else if (ua.find (lazy_shared_ptr<repository> (db, r)) != ua.end ())
+      while (args.more ())
       {
-        l4 ([&]{trace << "cleaning " << r->name;});
+        // Try to map the argument to a user-added repository.
+        //
+        // If this is a repository name then it must be present in the
+        // configuration. If this is a repository location then we add it to
+        // the configuration.
+        //
+        lazy_shared_ptr<repository> r;
+        string a (args.next ());
 
-        r->complements.clear ();
-        r->prerequisites.clear ();
-        r->fetched = false;
-        db.update (r);
-      }
-      else
-      {
-        l4 ([&]{trace << "erasing " << r->name;});
-        db.erase (r);
-      }
-    }
+        if (repository_name (a))
+        {
+          lazy_shared_ptr<repository> rp (db, a);
 
-    // Now recursively fetch prerequisite/complement repositories and
-    // their packages.
-    //
-    for (const lazy_shared_ptr<repository>& lp: ua)
-    {
-      shared_ptr<repository> r (lp.load ());
+          if (ua.find (rp) != ua.end ())
+            r = move (rp);
+          else
+            fail << "repository '" << a << "' does not exist in this "
+                 << "configuration";
+        }
+        else
+          //@@ TODO: check if exists in root & same location and avoid
+          // calling rep_add. Get rid of quiet mode.
+          //
+          r = lazy_shared_ptr<repository> (
+            db, rep_add (db, parse_location (a, nullopt /* type */)));
 
-      if (!r->fetched) // Can already be loaded as a prerequisite/complement.
-        rep_fetch (o, t, r, root, ""); // No reason (user-added).
+        repos.emplace_back (move (r));
+      }
     }
 
+    rep_fetch (o, c, t, repos);
+
     size_t rcount (0), pcount (0);
     if (verb)
     {
diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx
index 0a7cacd..b26971c 100644
--- a/bpkg/rep-fetch.hxx
+++ b/bpkg/rep-fetch.hxx
@@ -8,6 +8,7 @@
 #include <libbpkg/manifest.hxx>
 
 #include <bpkg/types.hxx>
+#include <bpkg/forward.hxx> // database
 #include <bpkg/utility.hxx>
 
 #include <bpkg/rep-fetch-options.hxx>
@@ -44,10 +45,21 @@ namespace bpkg
   };
 
   rep_fetch_data
-  rep_fetch (const common_options& co,
+  rep_fetch (const common_options&,
              const dir_path* conf,
-             const repository_location& rl,
+             const repository_location&,
              bool ignore_unknown);
+
+  // Add (or update) repository locations to the configuration and fetch
+  // them. On failure clean up the configuration (see rep_remove_clean() for
+  // details). Note that it starts a new transaction and should be called in
+  // session.
+  //
+  void
+  rep_fetch (const common_options&,
+             const dir_path& conf,
+             database&,
+             const vector<repository_location>&);
 }
 
 #endif // BPKG_REP_FETCH_HXX
diff --git a/bpkg/rep-remove.cli b/bpkg/rep-remove.cli
index 3662b2b..25280b2 100644
--- a/bpkg/rep-remove.cli
+++ b/bpkg/rep-remove.cli
@@ -17,8 +17,9 @@ namespace bpkg
 
     "\h|SYNOPSIS|
 
-     \c{\b{bpkg rep-remove}|\b{remove} [<options>] (<name>|<location>)... |
-        (--all|-a)}
+     \c{\b{bpkg rep-remove}|\b{remove} [<options>] (<name>|<location>)...}\n
+     \c{\b{bpkg rep-remove}|\b{remove} [<options>] (--all|-a)}\n
+     \c{\b{bpkg rep-remove}|\b{remove} [<options>] --clean}
 
      \h|DESCRIPTION|
 
@@ -29,7 +30,10 @@ namespace bpkg
 
      Alternatively, the \cb{--all|-a} option can be used to remove all the
      repositories that were previously added (\l{bpkg-rep-add(1)}) to the
-     configuration."
+     configuration.
+
+     Finally, the \cb{--clean} option can be used to revert the repositories
+     to the clean state, as if they were added but never fetched."
   }
 
   class rep_remove_options: configuration_options
@@ -40,5 +44,10 @@ namespace bpkg
     {
       "Remove all the repositories."
     }
+
+    bool --clean
+    {
+      "Clean the repository state."
+    }
   };
 }
diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx
index d561c0c..df9acde 100644
--- a/bpkg/rep-remove.cxx
+++ b/bpkg/rep-remove.cxx
@@ -7,6 +7,8 @@
 #include <set>
 #include <algorithm> // find()
 
+#include <libbutl/filesystem.mxx> // dir_iterator
+
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
 #include <bpkg/database.hxx>
@@ -14,6 +16,7 @@
 #include <bpkg/manifest-utility.hxx>
 
 using namespace std;
+using namespace butl;
 
 namespace bpkg
 {
@@ -68,30 +71,18 @@ namespace bpkg
     return reachable (db, r, traversed);
   }
 
-  // Remove a repository if it is not reachable from the root (and thus is not
-  // required by any user-added repository).
-  //
-  static void
-  rep_remove (const dir_path& c, database& db, const shared_ptr<repository>& r)
+  void
+  rep_remove_package_locations (database& db, const string& name)
   {
-    const string& nm (r->name);
-    assert (!nm.empty ());      // Can't be the root repository.
-
-    if (reachable (db, r))
-      return;
-
-    // Remove this repository from locations of the available packages it
-    // contains. Remove packages that come from only this repository.
-    //
     for (const auto& rp: db.query<repository_package> (
-           query<repository_package>::repository::name == nm))
+           query<repository_package>::repository::name == name))
     {
       const shared_ptr<available_package>& p (rp);
       vector<package_location>& ls (p->locations);
 
       for (auto i (ls.cbegin ()), e (ls.cend ()); i != e; ++i)
       {
-        if (i->repository.object_id () == nm)
+        if (i->repository.object_id () == name)
         {
           ls.erase (i);
           break;
@@ -103,6 +94,33 @@ namespace bpkg
       else
         db.update (p);
     }
+  }
+
+  // Remove a directory moving it to the temporary directory first, increasing
+  // the chances for the operation to succeed.
+  //
+  static void
+  rmdir (const dir_path& d)
+  {
+    dir_path td (temp_dir / d.leaf ());
+
+    if (exists (td))
+      rm_r (td);
+
+    mv (d, td);
+    rm_r (td, true /* dir_itself */, 3, rm_error_mode::warn);
+  }
+
+  void
+  rep_remove (const dir_path& c, database& db, const shared_ptr<repository>& r)
+  {
+    const string& nm (r->name);
+    assert (!nm.empty ());      // Can't be the root repository.
+
+    if (reachable (db, r))
+      return;
+
+    rep_remove_package_locations (db, nm);
 
     // Cleanup the repository state if present.
     //
@@ -122,7 +140,7 @@ namespace bpkg
       dir_path sd (c / repos_dir / d);
 
       if (exists (sd))
-        rm_r (sd);
+        rmdir (sd);
     }
 
     // Note that it is essential to erase the repository object from the
@@ -157,6 +175,72 @@ namespace bpkg
       remove (lazy_shared_ptr<repository> (pr));
   }
 
+  void
+  rep_remove_clean (const dir_path& c,
+                    database& db,
+                    bool quiet)
+  {
+    tracer trace ("rep_remove_clean");
+
+    assert (!transaction::has_current ());
+
+    // Clean repositories and available packages. At the end only repositories
+    // that were explicitly added by the user and the special root repository
+    // should remain.
+    //
+    {
+      // Note that we don't rely on being in session nor create one.
+      //
+      transaction t (db.begin ());
+
+      db.erase_query<available_package> ();
+
+      shared_ptr<repository> root (db.load<repository> (""));
+      repository::complements_type& ua (root->complements);
+
+      for (shared_ptr<repository> r: pointer_result (db.query<repository> ()))
+      {
+        if (r->name == "")
+        {
+          l5 ([&]{trace << "skipping root";});
+        }
+        else if (ua.find (lazy_shared_ptr<repository> (db, r)) != ua.end ())
+        {
+          r->complements.clear ();
+          r->prerequisites.clear ();
+          db.update (r);
+
+          if (verb >= (quiet ? 2 : 1))
+            text << "cleaned " << r->name;
+        }
+        else
+        {
+          l4 ([&]{trace << "erasing " << r->name;});
+          db.erase (r);
+        }
+      }
+
+      t.commit ();
+    }
+
+    // Remove repository state subdirectories.
+    //
+    dir_path rd (c / repos_dir);
+
+    try
+    {
+      for (const dir_entry& de: dir_iterator (rd)) // system_error
+      {
+        if (de.ltype () == entry_type::directory)
+          rmdir (rd / path_cast<dir_path> (de.path ()));
+      }
+    }
+    catch (const system_error& e)
+    {
+      fail << "unable to scan directory " << rd << ": " << e;
+    }
+  }
+
   int
   rep_remove (const rep_remove_options& o, cli::scanner& args)
   {
@@ -167,21 +251,44 @@ namespace bpkg
 
     // Check that options and arguments are consistent.
     //
-    if (o.all ())
     {
-      if (args.more ())
-        fail << "both --all|-a and repository name specified" <<
-          info << "run 'bpkg help rep-remove' for more information";
+      diag_record dr;
+
+      if (o.clean ())
+      {
+        if (o.all ())
+          dr << fail << "both --clean and --all|-a specified";
+        else if (args.more ())
+          dr << fail << "both --clean and repository argument specified";
+      }
+      else if (o.all ())
+      {
+        if (args.more ())
+          dr << fail << "both --all|-a and repository argument specified";
+      }
+      else if (!args.more ())
+        dr << fail << "repository name or location argument expected";
+
+      if (!dr.empty ())
+        dr << info << "run 'bpkg help rep-remove' for more information";
     }
-    else if (!args.more ())
-      fail << "repository name or location argument expected" <<
-        info << "run 'bpkg help rep-remove' for more information";
 
+    database db (open (c, trace));
+
+    // Clean the configuration if requested.
+    //
+    if (o.clean ())
+    {
+      rep_remove_clean (c, db, false /* quiet */);
+      return 0;
+    }
+
+    // Remove the specified repositories.
+    //
     // Build the list of repositories the user wants removed.
     //
     vector<lazy_shared_ptr<repository>> repos;
 
-    database db (open (c, trace));
     transaction t (db.begin ());
     session s; // Repository dependencies can have cycles.
 
@@ -259,14 +366,14 @@ namespace bpkg
     // Note that for efficiency we un-reference all the top-level repositories
     // before starting to delete them.
     //
-    for (lazy_shared_ptr<repository>& r: repos)
+    for (const lazy_shared_ptr<repository>& r: repos)
       ua.erase (r);
 
     db.update (root);
 
     // Remove the dangling repositories from the database, recursively.
     //
-    for (lazy_shared_ptr<repository>& r: repos)
+    for (const lazy_shared_ptr<repository>& r: repos)
     {
       rep_remove (c, db, r.load ());
 
@@ -274,6 +381,11 @@ namespace bpkg
         text << "removed " << r.object_id ();
     }
 
+    // If the --all option is specified then no user-added repositories should
+    // remain.
+    //
+    assert (!o.all () || ua.empty ());
+
     // If we removed all the user-added repositories then no repositories nor
     // packages should stay in the database.
     //
diff --git a/bpkg/rep-remove.hxx b/bpkg/rep-remove.hxx
index 8bc6144..a7a6418 100644
--- a/bpkg/rep-remove.hxx
+++ b/bpkg/rep-remove.hxx
@@ -6,6 +6,7 @@
 #define BPKG_REP_REMOVE_HXX
 
 #include <bpkg/types.hxx>
+#include <bpkg/forward.hxx> // database, repository
 #include <bpkg/utility.hxx>
 
 #include <bpkg/rep-remove-options.hxx>
@@ -14,6 +15,37 @@ namespace bpkg
 {
   int
   rep_remove (const rep_remove_options&, cli::scanner& args);
+
+  // Remove a repository if it is not reachable from the root (and thus is not
+  // required by any user-added repository).
+  //
+  void
+  rep_remove (const dir_path& conf, database&, const shared_ptr<repository>&);
+
+  // Bring the configuration to the clean state as if repositories were added
+  // but never fetched. Leave selected packages intact. Note that it should be
+  // called out of the database transaction.
+  //
+  // Specifically:
+  //
+  // - Clean prerequisite and complement repository sets for the top-level
+  //   repositories.
+  //
+  // - Remove all repositories except the top-level ones and the root.
+  //
+  // - Remove all repository state directories (regardless of whether they
+  //   actually relate to any existing repositories).
+  //
+  // - Remove all available packages.
+  //
+  void
+  rep_remove_clean (const dir_path& conf, database&, bool quiet = true);
+
+  // Remove a repository from locations of the available packages it
+  // contains. Remove packages that come from only this repository.
+  //
+  void
+  rep_remove_package_locations (database&, const string& repository_name);
 }
 
 #endif // BPKG_REP_REMOVE_HXX
diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx
index 252bdef..d80e55e 100644
--- a/bpkg/utility.cxx
+++ b/bpkg/utility.cxx
@@ -69,7 +69,11 @@ namespace bpkg
   {
     if (!temp_dir.empty ())
     {
-      rm_r (temp_dir, true /* dir_itself */, 3, ignore_error);
+      rm_r (temp_dir,
+            true /* dir_itself */,
+            3,
+            ignore_error ? rm_error_mode::ignore : rm_error_mode::fail);
+
       temp_dir.clear ();
     }
   }
@@ -202,19 +206,24 @@ namespace bpkg
   }
 
   void
-  rm_r (const dir_path& d, bool dir, uint16_t v, bool ignore_error)
+  rm_r (const dir_path& d, bool dir, uint16_t v, rm_error_mode m)
   {
     if (verb >= v)
       text << (dir ? "rmdir -r " : "rm -r ") << (dir ? d : d / dir_path ("*"));
 
     try
     {
-      rmdir_r (d, dir, ignore_error);
+      rmdir_r (d, dir, m == rm_error_mode::ignore);
     }
     catch (const system_error& e)
     {
-      fail << "unable to remove " << (dir ? "" : "contents of ")
-           << "directory " << d << ": " << e;
+      bool w (m == rm_error_mode::warn);
+
+      (w ? warn : error) << "unable to remove " << (dir ? "" : "contents of ")
+                         << "directory " << d << ": " << e;
+
+      if (!w)
+        throw failed ();
     }
   }
 
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index b05f668..b5e05d5 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -107,11 +107,13 @@ namespace bpkg
   void
   rm (const path&, uint16_t verbosity = 3);
 
+  enum class rm_error_mode {ignore, warn, fail};
+
   void
   rm_r (const dir_path&,
         bool dir_itself = true,
         uint16_t verbosity = 3,
-        bool ignore_error = false);
+        rm_error_mode = rm_error_mode::fail);
 
   void
   mv (const dir_path& from, const dir_path& to);