From 780290277a51853b2e515b16898ca0fcfa1e9e71 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 26 Feb 2018 22:00:22 +0300 Subject: Update rep-fetch --- bpkg/fetch-git.cxx | 95 ++++++++++---- bpkg/forward.hxx | 1 + bpkg/manifest-utility.cxx | 2 +- bpkg/manifest-utility.hxx | 2 +- bpkg/package.hxx | 6 - bpkg/pkg-build.cxx | 4 +- bpkg/rep-add.cxx | 68 +++++----- bpkg/rep-add.hxx | 10 ++ bpkg/rep-fetch.cxx | 315 ++++++++++++++++++++++++++++++++++------------ bpkg/rep-fetch.hxx | 16 ++- bpkg/rep-remove.cli | 15 ++- bpkg/rep-remove.cxx | 166 ++++++++++++++++++++---- bpkg/rep-remove.hxx | 32 +++++ bpkg/utility.cxx | 19 ++- bpkg/utility.hxx | 4 +- 15 files changed, 578 insertions(+), 177 deletions(-) (limited to 'bpkg') diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index 7ffaaaf..17c56c5 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -292,7 +292,7 @@ namespace bpkg // histories we will use the file URL notation for local repositories. // static string - git_url (const repository_url& url) + to_git_url (const repository_url& url) { if (url.scheme != repository_protocol::file) return url.string (); @@ -326,6 +326,23 @@ namespace bpkg #endif } + // Create the URL object from a string representation printed by git + // commands. + // + static repository_url + from_git_url (string&& u) + { + // Fix-up the broken Windows file URL notation (see to_git_url() for + // details). + // +#ifdef _WIN32 + if (casecmp (u, "file://", 7) == 0 && u[7] != '/') + u.insert (7, 1, '/'); +#endif + + return repository_url (u); + } + // Sense the git protocol capabilities for a specified URL. // // Protocols other than HTTP(S) are considered smart but without the @@ -477,7 +494,7 @@ namespace bpkg co.git_option (), "ls-remote", "--refs", - git_url (url))); + to_git_url (url))); pipe.out.close (); // Shouldn't throw, unless something is severely damaged. @@ -798,21 +815,12 @@ namespace bpkg try { - string u (git_string (co, "submodule URL", - co.git_option (), - "-C", dir, - "config", - "--get", - "submodule." + name + ".url")); - - // Fix-up the broken Windows file URL notation (see the git_url() - // function for details). - // -#ifdef _WIN32 - if (casecmp (u, "file://", 7) == 0 && u[7] != '/') - u.insert (7, 1, '/'); -#endif - url = repository_url (u); + url = from_git_url (git_string (co, "submodule URL", + co.git_option (), + "-C", dir, + "config", + "--get", + "submodule." + name + ".url")); } catch (const invalid_argument& e) { @@ -859,7 +867,7 @@ namespace bpkg "--name", name, "--path", sdir, - "--url", git_url (url), + "--url", to_git_url (url), shallow ? cstrings ({"--depth", "1"}) : cstrings (), @@ -978,7 +986,7 @@ namespace bpkg ref.commit ? opt ("--no-checkout") : nullopt, verb < 1 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt, - git_url (url), + to_git_url (url), d)) fail << "unable to clone " << url << endg; @@ -1012,12 +1020,55 @@ namespace bpkg assert (ref.branch); - capabilities cap (sense_capabilities (co, url)); - bool shallow (shallow_fetch (co, url, cap, ref)); - dir_path d (destdir); d /= dir_path (*ref.branch); + // If the repository location differs from the one that was used to clone + // the repository then we re-clone it from the new location. + // + // Another (more hairy) way of doing this would be fixing up the remote + // origin URLs recursively prior to fetching. + // + try + { + repository_url u (from_git_url (git_string (co, "remote repository URL", + co.git_option (), + "-C", d, + "config", + "--get", + "remote.origin.url"))); + if (u != url) + { + // Note that the repository canonical name can not change under the + // legal scenarios that lead to the location change. Changed canonical + // name means that the repository was manually amended. We could + // re-clone such repositories as well but want to leave the backdoor + // for tests. + // + u.fragment = rl.url ().fragment; // Restore the fragment. + repository_location l (u, rl.type ()); + + if (rl.canonical_name () == l.canonical_name ()) + { + if (verb) + info << "re-cloning " << rl.canonical_name () + << " due to location change" << + info << "new location " << rl.url () << + info << "old location " << u; + + rm_r (d); + return git_clone (co, rl, destdir); + } + } + } + catch (const invalid_argument& e) + { + fail << "invalid remote.origin.url configuration value: " << e << endg; + } + + capabilities cap (sense_capabilities (co, url)); + bool shallow (shallow_fetch (co, url, cap, ref)); + update_tree (co, d, dir_path (), diff --git a/bpkg/forward.hxx b/bpkg/forward.hxx index cb49926..c253e72 100644 --- a/bpkg/forward.hxx +++ b/bpkg/forward.hxx @@ -14,6 +14,7 @@ namespace bpkg // // + class repository; class selected_package; } diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 1500cee..8377bab 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -68,7 +68,7 @@ namespace bpkg } repository_location - parse_location (const char* s, optional ot) + parse_location (const string& s, optional ot) try { repository_url u (s); diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index f0d5cdd..a0966d4 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -39,7 +39,7 @@ namespace bpkg // current working directory. Diagnose invalid locations and throw failed. // repository_location - parse_location (const char*, optional); + parse_location (const string&, optional); // Return the repository state subdirectory for the specified location as it // appears under .bpkg/repositories/ in the bpkg configuration. Return empty diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 54efca1..e864b59 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -244,12 +244,6 @@ namespace bpkg complements_type complements; prerequisites_type prerequisites; - // Used to detect recursive fetching. Will probably be replaced - // by the 'repositories' file timestamp or hashsum later. - // - #pragma db transient - bool fetched = false; - public: explicit repository (repository_location l): location (move (l)) diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index 384cb91..fc6ff26 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -618,7 +618,9 @@ namespace bpkg if (!ar->location.empty ()) dr << info << "repository " << ar->location << " appears to " - << "be broken"; + << "be broken" << + info << "or the repository state could be stale" << + info << "run 'bpkg rep-fetch' to update"; } // If all that's available is a stub then we need to make sure the diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx index 690b584..2dcce9a 100644 --- a/bpkg/rep-add.cxx +++ b/bpkg/rep-add.cxx @@ -15,6 +15,42 @@ using namespace butl; namespace bpkg { + shared_ptr + rep_add (database& db, const repository_location& rl) + { + const string& rn (rl.canonical_name ()); + + shared_ptr r (db.find (rn)); + + bool updated (false); + + if (r == nullptr) + { + r.reset (new repository (rl)); + db.persist (r); + } + else if (r->location.url () != rl.url ()) + { + r->location = rl; + db.update (r); + + updated = true; + } + + shared_ptr root (db.load ("")); + + bool added ( + root->complements.insert (lazy_shared_ptr (db, r)).second); + + if (added) + db.update (root); + + if (verb) + text << (added ? "added " : updated ? "updated " : "unchanged ") << rn; + + return r; + } + int rep_add (const rep_add_options& o, cli::scanner& args) { @@ -31,8 +67,6 @@ namespace bpkg transaction t (db.begin ()); session s; // Repository dependencies can have cycles. - shared_ptr root (db.load ("")); - while (args.more ()) { repository_location rl ( @@ -41,37 +75,9 @@ namespace bpkg ? optional (o.type ()) : nullopt)); - const string& rn (rl.canonical_name ()); - - // Create the new repository if it is not in the database yet. Otherwise - // update its location. Add it as a complement to the root repository (if - // it is not there yet). - // - shared_ptr r (db.find (rn)); - - bool updated (false); - - if (r == nullptr) - { - r.reset (new repository (rl)); - db.persist (r); - } - else if (r->location.url () != rl.url ()) - { - r->location = rl; - db.update (r); - - updated = true; - } - - bool added ( - root->complements.insert (lazy_shared_ptr (db, r)).second); - - if (verb) - text << (added ? "added " : updated ? "updated " : "unchanged ") << rn; + rep_add (db, rl); } - db.update (root); t.commit (); return 0; diff --git a/bpkg/rep-add.hxx b/bpkg/rep-add.hxx index 9f91713..192df05 100644 --- a/bpkg/rep-add.hxx +++ b/bpkg/rep-add.hxx @@ -5,7 +5,10 @@ #ifndef BPKG_REP_ADD_HXX #define BPKG_REP_ADD_HXX +#include + #include +#include // database, repository #include #include @@ -14,6 +17,13 @@ namespace bpkg { int rep_add (const rep_add_options&, cli::scanner& args); + + // Create the new repository if it is not in the database yet or update its + // location if it differs. Then add it as a complement to the root + // repository if it is not already. + // + shared_ptr + rep_add (database&, const repository_location&); } #endif // BPKG_REP_ADD_HXX diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 3956c5a..3bede3c 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -4,15 +4,19 @@ #include +#include + #include #include // operator<<(ostream, process_path) #include #include #include +#include #include #include #include +#include #include #include @@ -21,6 +25,22 @@ using namespace butl; namespace bpkg { + // The fetch operation failure may result in mismatch of the (rolled back) + // repository database state and the repository filesystem state. Restoring + // the filesystem state on failure would require making copies which seems + // unnecessarily pessimistic. So instead, we will revert the repository + // state to the clean state as if repositories were added but never fetched + // (see rep_remove_clean() for more details). + // + // The following flag is set by the rep_fetch_*() functions when they are + // about to change the repository filesystem state. That, in particular, + // means that the flag will be set even if the subsequent fetch operation + // fails, and so the caller can rely on it while handling the thrown + // exception. The flag must be reset by such a caller prior to the + // rep_fetch_*() call. + // + static bool filesystem_state_changed; + static rep_fetch_data rep_fetch_bpkg (const common_options& co, const dir_path* conf, @@ -157,6 +177,14 @@ namespace bpkg auto_rmdir rm (temp_dir / sd); dir_path& td (rm.path); + // We are about to modify the repository filesystem state. + // + // In the future we can probably do something smarter about the flag, + // keeping it unset unless the repository state directory is really + // changed. + // + filesystem_state_changed = true; + if (exists (td)) rm_r (td); @@ -389,21 +417,40 @@ namespace bpkg return rep_fetch_data (); } + using repositories = set>; + static void - rep_fetch (const configuration_options& co, - transaction& t, + rep_fetch (const common_options& co, + const dir_path& conf, + database& db, const shared_ptr& r, - const shared_ptr& root, - const string& reason) + repositories& fetched, + repositories& removed, + const string& reason = string ()) { tracer trace ("rep_fetch(rep)"); - database& db (t.database ()); tracer_guard tg (db, trace); + // Check that the repository is not fetched yet and register it as fetched + // otherwise. + // + // Note that we can end up with a repository dependency cycle via + // prerequisites. Thus we register the repository before recursing into its + // dependencies. + // + if (!fetched.insert (r).second) // Is already fetched. + return; + const repository_location& rl (r->location); l4 ([&]{trace << r->name << " " << rl;}); - assert (rl.absolute () || rl.remote ()); + + // Cancel the repository removal. + // + // Note that this is an optimization as the rep_remove() function checks + // for reachability of the repository being removed. + // + removed.erase (r); // The fetch_*() functions below will be quiet at level 1, which // can be quite confusing if the download hangs. @@ -414,23 +461,45 @@ namespace bpkg dr << "fetching " << r->name; - const auto& ua (root->complements); - - if (ua.find (lazy_shared_ptr (db, r)) == ua.end ()) - { - assert (!reason.empty ()); + if (!reason.empty ()) dr << " (" << reason << ")"; - } } - r->fetched = true; // Mark as being fetched. + // Register complements and prerequisites for potential removal unless + // they are fetched. Clear repository dependency sets afterwards. + // + auto remove = [&fetched, &removed] (const lazy_shared_ptr& rp) + { + shared_ptr r (rp.load ()); + if (fetched.find (r) == fetched.end ()) + removed.insert (move (r)); + }; - // Load the repositories and packages and use it to populate the + for (const lazy_shared_ptr& cr: r->complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + remove (cr); + } + + for (const lazy_weak_ptr& pr: r->prerequisites) + remove (lazy_shared_ptr (pr)); + + r->complements.clear (); + r->prerequisites.clear (); + + // Remove this repository from locations of the available packages it + // contains. + // + rep_remove_package_locations (db, r->name); + + // Load the repository and package manifests and use them to populate the // prerequisite and complement repository sets as well as available // packages. // - rep_fetch_data rfd ( - rep_fetch (co, &co.directory (), rl, true /* ignore_unknow */)); + rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */)); for (repository_manifest& rm: rfd.repositories) { @@ -439,51 +508,53 @@ namespace bpkg if (rr == repository_role::base) continue; // Entry for this repository. + repository_location& l (rm.location); + // If the location is relative, complete it using this repository // as a base. // - if (rm.location.relative ()) + if (l.relative ()) { try { - rm.location = repository_location (rm.location, rl); + l = repository_location (l, rl); } catch (const invalid_argument& e) { - fail << "invalid relative repository location '" << rm.location + fail << "invalid relative repository location '" << l << "': " << e << info << "base repository location is " << rl; } } - // We might already have this repository in the database. + // Create the new repository if it is not in the database yet. Otherwise + // update its location. // - shared_ptr pr ( - db.find ( - rm.location.canonical_name ())); + shared_ptr pr (db.find (l.canonical_name ())); if (pr == nullptr) { - pr = make_shared (move (rm.location)); + pr = make_shared (move (l)); db.persist (pr); // Enter into session, important if recursive. } + else if (pr->location.url () != l.url ()) + { + pr->location = move (l); + db.update (r); + } - // Load the prerequisite repository unless it has already been - // (or is already being) fetched. + // Load the prerequisite repository. // - if (!pr->fetched) + string reason; + switch (rr) { - string reason; - switch (rr) - { - case repository_role::complement: reason = "complements "; break; - case repository_role::prerequisite: reason = "prerequisite of "; break; - case repository_role::base: assert (false); - } - reason += r->name; - - rep_fetch (co, t, pr, root, reason); + case repository_role::complement: reason = "complements "; break; + case repository_role::prerequisite: reason = "prerequisite of "; break; + case repository_role::base: assert (false); } + reason += r->name; + + rep_fetch (co, conf, db, pr, fetched, removed, reason); // @@ What if we have duplicated? Ideally, we would like to check // this once and as early as possible. The original idea was to @@ -530,7 +601,11 @@ namespace bpkg if (rl.type () == repository_type::git && r->complements.empty () && r->prerequisites.empty ()) - r->complements.insert (lazy_shared_ptr (db, root)); + r->complements.insert (lazy_shared_ptr (db, string ())); + + // Save the changes to the repository object. + // + db.update (r); // "Suspend" session while persisting packages to reduce memory // consumption. @@ -600,70 +675,156 @@ namespace bpkg } session::current (s); // "Resume". + } - // Save the changes to the repository object. + static void + rep_fetch (const common_options& o, + const dir_path& conf, + transaction& t, + const vector>& repos) + { + database& db (t.database ()); + + // As a fist step we fetch repositories recursively building the list of + // the former prerequisites and complements to be considered for removal. // - db.update (r); + // We delay the actual removal until we fetch all the required repositories + // as a dependency dropped by one repository can appear for another one. + // + try + { + // If fetch fails and the repository filesystem state is changed, then + // the configuration is broken, and we have to take some drastic + // measures (see below). + // + filesystem_state_changed = false; + + repositories fetched; + repositories removed; + + for (const lazy_shared_ptr& r: repos) + rep_fetch (o, conf, db, r.load (), fetched, removed); + + // Finally, remove dangling repositories. + // + for (const shared_ptr& r: removed) + rep_remove (conf, db, r); + } + catch (const failed&) + { + t.rollback (); + + if (filesystem_state_changed) + { + // Warn prior to the cleanup operation that potentially can also fail. + // Note that we assume that the diagnostics has already been issued. + // + warn << "repository state is now broken and will be cleaned up" << + info << "run 'bpkg rep-fetch' to update"; + + rep_remove_clean (conf, db); + } + + throw; + } + } + + void + rep_fetch (const common_options& o, + const dir_path& conf, + database& db, + const vector& rls) + { + vector> repos; + repos.reserve (rls.size ()); + + transaction t (db.begin ()); + + shared_ptr root (db.load ("")); + repository::complements_type& ua (root->complements); // User-added repos. + + for (const repository_location& rl: rls) + { + lazy_shared_ptr r (db, rl.canonical_name ()); + + // Add the repository, unless it is already a top-level one and has the + // same location. + // + if (ua.find (r) == ua.end () || r.load ()->location.url () != rl.url ()) + rep_add (db, rl); + + repos.emplace_back (r); + } + + rep_fetch (o, conf, t, repos); + + t.commit (); } int - rep_fetch (const rep_fetch_options& o, cli::scanner&) + rep_fetch (const rep_fetch_options& o, cli::scanner& args) { tracer trace ("rep_fetch"); dir_path c (o.directory ()); l4 ([&]{trace << "configuration: " << c;}); + // Build the list of repositories the user wants to fetch. + // + vector> repos; + database db (open (c, trace)); transaction t (db.begin ()); session s; // Repository dependencies can have cycles. shared_ptr root (db.load ("")); - const auto& ua (root->complements); // User-added repositories. + repository::complements_type& ua (root->complements); // User-added repos. - if (ua.empty ()) - fail << "configuration " << c << " has no repositories" << - info << "use 'bpkg rep-add' to add a repository"; - - // Clean repositories and available packages. At the end only - // repositories that were explicitly added by the user and the - // special root repository should remain. - // - db.erase_query (); + if (!args.more ()) + { + if (ua.empty ()) + fail << "configuration " << c << " has no repositories" << + info << "use 'bpkg rep-add' to add a repository"; - for (shared_ptr r: pointer_result (db.query ())) + for (const lazy_shared_ptr& r: ua) + repos.push_back (r); + } + else { - if (r == root) - { - l5 ([&]{trace << "skipping root";}); - } - else if (ua.find (lazy_shared_ptr (db, r)) != ua.end ()) + while (args.more ()) { - l4 ([&]{trace << "cleaning " << r->name;}); + // Try to map the argument to a user-added repository. + // + // If this is a repository name then it must be present in the + // configuration. If this is a repository location then we add it to + // the configuration. + // + lazy_shared_ptr r; + string a (args.next ()); - r->complements.clear (); - r->prerequisites.clear (); - r->fetched = false; - db.update (r); - } - else - { - l4 ([&]{trace << "erasing " << r->name;}); - db.erase (r); - } - } + if (repository_name (a)) + { + lazy_shared_ptr rp (db, a); - // Now recursively fetch prerequisite/complement repositories and - // their packages. - // - for (const lazy_shared_ptr& lp: ua) - { - shared_ptr r (lp.load ()); + if (ua.find (rp) != ua.end ()) + r = move (rp); + else + fail << "repository '" << a << "' does not exist in this " + << "configuration"; + } + else + //@@ TODO: check if exists in root & same location and avoid + // calling rep_add. Get rid of quiet mode. + // + r = lazy_shared_ptr ( + db, rep_add (db, parse_location (a, nullopt /* type */))); - if (!r->fetched) // Can already be loaded as a prerequisite/complement. - rep_fetch (o, t, r, root, ""); // No reason (user-added). + repos.emplace_back (move (r)); + } } + rep_fetch (o, c, t, repos); + size_t rcount (0), pcount (0); if (verb) { diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 0a7cacd..b26971c 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -8,6 +8,7 @@ #include #include +#include // database #include #include @@ -44,10 +45,21 @@ namespace bpkg }; rep_fetch_data - rep_fetch (const common_options& co, + rep_fetch (const common_options&, const dir_path* conf, - const repository_location& rl, + const repository_location&, bool ignore_unknown); + + // Add (or update) repository locations to the configuration and fetch + // them. On failure clean up the configuration (see rep_remove_clean() for + // details). Note that it starts a new transaction and should be called in + // session. + // + void + rep_fetch (const common_options&, + const dir_path& conf, + database&, + const vector&); } #endif // BPKG_REP_FETCH_HXX diff --git a/bpkg/rep-remove.cli b/bpkg/rep-remove.cli index 3662b2b..25280b2 100644 --- a/bpkg/rep-remove.cli +++ b/bpkg/rep-remove.cli @@ -17,8 +17,9 @@ namespace bpkg "\h|SYNOPSIS| - \c{\b{bpkg rep-remove}|\b{remove} [] (|)... | - (--all|-a)} + \c{\b{bpkg rep-remove}|\b{remove} [] (|)...}\n + \c{\b{bpkg rep-remove}|\b{remove} [] (--all|-a)}\n + \c{\b{bpkg rep-remove}|\b{remove} [] --clean} \h|DESCRIPTION| @@ -29,7 +30,10 @@ namespace bpkg Alternatively, the \cb{--all|-a} option can be used to remove all the repositories that were previously added (\l{bpkg-rep-add(1)}) to the - configuration." + configuration. + + Finally, the \cb{--clean} option can be used to revert the repositories + to the clean state, as if they were added but never fetched." } class rep_remove_options: configuration_options @@ -40,5 +44,10 @@ namespace bpkg { "Remove all the repositories." } + + bool --clean + { + "Clean the repository state." + } }; } diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx index d561c0c..df9acde 100644 --- a/bpkg/rep-remove.cxx +++ b/bpkg/rep-remove.cxx @@ -7,6 +7,8 @@ #include #include // find() +#include // dir_iterator + #include #include #include @@ -14,6 +16,7 @@ #include using namespace std; +using namespace butl; namespace bpkg { @@ -68,30 +71,18 @@ namespace bpkg return reachable (db, r, traversed); } - // Remove a repository if it is not reachable from the root (and thus is not - // required by any user-added repository). - // - static void - rep_remove (const dir_path& c, database& db, const shared_ptr& r) + void + rep_remove_package_locations (database& db, const string& name) { - const string& nm (r->name); - assert (!nm.empty ()); // Can't be the root repository. - - if (reachable (db, r)) - return; - - // Remove this repository from locations of the available packages it - // contains. Remove packages that come from only this repository. - // for (const auto& rp: db.query ( - query::repository::name == nm)) + query::repository::name == name)) { const shared_ptr& p (rp); vector& ls (p->locations); for (auto i (ls.cbegin ()), e (ls.cend ()); i != e; ++i) { - if (i->repository.object_id () == nm) + if (i->repository.object_id () == name) { ls.erase (i); break; @@ -103,6 +94,33 @@ namespace bpkg else db.update (p); } + } + + // Remove a directory moving it to the temporary directory first, increasing + // the chances for the operation to succeed. + // + static void + rmdir (const dir_path& d) + { + dir_path td (temp_dir / d.leaf ()); + + if (exists (td)) + rm_r (td); + + mv (d, td); + rm_r (td, true /* dir_itself */, 3, rm_error_mode::warn); + } + + void + rep_remove (const dir_path& c, database& db, const shared_ptr& r) + { + const string& nm (r->name); + assert (!nm.empty ()); // Can't be the root repository. + + if (reachable (db, r)) + return; + + rep_remove_package_locations (db, nm); // Cleanup the repository state if present. // @@ -122,7 +140,7 @@ namespace bpkg dir_path sd (c / repos_dir / d); if (exists (sd)) - rm_r (sd); + rmdir (sd); } // Note that it is essential to erase the repository object from the @@ -157,6 +175,72 @@ namespace bpkg remove (lazy_shared_ptr (pr)); } + void + rep_remove_clean (const dir_path& c, + database& db, + bool quiet) + { + tracer trace ("rep_remove_clean"); + + assert (!transaction::has_current ()); + + // Clean repositories and available packages. At the end only repositories + // that were explicitly added by the user and the special root repository + // should remain. + // + { + // Note that we don't rely on being in session nor create one. + // + transaction t (db.begin ()); + + db.erase_query (); + + shared_ptr root (db.load ("")); + repository::complements_type& ua (root->complements); + + for (shared_ptr r: pointer_result (db.query ())) + { + if (r->name == "") + { + l5 ([&]{trace << "skipping root";}); + } + else if (ua.find (lazy_shared_ptr (db, r)) != ua.end ()) + { + r->complements.clear (); + r->prerequisites.clear (); + db.update (r); + + if (verb >= (quiet ? 2 : 1)) + text << "cleaned " << r->name; + } + else + { + l4 ([&]{trace << "erasing " << r->name;}); + db.erase (r); + } + } + + t.commit (); + } + + // Remove repository state subdirectories. + // + dir_path rd (c / repos_dir); + + try + { + for (const dir_entry& de: dir_iterator (rd)) // system_error + { + if (de.ltype () == entry_type::directory) + rmdir (rd / path_cast (de.path ())); + } + } + catch (const system_error& e) + { + fail << "unable to scan directory " << rd << ": " << e; + } + } + int rep_remove (const rep_remove_options& o, cli::scanner& args) { @@ -167,21 +251,44 @@ namespace bpkg // Check that options and arguments are consistent. // - if (o.all ()) { - if (args.more ()) - fail << "both --all|-a and repository name specified" << - info << "run 'bpkg help rep-remove' for more information"; + diag_record dr; + + if (o.clean ()) + { + if (o.all ()) + dr << fail << "both --clean and --all|-a specified"; + else if (args.more ()) + dr << fail << "both --clean and repository argument specified"; + } + else if (o.all ()) + { + if (args.more ()) + dr << fail << "both --all|-a and repository argument specified"; + } + else if (!args.more ()) + dr << fail << "repository name or location argument expected"; + + if (!dr.empty ()) + dr << info << "run 'bpkg help rep-remove' for more information"; } - else if (!args.more ()) - fail << "repository name or location argument expected" << - info << "run 'bpkg help rep-remove' for more information"; + database db (open (c, trace)); + + // Clean the configuration if requested. + // + if (o.clean ()) + { + rep_remove_clean (c, db, false /* quiet */); + return 0; + } + + // Remove the specified repositories. + // // Build the list of repositories the user wants removed. // vector> repos; - database db (open (c, trace)); transaction t (db.begin ()); session s; // Repository dependencies can have cycles. @@ -259,14 +366,14 @@ namespace bpkg // Note that for efficiency we un-reference all the top-level repositories // before starting to delete them. // - for (lazy_shared_ptr& r: repos) + for (const lazy_shared_ptr& r: repos) ua.erase (r); db.update (root); // Remove the dangling repositories from the database, recursively. // - for (lazy_shared_ptr& r: repos) + for (const lazy_shared_ptr& r: repos) { rep_remove (c, db, r.load ()); @@ -274,6 +381,11 @@ namespace bpkg text << "removed " << r.object_id (); } + // If the --all option is specified then no user-added repositories should + // remain. + // + assert (!o.all () || ua.empty ()); + // If we removed all the user-added repositories then no repositories nor // packages should stay in the database. // diff --git a/bpkg/rep-remove.hxx b/bpkg/rep-remove.hxx index 8bc6144..a7a6418 100644 --- a/bpkg/rep-remove.hxx +++ b/bpkg/rep-remove.hxx @@ -6,6 +6,7 @@ #define BPKG_REP_REMOVE_HXX #include +#include // database, repository #include #include @@ -14,6 +15,37 @@ namespace bpkg { int rep_remove (const rep_remove_options&, cli::scanner& args); + + // Remove a repository if it is not reachable from the root (and thus is not + // required by any user-added repository). + // + void + rep_remove (const dir_path& conf, database&, const shared_ptr&); + + // Bring the configuration to the clean state as if repositories were added + // but never fetched. Leave selected packages intact. Note that it should be + // called out of the database transaction. + // + // Specifically: + // + // - Clean prerequisite and complement repository sets for the top-level + // repositories. + // + // - Remove all repositories except the top-level ones and the root. + // + // - Remove all repository state directories (regardless of whether they + // actually relate to any existing repositories). + // + // - Remove all available packages. + // + void + rep_remove_clean (const dir_path& conf, database&, bool quiet = true); + + // Remove a repository from locations of the available packages it + // contains. Remove packages that come from only this repository. + // + void + rep_remove_package_locations (database&, const string& repository_name); } #endif // BPKG_REP_REMOVE_HXX diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx index 252bdef..d80e55e 100644 --- a/bpkg/utility.cxx +++ b/bpkg/utility.cxx @@ -69,7 +69,11 @@ namespace bpkg { if (!temp_dir.empty ()) { - rm_r (temp_dir, true /* dir_itself */, 3, ignore_error); + rm_r (temp_dir, + true /* dir_itself */, + 3, + ignore_error ? rm_error_mode::ignore : rm_error_mode::fail); + temp_dir.clear (); } } @@ -202,19 +206,24 @@ namespace bpkg } void - rm_r (const dir_path& d, bool dir, uint16_t v, bool ignore_error) + rm_r (const dir_path& d, bool dir, uint16_t v, rm_error_mode m) { if (verb >= v) text << (dir ? "rmdir -r " : "rm -r ") << (dir ? d : d / dir_path ("*")); try { - rmdir_r (d, dir, ignore_error); + rmdir_r (d, dir, m == rm_error_mode::ignore); } catch (const system_error& e) { - fail << "unable to remove " << (dir ? "" : "contents of ") - << "directory " << d << ": " << e; + bool w (m == rm_error_mode::warn); + + (w ? warn : error) << "unable to remove " << (dir ? "" : "contents of ") + << "directory " << d << ": " << e; + + if (!w) + throw failed (); } } diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx index b05f668..b5e05d5 100644 --- a/bpkg/utility.hxx +++ b/bpkg/utility.hxx @@ -107,11 +107,13 @@ namespace bpkg void rm (const path&, uint16_t verbosity = 3); + enum class rm_error_mode {ignore, warn, fail}; + void rm_r (const dir_path&, bool dir_itself = true, uint16_t verbosity = 3, - bool ignore_error = false); + rm_error_mode = rm_error_mode::fail); void mv (const dir_path& from, const dir_path& to); -- cgit v1.1