From 4fcd32b536f3d29755b1fecc7e3f06be64f996ca Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sat, 24 Feb 2018 18:21:39 +0300 Subject: Add support for rep-list and rep-remove, update rep-add --- bpkg/bpkg.cxx | 4 + bpkg/buildfile | 8 +- bpkg/manifest-utility.cxx | 26 +++++ bpkg/manifest-utility.hxx | 5 + bpkg/package.hxx | 55 +++++++++ bpkg/rep-add.cxx | 62 +++++----- bpkg/rep-info.cxx | 2 +- bpkg/rep-list.cxx | 124 ++++++++++++++++++++ bpkg/rep-list.hxx | 19 +++ bpkg/rep-remove.cxx | 288 ++++++++++++++++++++++++++++++++++++++++++++++ bpkg/rep-remove.hxx | 19 +++ bpkg/types.hxx | 5 + 12 files changed, 588 insertions(+), 29 deletions(-) create mode 100644 bpkg/rep-list.cxx create mode 100644 bpkg/rep-list.hxx create mode 100644 bpkg/rep-remove.cxx create mode 100644 bpkg/rep-remove.hxx (limited to 'bpkg') diff --git a/bpkg/bpkg.cxx b/bpkg/bpkg.cxx index 9606b66..457df93 100644 --- a/bpkg/bpkg.cxx +++ b/bpkg/bpkg.cxx @@ -43,6 +43,8 @@ #include #include #include +#include +#include using namespace std; using namespace bpkg; @@ -301,6 +303,8 @@ try REP_COMMAND (create); REP_COMMAND (fetch); REP_COMMAND (info); + REP_COMMAND (list); + REP_COMMAND (remove); assert (false); fail << "unhandled command"; diff --git a/bpkg/buildfile b/bpkg/buildfile index 9bdf29b..f25e08e 100644 --- a/bpkg/buildfile +++ b/bpkg/buildfile @@ -38,7 +38,9 @@ pkg-verify-options \ rep-add-options \ rep-create-options \ rep-fetch-options \ -rep-info-options +rep-info-options \ +rep-list-options \ +rep-remove-options help_topics = repository-signing @@ -96,9 +98,11 @@ if $cli.configured # rep-* command. # cli.cxx{rep-add-options}: cli{rep-add} + cli.cxx{rep-create-options}: cli{rep-create} cli.cxx{rep-fetch-options}: cli{rep-fetch} cli.cxx{rep-info-options}: cli{rep-info} - cli.cxx{rep-create-options}: cli{rep-create} + cli.cxx{rep-list-options}: cli{rep-list} + cli.cxx{rep-remove-options}: cli{rep-remove} # Help topics. # diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index ed50298..1500cee 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -142,4 +142,30 @@ namespace bpkg assert (false); // Can't be here. return dir_path (); } + + bool + repository_name (const string& s) + { + size_t n (s.size ()); + size_t p (s.find (':')); + + // If it has no scheme or starts with the URL scheme (followed by ://) then + // this is not a canonical name. + // + if (p == string::npos || (p + 2 < n && s[p + 1] == '/' && s[p + 2] == '/')) + return false; + + // This is a canonical name if the scheme is convertible to the repository + // type. + // + try + { + to_repository_type (string (s, 0, p)); + return true; + } + catch (const invalid_argument&) + { + return false; + } + } } diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index f672ba7..f0d5cdd 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -53,6 +53,11 @@ namespace bpkg // dir_path repository_state (const repository_location&); + + // Return true if the argument is a valid repository canonical name. + // + bool + repository_name (const string&); } #endif // BPKG_MANIFEST_UTILITY_HXX diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 23fe9b2..54efca1 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -761,6 +761,61 @@ namespace bpkg optional constraint; }; + // Return a list of repositories that depend on this repository as a + // complement. + // + // Note that the last db object pragma is required to produce an object + // loading view. + // + #pragma db view object(repository = complement) \ + table("repository_complements" = "rc" inner: \ + "rc.complement = " + complement::name) \ + object(repository inner: "rc.repository = " + repository::name) + struct repository_complement_dependent + { + shared_ptr object; + + operator const shared_ptr () const {return object;} + }; + + // Return a list of repositories that depend on this repository as a + // prerequisite. + // + // Note that the last db object pragma is required to produce an object + // loading view. + // + #pragma db view object(repository = prerequisite) \ + table("repository_prerequisites" = "rp" inner: \ + "rp.prerequisite = " + prerequisite::name) \ + object(repository inner: "rp.repository = " + repository::name) + struct repository_prerequisite_dependent + { + shared_ptr object; + + operator const shared_ptr () const {return object;} + }; + + // Return a list of packages available from this repository. + // + #pragma db view object(repository) \ + table("available_package_locations" = "pl" inner: \ + "pl.repository = " + repository::name) \ + object(available_package inner: \ + "pl.name = " + available_package::id.name + "AND" + \ + "pl.version_epoch = " + \ + available_package::id.version.epoch + "AND" + \ + "pl.version_canonical_upstream = " + \ + available_package::id.version.canonical_upstream + "AND" + \ + "pl.version_canonical_release = " + \ + available_package::id.version.canonical_release + "AND" + \ + "pl.version_revision = " + \ + available_package::id.version.revision) + struct repository_package + { + shared_ptr object; + + operator const shared_ptr () const {return object;} + }; // Version comparison operators. // diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx index 7adb843..690b584 100644 --- a/bpkg/rep-add.cxx +++ b/bpkg/rep-add.cxx @@ -27,43 +27,53 @@ namespace bpkg fail << "repository location argument expected" << info << "run 'bpkg help rep-add' for more information"; - repository_location rl ( - parse_location (args.next (), - o.type_specified () - ? optional (o.type ()) - : nullopt)); - - const string& rn (rl.canonical_name ()); - - // Create the new repository and add is as a complement to the root. - // database db (open (c, trace)); transaction t (db.begin ()); session s; // Repository dependencies can have cycles. - // It is possible that this repository is already in the database. - // For example, it might be a prerequisite of one of the already - // added repository. - // - shared_ptr r (db.find (rl.canonical_name ())); + shared_ptr root (db.load ("")); - if (r == nullptr) + while (args.more ()) { - r.reset (new repository (rl)); - db.persist (r); + repository_location rl ( + parse_location (args.next (), + o.type_specified () + ? optional (o.type ()) + : nullopt)); + + const string& rn (rl.canonical_name ()); + + // Create the new repository if it is not in the database yet. Otherwise + // update its location. Add it as a complement to the root repository (if + // it is not there yet). + // + shared_ptr r (db.find (rn)); + + bool updated (false); + + if (r == nullptr) + { + r.reset (new repository (rl)); + db.persist (r); + } + else if (r->location.url () != rl.url ()) + { + r->location = rl; + db.update (r); + + updated = true; + } + + bool added ( + root->complements.insert (lazy_shared_ptr (db, r)).second); + + if (verb) + text << (added ? "added " : updated ? "updated " : "unchanged ") << rn; } - shared_ptr root (db.load ("")); - - if (!root->complements.insert (lazy_shared_ptr (db, r)).second) - fail << rn << " is already a repository of this configuration"; - db.update (root); t.commit (); - if (verb) - text << "added repository " << rn; - return 0; } } diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx index 467c005..0e7f911 100644 --- a/bpkg/rep-info.cxx +++ b/bpkg/rep-info.cxx @@ -4,7 +4,7 @@ #include -#include // cout +#include // cout #include // sha256_to_fingerprint() #include diff --git a/bpkg/rep-list.cxx b/bpkg/rep-list.cxx new file mode 100644 index 0000000..f2aca2c --- /dev/null +++ b/bpkg/rep-list.cxx @@ -0,0 +1,124 @@ +// file : bpkg/rep-list.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include // cout + +#include +#include +#include +#include + +using namespace std; + +namespace bpkg +{ + // Print the repository dependencies, recursively. + // + // Each line has the following form: + // + // [(complement|prerequisite) ] + // + // and is indented with 2 additional spaces for each recursion level. + // + // Note that we can end up with a repository dependency cycle via + // prerequisites. Thus we need to make sure that the repository is not in + // the dependency chain yet. + // + using repositories = set>, + compare_reference_target>; + + static void + print_dependencies (const rep_list_options& o, + const shared_ptr& r, + string& indent, + repositories& chain) + { + assert (!r->name.empty ()); // Can't be the root repository. + + if (!chain.insert (r).second) // Is already in the chain. + return; + + indent += " "; + + if (o.complements ()) + { + for (const lazy_shared_ptr& rp: r->complements) + { + // Skip the root complement (see rep_fetch() for details). + // + if (rp.object_id () == "") + continue; + + shared_ptr r (rp.load ()); + + cout << indent << "complement " + << r->location.canonical_name () << " " << r->location << endl; + + print_dependencies (o, r, indent, chain); + } + } + + if (o.prerequisites ()) + { + for (const lazy_weak_ptr& rp: r->prerequisites) + { + shared_ptr r (rp.load ()); + + cout << indent << "prerequisite " + << r->location.canonical_name () << " " << r->location << endl; + + print_dependencies (o, r, indent, chain); + } + } + + indent.pop_back (); + indent.pop_back (); + + chain.erase (r); + } + + static inline void + print_dependencies (const rep_list_options& o, + const shared_ptr& r) + { + string indent; + repositories chain; + print_dependencies (o, r, indent, chain); + } + + int + rep_list (const rep_list_options& o, cli::scanner& args) + { + tracer trace ("rep_list"); + + dir_path c (o.directory ()); + l4 ([&]{trace << "configuration: " << c;}); + + if (args.more ()) + fail << "unexpected argument '" << args.next () << "'" << + info << "run 'bpkg help rep-list' for more information"; + + database db (open (c, trace)); + transaction t (db.begin ()); + session s; // Repository dependencies can have cycles. + + shared_ptr root (db.load ("")); + + for (const lazy_shared_ptr& rp: root->complements) + { + shared_ptr r (rp.load ()); + cout << r->location.canonical_name () << " " << r->location << endl; + + if (o.complements () || o.prerequisites ()) + print_dependencies (o, r); + } + + t.commit (); + + return 0; + } +} diff --git a/bpkg/rep-list.hxx b/bpkg/rep-list.hxx new file mode 100644 index 0000000..f0d4bab --- /dev/null +++ b/bpkg/rep-list.hxx @@ -0,0 +1,19 @@ +// file : bpkg/rep-list.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_REP_LIST_HXX +#define BPKG_REP_LIST_HXX + +#include +#include + +#include + +namespace bpkg +{ + int + rep_list (const rep_list_options&, cli::scanner& args); +} + +#endif // BPKG_REP_LIST_HXX diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx new file mode 100644 index 0000000..d561c0c --- /dev/null +++ b/bpkg/rep-remove.cxx @@ -0,0 +1,288 @@ +// file : bpkg/rep-remove.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include // find() + +#include +#include +#include +#include +#include + +using namespace std; + +namespace bpkg +{ + // Return true if the repository is reachable from the root repository via + // the complements or prerequisites chains, recursively. + // + // Note that we can end up with a repository dependency cycle via + // prerequisites. Thus we need to make sure that the repository was not + // traversed yet. + // + using repositories = set>, + compare_reference_target>; + + static bool + reachable (database& db, + const shared_ptr& r, + repositories& traversed) + { + const string& nm (r->name); + assert (!nm.empty ()); // Can't be the root repository. + + // We will go upstream until reach the root or traverse through all of the + // dependent repositories. + // + if (!traversed.insert (r).second) // We have already been here. + return false; + + for (const auto& rc: db.query ( + query::complement::name == nm)) + { + const shared_ptr& r (rc); + if (r->name.empty () /* Root? */ || reachable (db, r, traversed)) + return true; + } + + for (const auto& rd: db.query ( + query::prerequisite::name == nm)) + { + // Note that the root repository has no prerequisites. + // + if (reachable (db, rd, traversed)) + return true; + } + + return false; + } + + static inline bool + reachable (database& db, const shared_ptr& r) + { + repositories traversed; + return reachable (db, r, traversed); + } + + // Remove a repository if it is not reachable from the root (and thus is not + // required by any user-added repository). + // + static void + rep_remove (const dir_path& c, database& db, const shared_ptr& r) + { + const string& nm (r->name); + assert (!nm.empty ()); // Can't be the root repository. + + if (reachable (db, r)) + return; + + // Remove this repository from locations of the available packages it + // contains. Remove packages that come from only this repository. + // + for (const auto& rp: db.query ( + query::repository::name == nm)) + { + const shared_ptr& p (rp); + vector& ls (p->locations); + + for (auto i (ls.cbegin ()), e (ls.cend ()); i != e; ++i) + { + if (i->repository.object_id () == nm) + { + ls.erase (i); + break; + } + } + + if (ls.empty ()) + db.erase (p); + else + db.update (p); + } + + // Cleanup the repository state if present. + // + // Note that this step is irreversible on failure. If something goes wrong + // we will end up with a state-less fetched repository and the + // configuration will be broken. Though, this in unlikely to happen, so + // we will not bother for now. + // + // An alternative approach would be to collect all such directories and + // then remove them after committing the transaction. Though, we still may + // fail in the middle due to the filesystem error. + // + dir_path d (repository_state (r->location)); + + if (!d.empty ()) + { + dir_path sd (c / repos_dir / d); + + if (exists (sd)) + rm_r (sd); + } + + // Note that it is essential to erase the repository object from the + // database prior to its complements and prerequisites removal as they + // must be un-referenced first. + // + db.erase (r); + + // Remove dangling complements and prerequisites. + // + // Prior to removing a prerequisite/complement we need to make sure it + // still exists, which may not be the case due to the dependency cycle. + // + auto remove = [&c, &db] (const lazy_shared_ptr& rp) + { + shared_ptr r (db.find (rp.object_id ())); + + if (r) + rep_remove (c, db, r); + }; + + for (const lazy_shared_ptr& cr: r->complements) + { + // Remove the complement unless it is the root repository (see + // rep_fetch() for details). + // + if (cr.object_id () != "") + remove (cr); + } + + for (const lazy_weak_ptr& pr: r->prerequisites) + remove (lazy_shared_ptr (pr)); + } + + int + rep_remove (const rep_remove_options& o, cli::scanner& args) + { + tracer trace ("rep_remove"); + + dir_path c (o.directory ()); + l4 ([&]{trace << "configuration: " << c;}); + + // Check that options and arguments are consistent. + // + if (o.all ()) + { + if (args.more ()) + fail << "both --all|-a and repository name specified" << + info << "run 'bpkg help rep-remove' for more information"; + } + else if (!args.more ()) + fail << "repository name or location argument expected" << + info << "run 'bpkg help rep-remove' for more information"; + + // Build the list of repositories the user wants removed. + // + vector> repos; + + database db (open (c, trace)); + transaction t (db.begin ()); + session s; // Repository dependencies can have cycles. + + shared_ptr root (db.load ("")); + repository::complements_type& ua (root->complements); + + if (o.all ()) + { + for (const lazy_shared_ptr& r: ua) + repos.push_back (r); + } + else + { + while (args.more ()) + { + // Try to map the argument to a user-added repository. + // + lazy_shared_ptr r; + string a (args.next ()); + + if (repository_name (a)) + { + lazy_shared_ptr rp (db, a); + + // Note: we report repositories we could not find for both cases + // below. + // + if (ua.find (rp) != ua.end ()) + r = move (rp); + } + else + { + // Note that we can't obtain the canonical name by creating the + // repository location object as that would require the repository + // type, which is potentially impossible to guess at this stage. So + // lets just construct the repository URL and search for it among + // user-added repositories. The linear search should be fine as we + // don't expect too many of them. + // + try + { + repository_url u (a); + + if (u.empty ()) + fail << "empty repository location"; + + for (const lazy_shared_ptr& rp: ua) + { + if (rp.load ()->location.url () == u) + { + r = rp; + break; + } + } + } + catch (const invalid_argument& e) + { + fail << "invalid repository location '" << a << "': " << e; + } + } + + if (r == nullptr) + fail << "repository '" << a << "' does not exist in this " + << "configuration"; + + // Suppress duplicates. + // + if (find (repos.begin (), repos.end (), r) == repos.end ()) + repos.emplace_back (move (r)); + } + } + + // Remove the repository references from the root. + // + // Note that for efficiency we un-reference all the top-level repositories + // before starting to delete them. + // + for (lazy_shared_ptr& r: repos) + ua.erase (r); + + db.update (root); + + // Remove the dangling repositories from the database, recursively. + // + for (lazy_shared_ptr& r: repos) + { + rep_remove (c, db, r.load ()); + + if (verb) + text << "removed " << r.object_id (); + } + + // If we removed all the user-added repositories then no repositories nor + // packages should stay in the database. + // + assert (!ua.empty () || + (db.query_value () == 0 && + db.query_value () == 0)); + + t.commit (); + + return 0; + } +} diff --git a/bpkg/rep-remove.hxx b/bpkg/rep-remove.hxx new file mode 100644 index 0000000..8bc6144 --- /dev/null +++ b/bpkg/rep-remove.hxx @@ -0,0 +1,19 @@ +// file : bpkg/rep-remove.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_REP_REMOVE_HXX +#define BPKG_REP_REMOVE_HXX + +#include +#include + +#include + +namespace bpkg +{ + int + rep_remove (const rep_remove_options&, cli::scanner& args); +} + +#endif // BPKG_REP_REMOVE_HXX diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 82a6ac9..0433e92 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -23,6 +23,7 @@ #include #include +#include // compare_reference_target #include #include @@ -64,6 +65,10 @@ namespace bpkg using std::system_error; using io_error = std::ios_base::failure; + // + // + using butl::compare_reference_target; + // // using butl::optional; -- cgit v1.1