aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bpkg/package.hxx24
-rw-r--r--bpkg/pkg-build.cli6
-rw-r--r--bpkg/pkg-build.cxx2
-rw-r--r--bpkg/pkg-configure.cxx13
-rw-r--r--bpkg/rep-fetch.cli7
-rw-r--r--bpkg/rep-fetch.cxx159
-rw-r--r--bpkg/rep-fetch.hxx9
-rw-r--r--tests/rep-fetch.test64
8 files changed, 200 insertions, 84 deletions
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index 3567b10..5905dd2 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -597,11 +597,12 @@ namespace bpkg
bool purge_src;
// Path to the output directory of this package, if any. It is
- // always relative to the configuration directory and currently
- // is always <name>-<version>. It is only set once the package
- // is configured and its main purse is to keep track of what
- // needs to be cleaned by the user before a broken package can
- // be purged. Note that it could be the same as out_root.
+ // always relative to the configuration directory, and is <name>
+ // for external packages and <name>-<version> for others. It is
+ // only set once the package is configured and its main purse is
+ // to keep track of what needs to be cleaned by the user before
+ // a broken package can be purged. Note that it could be the
+ // same as src_root.
//
optional<dir_path> out_root;
@@ -618,6 +619,19 @@ namespace bpkg
return substate == package_substate::system;
}
+ bool
+ external () const
+ {
+ return
+ // pkg-unpack <name>/<version>
+ //
+ (!repository.empty () && repository.directory_based ()) ||
+
+ // pkg-unpack --existing <dir>
+ //
+ (repository.empty () && !archive);
+ }
+
// Represent the wildcard version with the "*" string. Represent naturally
// all other versions.
//
diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli
index 3c5568f..a4d49ef 100644
--- a/bpkg/pkg-build.cli
+++ b/bpkg/pkg-build.cli
@@ -141,5 +141,11 @@ namespace bpkg
"Print to \cb{STDOUT} what would be done without actually doing
anything."
}
+
+ bool --fetch-shallow
+ {
+ "Do not re-fetch complement and prerequisite repositories. Refer to
+ the \cb{--shallow} option in \l{bpkg-rep-fetch(1)} for details."
+ }
};
}
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index cc14f1e..8f80ff5 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -1178,7 +1178,7 @@ namespace bpkg
}
if (!locations.empty ())
- rep_fetch (o, c, db, locations);
+ rep_fetch (o, c, db, locations, o.fetch_shallow ());
// Expand <packages>@<location> arguments.
//
diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx
index c839d42..aaa426f 100644
--- a/bpkg/pkg-configure.cxx
+++ b/bpkg/pkg-configure.cxx
@@ -127,16 +127,9 @@ namespace bpkg
? *p->src_root
: c / *p->src_root);
- const repository_location& rl (p->repository);
-
- // For external packages call the output directory <pkg>, rather than
- // <pkg>-<ver>.
- //
- dir_path out_root (
- (!rl.empty () && rl.directory_based ()) || // pkg-unpack <name>/<version>
- ( rl.empty () && !p->archive) // pkg-unpack --existing <dir>
- ? c / dir_path (p->name)
- : c / dir_path (p->name + "-" + p->version.string ()));
+ dir_path out_root (p->external ()
+ ? c / dir_path (p->name)
+ : c / dir_path (p->name + "-" + p->version.string ()));
l4 ([&]{trace << "src_root: " << src_root << ", "
<< "out_root: " << out_root;});
diff --git a/bpkg/rep-fetch.cli b/bpkg/rep-fetch.cli
index 719232d..a39aafb 100644
--- a/bpkg/rep-fetch.cli
+++ b/bpkg/rep-fetch.cli
@@ -39,5 +39,12 @@ namespace bpkg
class rep_fetch_options: configuration_options
{
"\h|REP-FETCH OPTIONS|"
+
+ bool --shallow
+ {
+ "Do not re-fetch complement and prerequisite repositories of the
+ specified repositories unless the set of complements and/or
+ prerequisites has changed."
+ }
};
}
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 8f1528c..1f0f893 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -5,6 +5,7 @@
#include <bpkg/rep-fetch.hxx>
#include <set>
+#include <algorithm> // equal()
#include <libbutl/manifest-parser.mxx>
@@ -431,6 +432,7 @@ namespace bpkg
const shared_ptr<repository>& r,
repositories& fetched,
repositories& removed,
+ bool shallow,
const string& reason = string ())
{
tracer trace ("rep_fetch(rep)");
@@ -471,42 +473,19 @@ namespace bpkg
dr << " (" << reason << ")";
}
- // Register complements and prerequisites for potential removal unless
- // they are fetched. Clear repository dependency sets afterwards.
- //
- auto remove = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp)
- {
- shared_ptr<repository> r (rp.load ());
- if (fetched.find (r) == fetched.end ())
- removed.insert (move (r));
- };
-
- for (const lazy_shared_ptr<repository>& cr: r->complements)
- {
- // Remove the complement unless it is the root repository (see
- // rep_fetch() for details).
- //
- if (cr.object_id () != "")
- remove (cr);
- }
-
- for (const lazy_weak_ptr<repository>& pr: r->prerequisites)
- remove (lazy_shared_ptr<repository> (pr));
-
- r->complements.clear ();
- r->prerequisites.clear ();
-
- // Remove this repository from locations of the available packages it
- // contains.
- //
- rep_remove_package_locations (t, r->name);
-
// Load the repository and package manifests and use them to populate the
// prerequisite and complement repository sets as well as available
// packages.
//
rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */));
+ // Create the new prerequisite and complement repository sets. While doing
+ // this we may also reset the shallow flag if discover that any of these
+ // sets have changed.
+ //
+ repository::complements_type complements;
+ repository::prerequisites_type prerequisites;
+
for (repository_manifest& rm: rfd.repositories)
{
repository_role rr (rm.effective_role ());
@@ -542,25 +521,16 @@ namespace bpkg
{
pr = make_shared<repository> (move (l));
db.persist (pr); // Enter into session, important if recursive.
+
+ shallow = false;
}
else if (pr->location.url () != l.url ())
{
pr->location = move (l);
db.update (r);
- }
- // Load the prerequisite repository.
- //
- string reason;
- switch (rr)
- {
- case repository_role::complement: reason = "complements "; break;
- case repository_role::prerequisite: reason = "prerequisite of "; break;
- case repository_role::base: assert (false);
+ shallow = false;
}
- reason += r->name;
-
- rep_fetch (co, conf, t, pr, fetched, removed, reason);
// @@ What if we have duplicated? Ideally, we would like to check
// this once and as early as possible. The original idea was to
@@ -582,13 +552,13 @@ namespace bpkg
case repository_role::complement:
{
l4 ([&]{trace << pr->name << " complement of " << r->name;});
- r->complements.insert (lazy_shared_ptr<repository> (db, pr));
+ complements.insert (lazy_shared_ptr<repository> (db, pr));
break;
}
case repository_role::prerequisite:
{
l4 ([&]{trace << pr->name << " prerequisite of " << r->name;});
- r->prerequisites.insert (lazy_weak_ptr<repository> (db, pr));
+ prerequisites.insert (lazy_weak_ptr<repository> (db, pr));
break;
}
case repository_role::base:
@@ -610,8 +580,8 @@ namespace bpkg
case repository_type::git:
case repository_type::dir:
{
- if (r->complements.empty () && r->prerequisites.empty ())
- r->complements.insert (lazy_shared_ptr<repository> (db, string ()));
+ if (complements.empty () && prerequisites.empty ())
+ complements.insert (lazy_shared_ptr<repository> (db, string ()));
break;
}
@@ -624,9 +594,83 @@ namespace bpkg
}
}
- // Save the changes to the repository object.
+ // Reset the shallow flag if the set of complements and/or prerequisites
+ // has changed.
//
- db.update (r);
+ // Note that weak pointers are generally incomparable (as can point to
+ // expired objects), and thus we can't compare the prerequisite sets
+ // directly.
+ //
+ if (shallow)
+ shallow = r->complements == complements &&
+ equal (r->prerequisites.begin (), r->prerequisites.end (),
+ prerequisites.begin (), prerequisites.end (),
+ [] (const lazy_weak_ptr<repository>& x,
+ const lazy_weak_ptr<repository>& y)
+ {
+ return x.object_id () == y.object_id ();
+ });
+
+ // Fetch prerequisites and complements, unless this is a shallow fetch.
+ //
+ if (!shallow)
+ {
+ // Register complements and prerequisites for potential removal unless
+ // they are fetched. Clear repository dependency sets afterwards.
+ //
+ auto rm = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp)
+ {
+ shared_ptr<repository> r (rp.load ());
+ if (fetched.find (r) == fetched.end ())
+ removed.insert (move (r));
+ };
+
+ for (const lazy_shared_ptr<repository>& cr: r->complements)
+ {
+ // Remove the complement unless it is the root repository (see
+ // rep_fetch() for details).
+ //
+ if (cr.object_id () != "")
+ rm (cr);
+ }
+
+ for (const lazy_weak_ptr<repository>& pr: r->prerequisites)
+ rm (lazy_shared_ptr<repository> (pr));
+
+ r->complements = move (complements);
+ r->prerequisites = move (prerequisites);
+
+ // Fetch complements.
+ //
+ for (const auto& cr: r->complements)
+ {
+ if (cr.object_id () != "")
+ rep_fetch (co,
+ conf,
+ t,
+ cr.load (),
+ fetched,
+ removed,
+ false /* shallow */,
+ "complements " + r->name);
+ }
+
+ // Fetch prerequisites.
+ //
+ for (const auto& pr: r->prerequisites)
+ rep_fetch (co,
+ conf,
+ t,
+ pr.load (),
+ fetched,
+ removed,
+ false /* shallow */,
+ "prerequisite of " + r->name);
+
+ // Save the changes to the repository object.
+ //
+ db.update (r);
+ }
// "Suspend" session while persisting packages to reduce memory
// consumption.
@@ -634,6 +678,11 @@ namespace bpkg
session& s (session::current ());
session::reset_current ();
+ // Remove this repository from locations of the available packages it
+ // contains.
+ //
+ rep_remove_package_locations (t, r->name);
+
for (rep_fetch_data::package& fp: rfd.packages)
{
package_manifest& pm (fp.manifest);
@@ -702,7 +751,8 @@ namespace bpkg
rep_fetch (const common_options& o,
const dir_path& conf,
transaction& t,
- const vector<lazy_shared_ptr<repository>>& repos)
+ const vector<lazy_shared_ptr<repository>>& repos,
+ bool shallow)
{
// As a fist step we fetch repositories recursively building the list of
// the former prerequisites and complements to be considered for removal.
@@ -722,7 +772,7 @@ namespace bpkg
repositories removed;
for (const lazy_shared_ptr<repository>& r: repos)
- rep_fetch (o, conf, t, r.load (), fetched, removed);
+ rep_fetch (o, conf, t, r.load (), fetched, removed, shallow);
// Finally, remove dangling repositories.
//
@@ -752,8 +802,11 @@ namespace bpkg
rep_fetch (const common_options& o,
const dir_path& conf,
database& db,
- const vector<repository_location>& rls)
+ const vector<repository_location>& rls,
+ bool shallow)
{
+ assert (session::has_current ());
+
vector<lazy_shared_ptr<repository>> repos;
repos.reserve (rls.size ());
@@ -775,7 +828,7 @@ namespace bpkg
repos.emplace_back (r);
}
- rep_fetch (o, conf, t, repos);
+ rep_fetch (o, conf, t, repos, shallow);
t.commit ();
}
@@ -846,7 +899,7 @@ namespace bpkg
}
}
- rep_fetch (o, c, t, repos);
+ rep_fetch (o, c, t, repos, o.shallow ());
size_t rcount (0), pcount (0);
if (verb)
diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx
index 030b192..18a50e9 100644
--- a/bpkg/rep-fetch.hxx
+++ b/bpkg/rep-fetch.hxx
@@ -51,14 +51,17 @@ namespace bpkg
bool ignore_unknown);
// Add (or update) repository locations to the configuration and fetch
- // them. On failure clean up the configuration (see rep_remove_clean() for
- // details). Note that it should be called in session.
+ // them. If shallow is true, then don't fetch their prerequisite and/or
+ // complements unless the respective sets have changed. On failure clean up
+ // the configuration (see rep_remove_clean() for details). Note that it
+ // should be called in session.
//
void
rep_fetch (const common_options&,
const dir_path& conf,
database&,
- const vector<repository_location>&);
+ const vector<repository_location>&,
+ bool shallow);
}
#endif // BPKG_REP_FETCH_HXX
diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test
index f0c0e2d..2339b91 100644
--- a/tests/rep-fetch.test
+++ b/tests/rep-fetch.test
@@ -142,19 +142,19 @@ $* 2>>/EOE != 0
$* --trust-yes 2>>EOE;
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
5 package(s) in 5 repository(s)
EOE
$* 2>>EOE
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
5 package(s) in 5 repository(s)
EOE
}
@@ -166,20 +166,20 @@ $* 2>>/EOE != 0
$* --trust-yes 2>>EOE &cfg/.bpkg/certs/**;
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
fetching pkg:build2.org/rep-fetch/hello
6 package(s) in 6 repository(s)
EOE
$* 2>>EOE
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
fetching pkg:build2.org/rep-fetch/hello
6 package(s) in 6 repository(s)
EOE
@@ -193,19 +193,19 @@ $* 2>>/EOE != 0
$* --trust-yes $rep/bar/unstable 2>>EOE;
added pkg:build2.org/rep-fetch/bar/unstable
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
5 package(s) in 5 repository(s)
EOE
$* 'pkg:build2.org/rep-fetch/bar/unstable' 2>>EOE;
fetching pkg:build2.org/rep-fetch/bar/unstable
- fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
- fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+ fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+ fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
5 package(s) in 5 repository(s)
EOE
@@ -369,6 +369,46 @@ if ($remote != true)
prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
EOO
}
+
+ : shallow
+ :
+ {
+ +$clone_cfg
+
+ : respected
+ :
+ {
+ $clone_cfg;
+ $* 2>!;
+
+ $* --shallow "dir:($rep/libbar.git)" 2>>"EOE";
+ fetching dir:($rep/libbar.git)
+ 3 package\(s\) in 2 repository\(s\)
+ EOE
+
+ $rep_list >>"EOO"
+ dir:($rep/libbar.git) ($rep/libbar.git)
+ prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
+ EOO
+ }
+
+ : ignored
+ :
+ {
+ $clone_cfg;
+
+ $* --shallow 2>>"EOE";
+ fetching dir:($rep/libbar.git)
+ fetching dir:($rep/style-basic.git) \(prerequisite of dir:($rep/libbar.git)\)
+ 3 package\(s\) in 2 repository\(s\)
+ EOE
+
+ $rep_list >>"EOO"
+ dir:($rep/libbar.git) ($rep/libbar.git)
+ prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
+ EOO
+ }
+ }
}
: git-rep