aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2018-04-19 22:42:10 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2018-04-20 15:56:25 +0300
commit1f490ec2c9d1e5c80697b370d82f0bb3530aa450 (patch)
treed0a883b0d2490b1a1276c02bd1fc4b393eb86c33
parentf2f08e0758243a820fe47128ffabaa474c0e86e7 (diff)
Add support for comma-separated list of git reference filter
-rw-r--r--bpkg/fetch-git.cxx160
-rw-r--r--bpkg/fetch.hxx14
-rw-r--r--bpkg/rep-fetch.cxx58
-rw-r--r--tests/rep-fetch-git.test7
4 files changed, 148 insertions, 91 deletions
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index fba1c8e..e6836a6 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -554,9 +554,9 @@ namespace bpkg
//
struct ref
{
- string name; // Note: without the peel marker ('^{}').
+ string name; // Note: without the peel operation ('^{...}').
string commit;
- bool peeled; // True for '...^{}' references.
+ bool peeled; // True for '...^{...}' references.
};
// List of all refs and their commit ids advertized by a repository (i.e.,
@@ -823,33 +823,32 @@ namespace bpkg
return false;
}
- // @@ Move to libbpkg when adding support for multiple fragments.
+ // Fetch and return repository fragments obtained with the repository
+ // reference filters specified. If there are no filters specified, then
+ // fetch all the tags and branches.
//
- using git_ref_filters = vector<git_ref_filter>;
-
- // Fetch the references and return their commit ids. If there are no
- // reference filters specified, then fetch all the tags and branches.
- //
- static strings
+ static vector<git_fragment>
fetch (const common_options& co,
const dir_path& dir,
const dir_path& submodule, // Used only for diagnostics.
const git_ref_filters& rfs)
{
- strings r;
+ using git_fragments = vector<git_fragment>;
+
+ git_fragments r;
capabilities cap;
repository_url url;
- strings scs; // Shallow fetch commits.
- strings dcs; // Deep fetch commits.
+ strings scs; // Shallow fetch commits.
+ strings dcs; // Deep fetch commits.
bool fetch_repo (false);
- // Translate a name to the corresponding commit. Fail if the name is not
+ // Translate a name to the corresponding reference. Fail if the name is not
// known by the remote repository.
//
- auto commit = [&co, &dir, &url] (const string& nm, bool abbr_commit)
- -> const string&
+ auto reference = [&co, &dir, &url] (const string& nm, bool abbr_commit)
+ -> const ref&
{
if (url.empty ())
url = origin_url (co, dir);
@@ -860,17 +859,44 @@ namespace bpkg
fail << "unable to fetch " << nm << " from " << url <<
info << "name is not recognized" << endg;
- return r->commit;
+ return *r;
};
- // Add a commit to the list, suppressing duplicates.
+ // Add a fragment to the resulting list, suppressing duplicates.
//
- auto add = [] (const string& c, strings& cs)
+ auto add_frag = [&r] (const git_fragment& f)
+ {
+ auto i (find_if (r.begin (), r.end (),
+ [&f] (const git_fragment& i)
+ {
+ return i.commit == f.commit;
+ }));
+
+ if (i == r.end ())
+ r.push_back (f);
+ else if (i->name.empty ())
+ i->name = f.name;
+ };
+
+ // Add a commit to the list for subsequent fetching.
+ //
+ auto add_commit = [] (const string& c, strings& cs)
{
if (find (cs.begin (), cs.end (), c) == cs.end ())
cs.push_back (c);
};
+ // Return a user-friendly git reference name.
+ //
+ auto friendly_name = [] (const string& n) -> string
+ {
+ return n.compare (0, 11, "refs/heads/") == 0
+ ? "branch " + string (n, 11)
+ : n.compare (0, 10, "refs/tags/") == 0
+ ? "tag " + string (n, 10)
+ : "reference " + n;
+ };
+
if (rfs.empty ())
{
url = origin_url (co, dir);
@@ -884,13 +910,14 @@ namespace bpkg
rf.name.compare (0, 10, "refs/tags/") != 0))
continue;
- // Add an already fetched commit to the resulting list.
+ // Add the commit to the resulting list.
+ //
+ add_frag (git_fragment {rf.commit, friendly_name (rf.name)});
+
+ // Skip the commit if it is already fetched.
//
if (commit_fetched (co, dir, rf.commit))
- {
- add (rf.commit, r);
continue;
- }
// Evaluate if the commit can be obtained with the shallow fetch and
// add it to the proper list.
@@ -901,7 +928,7 @@ namespace bpkg
// The commit is advertised, so we can fetch shallow, unless the
// protocol is dumb.
//
- add (rf.commit, cap != capabilities::dumb ? scs : dcs);
+ add_commit (rf.commit, cap != capabilities::dumb ? scs : dcs);
}
}
else
@@ -912,7 +939,7 @@ namespace bpkg
//
if (rf.commit && commit_fetched (co, dir, *rf.commit))
{
- add (*rf.commit, r);
+ add_frag (git_fragment {*rf.commit, string ()});
continue;
}
@@ -920,11 +947,11 @@ namespace bpkg
{
assert (rf.name);
- const string& c (commit (*rf.name, true /* abbr_commit */));
+ const ref& rfc (reference (*rf.name, true /* abbr_commit */));
- if (commit_fetched (co, dir, c))
+ if (commit_fetched (co, dir, rfc.commit))
{
- add (c, r);
+ add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)});
continue;
}
}
@@ -932,6 +959,9 @@ namespace bpkg
// Evaluate if the commit can be obtained with the shallow fetch and
// add it to the proper list.
//
+ // Get the remote origin URL and sense the git protocol capabilities
+ // for it, if not done yet.
+ //
if (scs.empty () && dcs.empty ())
{
if (url.empty ()) // Can already be assigned by commit() lambda.
@@ -941,7 +971,7 @@ namespace bpkg
}
bool shallow (shallow_fetch (co, url, cap, rf));
- strings& commits (shallow ? scs : dcs);
+ strings& fetch_list (shallow ? scs : dcs);
// If commit is not specified, then we fetch the commit the refname
// translates to.
@@ -950,29 +980,42 @@ namespace bpkg
{
assert (rf.name);
- add (commit (*rf.name, true /* abbr_commit */), commits);
- }
+ const ref& rfc (reference (*rf.name, true /* abbr_commit */));
+ add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)});
+ add_commit (rfc.commit, fetch_list);
+ }
// If commit is specified and the shallow fetch is possible, then we
// fetch the commit.
//
else if (shallow)
- add (*rf.commit, commits);
-
+ {
+ add_frag (git_fragment {*rf.commit, string ()});
+ add_commit (*rf.commit, fetch_list);
+ }
// If commit is specified and the shallow fetch is not possible, but
// the refname containing the commit is specified, then we fetch the
// whole refname history.
//
else if (rf.name)
- add (commit (*rf.name, false /* abbr_commit */), commits);
+ {
+ // Note that commits we return and fetch are likely to differ.
+ //
+ add_frag (git_fragment {*rf.commit, string ()});
+ add_commit (reference (*rf.name, false /* abbr_commit */).commit,
+ fetch_list);
+ }
// Otherwise, if the refname is not specified and the commit is not
// advertised, we have to fetch the whole repository history.
//
else
{
- add (*rf.commit, commits);
- fetch_repo = !commit_advertized (co, url, *rf.commit);
+ add_frag (git_fragment {*rf.commit, string ()});
+ add_commit (*rf.commit, fetch_list);
+
+ if (!commit_advertized (co, url, *rf.commit))
+ fetch_repo = true;
}
}
}
@@ -1057,38 +1100,33 @@ namespace bpkg
dr << " in '" << dir.posix_string () << "'"; // Is used by tests.
}
- // Note that the shallow, deep and the resulting lists don't overlap.
- // Thus, we will be moving commits between the lists not caring about
- // suppressing duplicates.
- //
-
- // First, we fetch deep commits.
+ // First, we perform the deep fetching.
//
if (!dcs.empty ())
- {
fetch (!fetch_repo ? dcs : strings (), false);
- r.insert (r.end (),
- make_move_iterator (dcs.begin ()),
- make_move_iterator (dcs.end ()));
+ // After the deep fetching some of the shallow commits might also be
+ // fetched, so we drop them from the fetch list.
+ //
+ for (auto i (scs.begin ()); i != scs.end (); )
+ {
+ if (commit_fetched (co, dir, *i))
+ i = scs.erase (i);
+ else
+ ++i;
}
- // After the deep fetching some of the shallow commits might also be
- // fetched, so we move them to the resulting list.
+ // Finally, we perform the shallow fetching.
//
- strings cs;
- for (auto& c: scs)
- (commit_fetched (co, dir, c) ? r : cs).push_back (move (c));
+ if (!scs.empty ())
+ fetch (scs, true);
- // Finally, fetch shallow commits.
+ // Name the unnamed commits with a 12-character length abbreviation.
//
- if (!cs.empty ())
+ for (auto& fr: r)
{
- fetch (cs, true);
-
- r.insert (r.end (),
- make_move_iterator (cs.begin ()),
- make_move_iterator (cs.end ()));
+ if (fr.name.empty ())
+ fr.name = "commit " + fr.commit.substr (0, 12);
}
return r;
@@ -1339,18 +1377,18 @@ namespace bpkg
init (co, dir, url);
}
- strings
+ vector<git_fragment>
git_fetch (const common_options& co,
const repository_location& rl,
const dir_path& dir)
{
- git_ref_filters refs;
+ git_ref_filters rfs;
repository_url url (rl.url ());
if (url.fragment)
try
{
- refs.emplace_back (*url.fragment);
+ rfs = parse_git_ref_filters (*url.fragment);
url.fragment = nullopt;
}
catch (const invalid_argument& e)
@@ -1383,7 +1421,7 @@ namespace bpkg
}
}
- return fetch (co, dir, dir_path () /* submodule */, refs);
+ return fetch (co, dir, dir_path () /* submodule */, rfs);
}
void
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index 9530455..d05b8da 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -59,13 +59,19 @@ namespace bpkg
const dir_path&);
// Fetch a git repository in the specifid directory (previously created by
- // git_init()) for the references obtained from the repository URL fragment
- // returning commit ids these references resolve to. After fetching the
- // repository working tree state is unspecified (see git_checkout ()).
+ // git_init() for the references obtained with the repository URL fragment
+ // filters, returning commit ids these references resolve to. After fetching
+ // the repository working tree state is unspecified (see git_checkout ()).
//
// Note that submodules are not fetched.
//
- strings
+ struct git_fragment
+ {
+ string commit;
+ string name; // User-friendly name (like 'branch foo', 'tag bar', ...).
+ };
+
+ vector<git_fragment>
git_fetch (const common_options&,
const repository_location&,
const dir_path&);
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 26a1dec..c92db10 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -106,7 +106,10 @@ namespace bpkg
template <typename M>
static M
- parse_manifest (const path& f, bool iu, const repository_location& rl)
+ parse_manifest (const path& f,
+ bool iu,
+ const repository_location& rl,
+ const string& frag)
{
try
{
@@ -117,12 +120,11 @@ namespace bpkg
catch (const manifest_parsing& e)
{
fail (e.name, e.line, e.column) << e.description <<
- info << "repository " << rl << endf;
+ info << "repository " << rl << ' ' << frag << endf;
}
catch (const io_error& e)
{
- fail << "unable to read from " << f << ": " << e <<
- info << "repository " << rl << endf;
+ fail << "unable to read from " << f << ": " << e << endf;
}
}
@@ -133,11 +135,12 @@ namespace bpkg
static M
parse_repository_manifests (const path& f,
bool iu,
- const repository_location& rl)
+ const repository_location& rl,
+ const string& frag)
{
M r;
if (exists (f))
- r = parse_manifest<M> (f, iu, rl);
+ r = parse_manifest<M> (f, iu, rl, frag);
else
r.emplace_back (repository_manifest ()); // Add the base repository.
@@ -152,11 +155,12 @@ namespace bpkg
static M
parse_directory_manifests (const path& f,
bool iu,
- const repository_location& rl)
+ const repository_location& rl,
+ const string& frag)
{
M r;
if (exists (f))
- r = parse_manifest<M> (f, iu, rl);
+ r = parse_manifest<M> (f, iu, rl, frag);
else
{
r.push_back (package_manifest ());
@@ -174,7 +178,8 @@ namespace bpkg
const string& repo_fragment,
vector<package_manifest>&& sms,
bool iu,
- const repository_location& rl)
+ const repository_location& rl,
+ const string& frag)
{
vector<rep_fetch_data::package> fps;
fps.reserve (sms.size ());
@@ -183,14 +188,14 @@ namespace bpkg
{
assert (sm.location);
- auto package_info = [&sm, &rl] (diag_record& dr)
+ auto package_info = [&sm, &rl, &frag] (diag_record& dr)
{
dr << "package ";
if (!sm.location->current ())
dr << "'" << sm.location->string () << "' "; // Strip trailing '/'.
- dr << "in repository " << rl;
+ dr << "in repository " << rl << ' ' << frag;
};
auto failure = [&package_info] (const char* desc)
@@ -225,9 +230,7 @@ namespace bpkg
}
catch (const io_error& e)
{
- diag_record dr (fail);
- dr << "unable to read from " << f << ": " << e << info;
- package_info (dr);
+ fail << "unable to read from " << f << ": " << e;
}
// Fix-up the package version.
@@ -256,13 +259,15 @@ namespace bpkg
parse_repository_manifests<dir_repository_manifests> (
rd / repositories_file,
ignore_unknown,
- rl));
+ rl,
+ string () /* frag */));
dir_package_manifests pms (
parse_directory_manifests<dir_package_manifests> (
rd / packages_file,
ignore_unknown,
- rl));
+ rl,
+ string () /* frag */));
vector<rep_fetch_data::package> fps (
parse_package_manifests (co,
@@ -270,7 +275,8 @@ namespace bpkg
string () /* repo_fragment */,
move (pms),
ignore_unknown,
- rl));
+ rl,
+ string () /* frag */));
return rep_fetch_data {move (rms), move (fps), nullptr};
}
@@ -327,9 +333,6 @@ namespace bpkg
// Fetch the repository in the temporary directory.
//
- strings commits (git_fetch (co, rl, td));
- assert (!commits.empty ());
-
// Go through fetched commits, checking them out and collecting the
// prerequisite repositories and packages.
//
@@ -358,9 +361,9 @@ namespace bpkg
git_repository_manifests rms;
vector<rep_fetch_data::package> fps;
- for (const string& c: commits)
+ for (const git_fragment& fr: git_fetch (co, rl, td))
{
- git_checkout (co, td, c);
+ git_checkout (co, td, fr.commit);
// Parse repository manifests.
//
@@ -368,7 +371,8 @@ namespace bpkg
rms = parse_repository_manifests<git_repository_manifests> (
td / repositories_file,
ignore_unknown,
- rl);
+ rl,
+ fr.name);
// Parse package skeleton manifests.
//
@@ -376,7 +380,8 @@ namespace bpkg
parse_directory_manifests<git_package_manifests> (
td / packages_file,
ignore_unknown,
- rl));
+ rl,
+ fr.name));
// Checkout submodules, if required.
//
@@ -396,10 +401,11 @@ namespace bpkg
vector<rep_fetch_data::package> cps (
parse_package_manifests (co,
td,
- c,
+ fr.commit,
move (pms),
ignore_unknown,
- rl));
+ rl,
+ fr.name));
fps.insert (fps.end (),
make_move_iterator (cps.begin ()),
diff --git a/tests/rep-fetch-git.test b/tests/rep-fetch-git.test
index 4b2a597..697d967 100644
--- a/tests/rep-fetch-git.test
+++ b/tests/rep-fetch-git.test
@@ -72,6 +72,13 @@ end
.include rep-fetch-git-refname.test
}
+: list
+:
+{
+ fragment = '#master,ltag,atag'
+ .include rep-fetch-git-refname.test
+}
+
: commit
:
{