diff options
| -rw-r--r-- | bpkg/fetch-git.cxx | 160 | ||||
| -rw-r--r-- | bpkg/fetch.hxx | 14 | ||||
| -rw-r--r-- | bpkg/rep-fetch.cxx | 58 | ||||
| -rw-r--r-- | tests/rep-fetch-git.test | 7 |
4 files changed, 148 insertions, 91 deletions
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index fba1c8e..e6836a6 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -554,9 +554,9 @@ namespace bpkg // struct ref { - string name; // Note: without the peel marker ('^{}'). + string name; // Note: without the peel operation ('^{...}'). string commit; - bool peeled; // True for '...^{}' references. + bool peeled; // True for '...^{...}' references. }; // List of all refs and their commit ids advertized by a repository (i.e., @@ -823,33 +823,32 @@ namespace bpkg return false; } - // @@ Move to libbpkg when adding support for multiple fragments. + // Fetch and return repository fragments obtained with the repository + // reference filters specified. If there are no filters specified, then + // fetch all the tags and branches. // - using git_ref_filters = vector<git_ref_filter>; - - // Fetch the references and return their commit ids. If there are no - // reference filters specified, then fetch all the tags and branches. - // - static strings + static vector<git_fragment> fetch (const common_options& co, const dir_path& dir, const dir_path& submodule, // Used only for diagnostics. const git_ref_filters& rfs) { - strings r; + using git_fragments = vector<git_fragment>; + + git_fragments r; capabilities cap; repository_url url; - strings scs; // Shallow fetch commits. - strings dcs; // Deep fetch commits. + strings scs; // Shallow fetch commits. + strings dcs; // Deep fetch commits. bool fetch_repo (false); - // Translate a name to the corresponding commit. Fail if the name is not + // Translate a name to the corresponding reference. Fail if the name is not // known by the remote repository. // - auto commit = [&co, &dir, &url] (const string& nm, bool abbr_commit) - -> const string& + auto reference = [&co, &dir, &url] (const string& nm, bool abbr_commit) + -> const ref& { if (url.empty ()) url = origin_url (co, dir); @@ -860,17 +859,44 @@ namespace bpkg fail << "unable to fetch " << nm << " from " << url << info << "name is not recognized" << endg; - return r->commit; + return *r; }; - // Add a commit to the list, suppressing duplicates. + // Add a fragment to the resulting list, suppressing duplicates. // - auto add = [] (const string& c, strings& cs) + auto add_frag = [&r] (const git_fragment& f) + { + auto i (find_if (r.begin (), r.end (), + [&f] (const git_fragment& i) + { + return i.commit == f.commit; + })); + + if (i == r.end ()) + r.push_back (f); + else if (i->name.empty ()) + i->name = f.name; + }; + + // Add a commit to the list for subsequent fetching. + // + auto add_commit = [] (const string& c, strings& cs) { if (find (cs.begin (), cs.end (), c) == cs.end ()) cs.push_back (c); }; + // Return a user-friendly git reference name. + // + auto friendly_name = [] (const string& n) -> string + { + return n.compare (0, 11, "refs/heads/") == 0 + ? "branch " + string (n, 11) + : n.compare (0, 10, "refs/tags/") == 0 + ? "tag " + string (n, 10) + : "reference " + n; + }; + if (rfs.empty ()) { url = origin_url (co, dir); @@ -884,13 +910,14 @@ namespace bpkg rf.name.compare (0, 10, "refs/tags/") != 0)) continue; - // Add an already fetched commit to the resulting list. + // Add the commit to the resulting list. + // + add_frag (git_fragment {rf.commit, friendly_name (rf.name)}); + + // Skip the commit if it is already fetched. // if (commit_fetched (co, dir, rf.commit)) - { - add (rf.commit, r); continue; - } // Evaluate if the commit can be obtained with the shallow fetch and // add it to the proper list. @@ -901,7 +928,7 @@ namespace bpkg // The commit is advertised, so we can fetch shallow, unless the // protocol is dumb. // - add (rf.commit, cap != capabilities::dumb ? scs : dcs); + add_commit (rf.commit, cap != capabilities::dumb ? scs : dcs); } } else @@ -912,7 +939,7 @@ namespace bpkg // if (rf.commit && commit_fetched (co, dir, *rf.commit)) { - add (*rf.commit, r); + add_frag (git_fragment {*rf.commit, string ()}); continue; } @@ -920,11 +947,11 @@ namespace bpkg { assert (rf.name); - const string& c (commit (*rf.name, true /* abbr_commit */)); + const ref& rfc (reference (*rf.name, true /* abbr_commit */)); - if (commit_fetched (co, dir, c)) + if (commit_fetched (co, dir, rfc.commit)) { - add (c, r); + add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); continue; } } @@ -932,6 +959,9 @@ namespace bpkg // Evaluate if the commit can be obtained with the shallow fetch and // add it to the proper list. // + // Get the remote origin URL and sense the git protocol capabilities + // for it, if not done yet. + // if (scs.empty () && dcs.empty ()) { if (url.empty ()) // Can already be assigned by commit() lambda. @@ -941,7 +971,7 @@ namespace bpkg } bool shallow (shallow_fetch (co, url, cap, rf)); - strings& commits (shallow ? scs : dcs); + strings& fetch_list (shallow ? scs : dcs); // If commit is not specified, then we fetch the commit the refname // translates to. @@ -950,29 +980,42 @@ namespace bpkg { assert (rf.name); - add (commit (*rf.name, true /* abbr_commit */), commits); - } + const ref& rfc (reference (*rf.name, true /* abbr_commit */)); + add_frag (git_fragment {rfc.commit, friendly_name (rfc.name)}); + add_commit (rfc.commit, fetch_list); + } // If commit is specified and the shallow fetch is possible, then we // fetch the commit. // else if (shallow) - add (*rf.commit, commits); - + { + add_frag (git_fragment {*rf.commit, string ()}); + add_commit (*rf.commit, fetch_list); + } // If commit is specified and the shallow fetch is not possible, but // the refname containing the commit is specified, then we fetch the // whole refname history. // else if (rf.name) - add (commit (*rf.name, false /* abbr_commit */), commits); + { + // Note that commits we return and fetch are likely to differ. + // + add_frag (git_fragment {*rf.commit, string ()}); + add_commit (reference (*rf.name, false /* abbr_commit */).commit, + fetch_list); + } // Otherwise, if the refname is not specified and the commit is not // advertised, we have to fetch the whole repository history. // else { - add (*rf.commit, commits); - fetch_repo = !commit_advertized (co, url, *rf.commit); + add_frag (git_fragment {*rf.commit, string ()}); + add_commit (*rf.commit, fetch_list); + + if (!commit_advertized (co, url, *rf.commit)) + fetch_repo = true; } } } @@ -1057,38 +1100,33 @@ namespace bpkg dr << " in '" << dir.posix_string () << "'"; // Is used by tests. } - // Note that the shallow, deep and the resulting lists don't overlap. - // Thus, we will be moving commits between the lists not caring about - // suppressing duplicates. - // - - // First, we fetch deep commits. + // First, we perform the deep fetching. // if (!dcs.empty ()) - { fetch (!fetch_repo ? dcs : strings (), false); - r.insert (r.end (), - make_move_iterator (dcs.begin ()), - make_move_iterator (dcs.end ())); + // After the deep fetching some of the shallow commits might also be + // fetched, so we drop them from the fetch list. + // + for (auto i (scs.begin ()); i != scs.end (); ) + { + if (commit_fetched (co, dir, *i)) + i = scs.erase (i); + else + ++i; } - // After the deep fetching some of the shallow commits might also be - // fetched, so we move them to the resulting list. + // Finally, we perform the shallow fetching. // - strings cs; - for (auto& c: scs) - (commit_fetched (co, dir, c) ? r : cs).push_back (move (c)); + if (!scs.empty ()) + fetch (scs, true); - // Finally, fetch shallow commits. + // Name the unnamed commits with a 12-character length abbreviation. // - if (!cs.empty ()) + for (auto& fr: r) { - fetch (cs, true); - - r.insert (r.end (), - make_move_iterator (cs.begin ()), - make_move_iterator (cs.end ())); + if (fr.name.empty ()) + fr.name = "commit " + fr.commit.substr (0, 12); } return r; @@ -1339,18 +1377,18 @@ namespace bpkg init (co, dir, url); } - strings + vector<git_fragment> git_fetch (const common_options& co, const repository_location& rl, const dir_path& dir) { - git_ref_filters refs; + git_ref_filters rfs; repository_url url (rl.url ()); if (url.fragment) try { - refs.emplace_back (*url.fragment); + rfs = parse_git_ref_filters (*url.fragment); url.fragment = nullopt; } catch (const invalid_argument& e) @@ -1383,7 +1421,7 @@ namespace bpkg } } - return fetch (co, dir, dir_path () /* submodule */, refs); + return fetch (co, dir, dir_path () /* submodule */, rfs); } void diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx index 9530455..d05b8da 100644 --- a/bpkg/fetch.hxx +++ b/bpkg/fetch.hxx @@ -59,13 +59,19 @@ namespace bpkg const dir_path&); // Fetch a git repository in the specifid directory (previously created by - // git_init()) for the references obtained from the repository URL fragment - // returning commit ids these references resolve to. After fetching the - // repository working tree state is unspecified (see git_checkout ()). + // git_init() for the references obtained with the repository URL fragment + // filters, returning commit ids these references resolve to. After fetching + // the repository working tree state is unspecified (see git_checkout ()). // // Note that submodules are not fetched. // - strings + struct git_fragment + { + string commit; + string name; // User-friendly name (like 'branch foo', 'tag bar', ...). + }; + + vector<git_fragment> git_fetch (const common_options&, const repository_location&, const dir_path&); diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 26a1dec..c92db10 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -106,7 +106,10 @@ namespace bpkg template <typename M> static M - parse_manifest (const path& f, bool iu, const repository_location& rl) + parse_manifest (const path& f, + bool iu, + const repository_location& rl, + const string& frag) { try { @@ -117,12 +120,11 @@ namespace bpkg catch (const manifest_parsing& e) { fail (e.name, e.line, e.column) << e.description << - info << "repository " << rl << endf; + info << "repository " << rl << ' ' << frag << endf; } catch (const io_error& e) { - fail << "unable to read from " << f << ": " << e << - info << "repository " << rl << endf; + fail << "unable to read from " << f << ": " << e << endf; } } @@ -133,11 +135,12 @@ namespace bpkg static M parse_repository_manifests (const path& f, bool iu, - const repository_location& rl) + const repository_location& rl, + const string& frag) { M r; if (exists (f)) - r = parse_manifest<M> (f, iu, rl); + r = parse_manifest<M> (f, iu, rl, frag); else r.emplace_back (repository_manifest ()); // Add the base repository. @@ -152,11 +155,12 @@ namespace bpkg static M parse_directory_manifests (const path& f, bool iu, - const repository_location& rl) + const repository_location& rl, + const string& frag) { M r; if (exists (f)) - r = parse_manifest<M> (f, iu, rl); + r = parse_manifest<M> (f, iu, rl, frag); else { r.push_back (package_manifest ()); @@ -174,7 +178,8 @@ namespace bpkg const string& repo_fragment, vector<package_manifest>&& sms, bool iu, - const repository_location& rl) + const repository_location& rl, + const string& frag) { vector<rep_fetch_data::package> fps; fps.reserve (sms.size ()); @@ -183,14 +188,14 @@ namespace bpkg { assert (sm.location); - auto package_info = [&sm, &rl] (diag_record& dr) + auto package_info = [&sm, &rl, &frag] (diag_record& dr) { dr << "package "; if (!sm.location->current ()) dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. - dr << "in repository " << rl; + dr << "in repository " << rl << ' ' << frag; }; auto failure = [&package_info] (const char* desc) @@ -225,9 +230,7 @@ namespace bpkg } catch (const io_error& e) { - diag_record dr (fail); - dr << "unable to read from " << f << ": " << e << info; - package_info (dr); + fail << "unable to read from " << f << ": " << e; } // Fix-up the package version. @@ -256,13 +259,15 @@ namespace bpkg parse_repository_manifests<dir_repository_manifests> ( rd / repositories_file, ignore_unknown, - rl)); + rl, + string () /* frag */)); dir_package_manifests pms ( parse_directory_manifests<dir_package_manifests> ( rd / packages_file, ignore_unknown, - rl)); + rl, + string () /* frag */)); vector<rep_fetch_data::package> fps ( parse_package_manifests (co, @@ -270,7 +275,8 @@ namespace bpkg string () /* repo_fragment */, move (pms), ignore_unknown, - rl)); + rl, + string () /* frag */)); return rep_fetch_data {move (rms), move (fps), nullptr}; } @@ -327,9 +333,6 @@ namespace bpkg // Fetch the repository in the temporary directory. // - strings commits (git_fetch (co, rl, td)); - assert (!commits.empty ()); - // Go through fetched commits, checking them out and collecting the // prerequisite repositories and packages. // @@ -358,9 +361,9 @@ namespace bpkg git_repository_manifests rms; vector<rep_fetch_data::package> fps; - for (const string& c: commits) + for (const git_fragment& fr: git_fetch (co, rl, td)) { - git_checkout (co, td, c); + git_checkout (co, td, fr.commit); // Parse repository manifests. // @@ -368,7 +371,8 @@ namespace bpkg rms = parse_repository_manifests<git_repository_manifests> ( td / repositories_file, ignore_unknown, - rl); + rl, + fr.name); // Parse package skeleton manifests. // @@ -376,7 +380,8 @@ namespace bpkg parse_directory_manifests<git_package_manifests> ( td / packages_file, ignore_unknown, - rl)); + rl, + fr.name)); // Checkout submodules, if required. // @@ -396,10 +401,11 @@ namespace bpkg vector<rep_fetch_data::package> cps ( parse_package_manifests (co, td, - c, + fr.commit, move (pms), ignore_unknown, - rl)); + rl, + fr.name)); fps.insert (fps.end (), make_move_iterator (cps.begin ()), diff --git a/tests/rep-fetch-git.test b/tests/rep-fetch-git.test index 4b2a597..697d967 100644 --- a/tests/rep-fetch-git.test +++ b/tests/rep-fetch-git.test @@ -72,6 +72,13 @@ end .include rep-fetch-git-refname.test } +: list +: +{ + fragment = '#master,ltag,atag' + .include rep-fetch-git-refname.test +} + : commit : { |