From 3d4838d3706de2ba0045dc9f99a3dc96398def64 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 19 Feb 2018 14:26:02 +0300 Subject: Parse repositories and packages files for git repositories --- bpkg/archive.cxx | 1 - bpkg/auth.cxx | 1 - bpkg/cfg-create.cxx | 3 - bpkg/checksum.cxx | 1 - bpkg/diagnostics.cxx | 16 +-- bpkg/fetch-bpkg.cxx | 1 - bpkg/fetch-git.cxx | 109 +++++++++------------ bpkg/fetch.cxx | 2 +- bpkg/fetch.hxx | 10 +- bpkg/package.hxx | 3 +- bpkg/pkg-verify.cxx | 1 - bpkg/rep-create.cxx | 1 - bpkg/rep-fetch.cxx | 267 +++++++++++++++++++++++++++++++++++++++++++++----- bpkg/satisfaction.cxx | 1 - bpkg/types.hxx | 9 ++ bpkg/utility.cxx | 26 +++++ bpkg/utility.hxx | 8 ++ 17 files changed, 340 insertions(+), 120 deletions(-) (limited to 'bpkg') diff --git a/bpkg/archive.cxx b/bpkg/archive.cxx index 03728a4..cc96cd1 100644 --- a/bpkg/archive.cxx +++ b/bpkg/archive.cxx @@ -5,7 +5,6 @@ #include #include -#include #include diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx index df9a259..df31a60 100644 --- a/bpkg/auth.cxx +++ b/bpkg/auth.cxx @@ -13,7 +13,6 @@ #include #include #include -#include #include #include diff --git a/bpkg/cfg-create.cxx b/bpkg/cfg-create.cxx index 1a21cfa..70d8848 100644 --- a/bpkg/cfg-create.cxx +++ b/bpkg/cfg-create.cxx @@ -4,15 +4,12 @@ #include -#include - #include #include #include #include using namespace std; -using namespace butl; namespace bpkg { diff --git a/bpkg/checksum.cxx b/bpkg/checksum.cxx index 0ada207..14c0bf7 100644 --- a/bpkg/checksum.cxx +++ b/bpkg/checksum.cxx @@ -13,7 +13,6 @@ #include #include -#include #include #include diff --git a/bpkg/diagnostics.cxx b/bpkg/diagnostics.cxx index 9672926..bb175d8 100644 --- a/bpkg/diagnostics.cxx +++ b/bpkg/diagnostics.cxx @@ -10,6 +10,7 @@ #include #include +#include // operator<<(ostream, process_arg) using namespace std; using namespace butl; @@ -25,19 +26,6 @@ namespace bpkg print_process (r, args, n); } - struct process_args - { - const char* const* a; - size_t n; - }; - - inline static ostream& - operator<< (ostream& o, const process_args& p) - { - process::print (o, p.a, p.n); - return o; - } - void print_process (diag_record& r, const char* const args[], size_t n) { @@ -104,7 +92,7 @@ namespace bpkg if (verb >= 3) { diag_record dr (*this); - process::print (dr.os, args, n); + print_process (dr, args, n); } } diff --git a/bpkg/fetch-bpkg.cxx b/bpkg/fetch-bpkg.cxx index 4fe0bfc..606843f 100644 --- a/bpkg/fetch-bpkg.cxx +++ b/bpkg/fetch-bpkg.cxx @@ -6,7 +6,6 @@ #include -#include #include // cpfile () #include diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index fa9723c..7ffaaaf 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -10,7 +10,6 @@ #include // digit(), xdigit() #include -#include #include #include @@ -34,32 +33,6 @@ namespace bpkg static const diag_noreturn_end endg; - static fdpipe - open_pipe () - { - try - { - return fdopen_pipe (); - } - catch (const io_error& e) - { - fail << "unable to open pipe: " << e << endf; - } - } - - static auto_fd - open_dev_null () - { - try - { - return fdnull (); - } - catch (const io_error& e) - { - fail << "unable to open null device: " << e << endf; - } - } - using opt = optional; // Program option. static strings @@ -167,7 +140,9 @@ namespace bpkg } if (v.empty ()) - fail << "unable to obtain git version from '" << s << "'" << endg; + fail << "'" << s << "' doesn't appear to contain a git version" << + info << "produced by '" << co.git () << "'; " + << "use --git to override" << endg; if (v.version < 20120000000) fail << "unsupported git version " << v.string () << @@ -191,15 +166,14 @@ namespace bpkg try { - ifdstream is (move (pipe.in), fdstream_mode::skip); + ifdstream is (move (pipe.in), + fdstream_mode::skip, + ifdstream::badbit); - while (is.peek () != ifdstream::traits_type::eof ()) + for (string l; !eof (getline (is, l)); ) { - string v; - getline (is, v); - - if (v != "GIT_CONFIG_PARAMETERS") - unset_vars->push_back (move (v)); + if (l != "GIT_CONFIG_PARAMETERS") + unset_vars->push_back (move (l)); } is.close (); @@ -246,7 +220,9 @@ namespace bpkg static process_exit run_git (const common_options& co, A&&... args) { - process pr (start_git (co, 1, 2, forward (args)...)); + process pr (start_git (co, + 1 /* stdout */, 2 /* stderr */, + forward (args)...)); pr.wait (); return *pr.exit; } @@ -508,16 +484,13 @@ namespace bpkg try { bool r (false); - ifdstream is (move (pipe.in), fdstream_mode::skip); + ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit); - while (is.peek () != ifdstream::traits_type::eof ()) + for (string l; !eof (getline (is, l)); ) { - string s; - getline (is, s); - - l4 ([&]{trace << "ref: " << s;}); + l4 ([&]{trace << "ref: " << l;}); - if (s.compare (0, commit.size (), commit) == 0) + if (l.compare (0, commit.size (), commit) == 0) { r = true; break; @@ -787,9 +760,9 @@ namespace bpkg try { - ifdstream is (move (pipe.in), fdstream_mode::skip); + ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit); - while (is.peek () != ifdstream::traits_type::eof ()) + for (string l; !eof (getline (is, l)); ) { // The line describing a submodule has the following form: // @@ -799,19 +772,16 @@ namespace bpkg // // 160000 658436a9522b5a0d016c3da0253708093607f95d 0 doc/style // - string s; - getline (is, s); - - l4 ([&]{trace << "submodule: " << s;}); + l4 ([&]{trace << "submodule: " << l;}); - if (!(s.size () > 50 && s[48] == '0' && s[49] == '\t')) + if (!(l.size () > 50 && l[48] == '0' && l[49] == '\t')) failure ("invalid submodule description"); - string commit (s.substr (7, 40)); + string commit (l.substr (7, 40)); // Submodule directory path, relative to the containing project. // - dir_path sdir (s.substr (50)); + dir_path sdir (l.substr (50)); // Submodule directory path, relative to the top project. // @@ -955,7 +925,22 @@ namespace bpkg } } - void + // Produce a repository directory name for the specified git reference. + // + // Truncate commit id-based directory names to shorten absolute directory + // paths, lowering the probability of hitting the limit on Windows. + // + // Note that we can't truncate them for branches/tags as chances to clash + // would be way higher than for commit ids. Though such names are normally + // short anyway. + // + static inline dir_path + repository_dir (const git_reference& ref) + { + return dir_path (ref.commit ? ref.commit->substr (0, 16) : *ref.branch); + } + + dir_path git_clone (const common_options& co, const repository_location& rl, const dir_path& destdir) @@ -975,14 +960,8 @@ namespace bpkg else fetch_warn (cap, single_branch ? "branch" : "repository"); - dir_path d (destdir); - - // Truncate commit id-based directory names to shorten the absolute - // directory path to lower probability of hitting the limit on Windows. - // Note that we can't do the same for branch/tag names as chances to clash - // would be way higher. Though such names are normally short anyway. - // - d /= dir_path (ref.branch ? *ref.branch : ref.commit->substr (0, 16)); + dir_path r (repository_dir (ref)); + dir_path d (destdir / r); strings to (timeout_opts (co, url.scheme)); @@ -1007,9 +986,10 @@ namespace bpkg update_tree (co, d, dir_path (), ref, cap, shallow, to); update_submodules (co, d, dir_path ()); + return r; } - void + dir_path git_fetch (const common_options& co, const repository_location& rl, const dir_path& destdir) @@ -1017,6 +997,8 @@ namespace bpkg repository_url url (rl.url ()); git_reference ref (parse_reference (url, "fetch")); + dir_path r (repository_dir (ref)); + // Fetch is noop if the specific commit is checked out. // // What if the user replaces the repository URL with a one with a new @@ -1026,7 +1008,7 @@ namespace bpkg // this should work correctly automatically. // if (ref.commit) - return; + return r; assert (ref.branch); @@ -1045,5 +1027,6 @@ namespace bpkg timeout_opts (co, url.scheme)); update_submodules (co, d, dir_path ()); + return r; } } diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx index c5366e3..680ab9e 100644 --- a/bpkg/fetch.cxx +++ b/bpkg/fetch.cxx @@ -4,7 +4,7 @@ #include -#include +#include #include diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx index f243e09..659e019 100644 --- a/bpkg/fetch.hxx +++ b/bpkg/fetch.hxx @@ -49,16 +49,18 @@ namespace bpkg // Repository type git (fetch-git.cxx). // - // Clone git repository into destdir//. + // Clone git repository into destdir//. Return the cloned repository + // directory name that was deduced from the repository URL fragment. // - void + dir_path git_clone (const common_options&, const repository_location&, const dir_path& destdir); - // Fetch git repository in destdir//. + // Fetch git repository in destdir//. Return the fetched repository + // directory name that was deduced from the repository URL fragment. // - void + dir_path git_fetch (const common_options&, const repository_location&, const dir_path& destdir); diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 95b1bb3..67b1753 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -354,7 +354,8 @@ namespace bpkg dependencies_type dependencies; - // Present for non-transient objects only. + // Present for non-transient objects only (and only for certain repository + // types). // optional sha256sum; diff --git a/bpkg/pkg-verify.cxx b/bpkg/pkg-verify.cxx index d46e5b6..2f01f94 100644 --- a/bpkg/pkg-verify.cxx +++ b/bpkg/pkg-verify.cxx @@ -5,7 +5,6 @@ #include #include -#include #include #include diff --git a/bpkg/rep-create.cxx b/bpkg/rep-create.cxx index 228b7f6..e9619ca 100644 --- a/bpkg/rep-create.cxx +++ b/bpkg/rep-create.cxx @@ -6,7 +6,6 @@ #include -#include #include // dir_iterator #include diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index d7f0386..029db1d 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -5,6 +5,9 @@ #include #include +#include +#include // operator<<(ostream, process_path) +#include #include #include @@ -74,30 +77,64 @@ namespace bpkg return rep_fetch_data {move (rms), move (pms), move (cert)}; } + template + static M + parse_manifest (const path& f, bool iu, const repository_location& rl) + { + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f.string ()); + return M (mp, iu); + } + catch (const manifest_parsing& e) + { + fail (e.name, e.line, e.column) << e.description << + info << "repository " << rl << endf; + } + catch (const io_error& e) + { + fail << "unable to read from " << f << ": " << e << + info << "repository " << rl << endf; + } + } + static rep_fetch_data rep_fetch_git (const common_options& co, const dir_path* conf, const repository_location& rl, - bool /* ignore_unknown */) + bool ignore_unknown) { // Plan: // // 1. Check repos_dir//: // - // 1.a If does not exist, git-clone into temp_dir//. + // 1.a If does not exist, git-clone into temp_dir///. // // 1.a Otherwise, move as temp_dir// and git-fetch. // - // 2. Move from temp_dir// to repos_dir// + // 2. Move from temp_dir// to repos_dir/// // - // 3. Load manifest from repos_dir/// + // 3. Check if repos_dir///repositories exists: // - // 4. Run 'b info' in repos_dir/// and fix-up - // package version. + // 3.a If exists, load. // - // 5. Synthesize repository manifest. + // 3.b Otherwise, synthesize repository list with base repository. // - // 6. Return repository and package manifest (certificate is NULL). + // 4. Check if repos_dir///packages exists: + // + // 4.a If exists, load. (into "skeleton" packages list to be filled?) + // + // 4.b Otherwise, synthesize as if single 'location: ./'. + // + // 5. For each package location obtained on step 4: + // + // 5.a Load repos_dir////manifest. + // + // 5.b Run 'b info: repos_dir////' and fix-up + // package version. + // + // 6. Return repository and package manifests (certificate is NULL). // if (conf != nullptr && conf->empty ()) @@ -105,6 +142,8 @@ namespace bpkg assert (conf == nullptr || !conf->empty ()); + // Clone or fetch the repository. + // dir_path h (sha256 (rl.canonical_name ()).abbreviated_string (16)); auto_rmdir rm (temp_dir / h); @@ -129,10 +168,7 @@ namespace bpkg } } - if (fetch) - git_fetch (co, rl, td); - else - git_clone (co, rl, td); + dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); if (!rd.empty ()) mv (td, rd); @@ -144,9 +180,179 @@ namespace bpkg rm.cancel (); - // @@ TODO + rd /= nm; + + // Produce repository manifest list. // - return rep_fetch_data (); + git_repository_manifests rms; + { + path f (rd / path ("repositories")); + + if (exists (f)) + rms = parse_manifest (f, ignore_unknown, rl); + else + rms.emplace_back (repository_manifest ()); // Add the base repository. + } + + // Produce the "skeleton" package manifest list. + // + git_package_manifests pms; + { + path f (rd / path ("packages")); + + if (exists (f)) + pms = parse_manifest (f, ignore_unknown, rl); + else + { + pms.push_back (package_manifest ()); + pms.back ().location = current_dir; + } + } + + // Fill "skeleton" package manifests. + // + for (package_manifest& sm: pms) + { + assert (sm.location); + + auto package_info = [&sm, &rl] (diag_record& dr) + { + dr << "package "; + + if (!sm.location->current ()) + dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. + + dr << "in repository " << rl; + }; + + auto failure = [&package_info] (const char* desc) + { + diag_record dr (fail); + dr << desc << " for "; + package_info (dr); + }; + + dir_path d (rd / path_cast (*sm.location)); + path f (d / path ("manifest")); + + if (!exists (f)) + failure ("no manifest file"); + + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f.string ()); + package_manifest m (bpkg_package_manifest (mp, ignore_unknown)); + + // Save the package manifest, preserving its location. + // + m.location = move (sm.location); + sm = move (m); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail (e.name, e.line, e.column)); + dr << e.description << info; + package_info (dr); + } + catch (const io_error& e) + { + diag_record dr (fail); + dr << "unable to read from " << f << ": " << e << info; + package_info (dr); + } + + // Fix-up the package version. + // + const char* b (name_b (co)); + + try + { + process_path pp (process::path_search (b, exec_dir)); + + fdpipe pipe (open_pipe ()); + + process pr ( + process_start_callback ( + [] (const char* const args[], size_t n) + { + if (verb >= 2) + print_process (args, n); + }, + 0 /* stdin */, pipe /* stdout */, 2 /* stderr */, + pp, + + verb < 2 + ? strings ({"-q"}) + : verb == 2 + ? strings ({"-v"}) + : strings ({"--verbose", to_string (verb)}), + + co.build_option (), + "info:", + d.representation ())); + + // Shouldn't throw, unless something is severely damaged. + // + pipe.out.close (); + + try + { + ifdstream is (move (pipe.in), + fdstream_mode::skip, + ifdstream::badbit); + + for (string l; !eof (getline (is, l)); ) + { + if (l.compare (0, 9, "version: ") == 0) + try + { + string v (l, 9); + + // An empty version indicates that the version module is not + // enabled for the project, and so we don't amend the package + // version. + // + if (!v.empty ()) + sm.version = version (v); + + break; + } + catch (const invalid_argument&) + { + fail << "no package version in '" << l << "'" << + info << "produced by '" << pp << "'; use --build to override"; + } + } + + is.close (); + + if (pr.wait ()) + continue; // Go to the next package. + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + failure ("unable to read information"); + + // Fall through. + } + + // We should only get here if the child exited with an error status. + // + assert (!pr.wait ()); + + failure ("unable to obtain information"); + } + catch (const process_error& e) + { + fail << "unable to execute " << b << ": " << e; + } + } + + return rep_fetch_data {move (rms), move (pms), nullptr}; } rep_fetch_data @@ -319,21 +525,28 @@ namespace bpkg { // Make sure this is the same package. // - assert (p->sha256sum && !p->locations.empty ()); // Can't be transient. + assert (!p->locations.empty ()); // Can't be transient. - if (*pm.sha256sum != *p->sha256sum) + // Note that sha256sum may not present for some repository types. + // + if (pm.sha256sum) { - // All the previous repositories that contain this package have the - // same checksum (since they passed this test), so we can pick any - // to show to the user. - // - const string& r1 (rl.canonical_name ()); - const string& r2 (p->locations[0].repository.object_id ()); - - fail << "checksum mismatch for " << pm.name << " " << pm.version << - info << r1 << " has " << *pm.sha256sum << - info << r2 << " has " << *p->sha256sum << - info << "consider reporting this to the repository maintainers"; + if (!p->sha256sum) + p->sha256sum = move (pm.sha256sum); + else if (*pm.sha256sum != *p->sha256sum) + { + // All the previous repositories that have checksum for this + // package have it the same (since they passed this test), so we + // can pick any to show to the user. + // + const string& r1 (rl.canonical_name ()); + const string& r2 (p->locations[0].repository.object_id ()); + + fail << "checksum mismatch for " << pm.name << " " << pm.version << + info << r1 << " has " << *pm.sha256sum << + info << r2 << " has " << *p->sha256sum << + info << "consider reporting this to the repository maintainers"; + } } } diff --git a/bpkg/satisfaction.cxx b/bpkg/satisfaction.cxx index 9aaca23..9211ac0 100644 --- a/bpkg/satisfaction.cxx +++ b/bpkg/satisfaction.cxx @@ -5,7 +5,6 @@ #include #include -#include #include #include diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 505d468..3133696 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -24,6 +24,7 @@ #include #include +#include namespace bpkg { @@ -82,6 +83,14 @@ namespace bpkg using paths = std::vector; using dir_paths = std::vector; + + // + // + using butl::auto_fd; + using butl::fdpipe; + using butl::ifdstream; + using butl::ofdstream; + using butl::fdstream_mode; } // In order to be found (via ADL) these have to be either in std:: or in diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx index 26716e8..252bdef 100644 --- a/bpkg/utility.cxx +++ b/bpkg/utility.cxx @@ -234,6 +234,32 @@ namespace bpkg } } + fdpipe + open_pipe () + { + try + { + return fdopen_pipe (); + } + catch (const io_error& e) + { + fail << "unable to open pipe: " << e << endf; + } + } + + auto_fd + open_dev_null () + { + try + { + return fdnull (); + } + catch (const io_error& e) + { + fail << "unable to open null device: " << e << endf; + } + } + dir_path exec_dir; void diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx index 05835b0..b05f668 100644 --- a/bpkg/utility.hxx +++ b/bpkg/utility.hxx @@ -116,6 +116,14 @@ namespace bpkg void mv (const dir_path& from, const dir_path& to); + // File descriptor streams. + // + fdpipe + open_pipe (); + + auto_fd + open_dev_null (); + // Process. // // By default the process command line is printed for verbosity >= 2 -- cgit v1.1