From 3d4838d3706de2ba0045dc9f99a3dc96398def64 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 19 Feb 2018 14:26:02 +0300 Subject: Parse repositories and packages files for git repositories --- bpkg/rep-fetch.cxx | 267 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 240 insertions(+), 27 deletions(-) (limited to 'bpkg/rep-fetch.cxx') diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index d7f0386..029db1d 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -5,6 +5,9 @@ #include #include +#include +#include // operator<<(ostream, process_path) +#include #include #include @@ -74,30 +77,64 @@ namespace bpkg return rep_fetch_data {move (rms), move (pms), move (cert)}; } + template + static M + parse_manifest (const path& f, bool iu, const repository_location& rl) + { + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f.string ()); + return M (mp, iu); + } + catch (const manifest_parsing& e) + { + fail (e.name, e.line, e.column) << e.description << + info << "repository " << rl << endf; + } + catch (const io_error& e) + { + fail << "unable to read from " << f << ": " << e << + info << "repository " << rl << endf; + } + } + static rep_fetch_data rep_fetch_git (const common_options& co, const dir_path* conf, const repository_location& rl, - bool /* ignore_unknown */) + bool ignore_unknown) { // Plan: // // 1. Check repos_dir//: // - // 1.a If does not exist, git-clone into temp_dir//. + // 1.a If does not exist, git-clone into temp_dir///. // // 1.a Otherwise, move as temp_dir// and git-fetch. // - // 2. Move from temp_dir// to repos_dir// + // 2. Move from temp_dir// to repos_dir/// // - // 3. Load manifest from repos_dir/// + // 3. Check if repos_dir///repositories exists: // - // 4. Run 'b info' in repos_dir/// and fix-up - // package version. + // 3.a If exists, load. // - // 5. Synthesize repository manifest. + // 3.b Otherwise, synthesize repository list with base repository. // - // 6. Return repository and package manifest (certificate is NULL). + // 4. Check if repos_dir///packages exists: + // + // 4.a If exists, load. (into "skeleton" packages list to be filled?) + // + // 4.b Otherwise, synthesize as if single 'location: ./'. + // + // 5. For each package location obtained on step 4: + // + // 5.a Load repos_dir////manifest. + // + // 5.b Run 'b info: repos_dir////' and fix-up + // package version. + // + // 6. Return repository and package manifests (certificate is NULL). // if (conf != nullptr && conf->empty ()) @@ -105,6 +142,8 @@ namespace bpkg assert (conf == nullptr || !conf->empty ()); + // Clone or fetch the repository. + // dir_path h (sha256 (rl.canonical_name ()).abbreviated_string (16)); auto_rmdir rm (temp_dir / h); @@ -129,10 +168,7 @@ namespace bpkg } } - if (fetch) - git_fetch (co, rl, td); - else - git_clone (co, rl, td); + dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); if (!rd.empty ()) mv (td, rd); @@ -144,9 +180,179 @@ namespace bpkg rm.cancel (); - // @@ TODO + rd /= nm; + + // Produce repository manifest list. // - return rep_fetch_data (); + git_repository_manifests rms; + { + path f (rd / path ("repositories")); + + if (exists (f)) + rms = parse_manifest (f, ignore_unknown, rl); + else + rms.emplace_back (repository_manifest ()); // Add the base repository. + } + + // Produce the "skeleton" package manifest list. + // + git_package_manifests pms; + { + path f (rd / path ("packages")); + + if (exists (f)) + pms = parse_manifest (f, ignore_unknown, rl); + else + { + pms.push_back (package_manifest ()); + pms.back ().location = current_dir; + } + } + + // Fill "skeleton" package manifests. + // + for (package_manifest& sm: pms) + { + assert (sm.location); + + auto package_info = [&sm, &rl] (diag_record& dr) + { + dr << "package "; + + if (!sm.location->current ()) + dr << "'" << sm.location->string () << "' "; // Strip trailing '/'. + + dr << "in repository " << rl; + }; + + auto failure = [&package_info] (const char* desc) + { + diag_record dr (fail); + dr << desc << " for "; + package_info (dr); + }; + + dir_path d (rd / path_cast (*sm.location)); + path f (d / path ("manifest")); + + if (!exists (f)) + failure ("no manifest file"); + + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f.string ()); + package_manifest m (bpkg_package_manifest (mp, ignore_unknown)); + + // Save the package manifest, preserving its location. + // + m.location = move (sm.location); + sm = move (m); + } + catch (const manifest_parsing& e) + { + diag_record dr (fail (e.name, e.line, e.column)); + dr << e.description << info; + package_info (dr); + } + catch (const io_error& e) + { + diag_record dr (fail); + dr << "unable to read from " << f << ": " << e << info; + package_info (dr); + } + + // Fix-up the package version. + // + const char* b (name_b (co)); + + try + { + process_path pp (process::path_search (b, exec_dir)); + + fdpipe pipe (open_pipe ()); + + process pr ( + process_start_callback ( + [] (const char* const args[], size_t n) + { + if (verb >= 2) + print_process (args, n); + }, + 0 /* stdin */, pipe /* stdout */, 2 /* stderr */, + pp, + + verb < 2 + ? strings ({"-q"}) + : verb == 2 + ? strings ({"-v"}) + : strings ({"--verbose", to_string (verb)}), + + co.build_option (), + "info:", + d.representation ())); + + // Shouldn't throw, unless something is severely damaged. + // + pipe.out.close (); + + try + { + ifdstream is (move (pipe.in), + fdstream_mode::skip, + ifdstream::badbit); + + for (string l; !eof (getline (is, l)); ) + { + if (l.compare (0, 9, "version: ") == 0) + try + { + string v (l, 9); + + // An empty version indicates that the version module is not + // enabled for the project, and so we don't amend the package + // version. + // + if (!v.empty ()) + sm.version = version (v); + + break; + } + catch (const invalid_argument&) + { + fail << "no package version in '" << l << "'" << + info << "produced by '" << pp << "'; use --build to override"; + } + } + + is.close (); + + if (pr.wait ()) + continue; // Go to the next package. + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + failure ("unable to read information"); + + // Fall through. + } + + // We should only get here if the child exited with an error status. + // + assert (!pr.wait ()); + + failure ("unable to obtain information"); + } + catch (const process_error& e) + { + fail << "unable to execute " << b << ": " << e; + } + } + + return rep_fetch_data {move (rms), move (pms), nullptr}; } rep_fetch_data @@ -319,21 +525,28 @@ namespace bpkg { // Make sure this is the same package. // - assert (p->sha256sum && !p->locations.empty ()); // Can't be transient. + assert (!p->locations.empty ()); // Can't be transient. - if (*pm.sha256sum != *p->sha256sum) + // Note that sha256sum may not present for some repository types. + // + if (pm.sha256sum) { - // All the previous repositories that contain this package have the - // same checksum (since they passed this test), so we can pick any - // to show to the user. - // - const string& r1 (rl.canonical_name ()); - const string& r2 (p->locations[0].repository.object_id ()); - - fail << "checksum mismatch for " << pm.name << " " << pm.version << - info << r1 << " has " << *pm.sha256sum << - info << r2 << " has " << *p->sha256sum << - info << "consider reporting this to the repository maintainers"; + if (!p->sha256sum) + p->sha256sum = move (pm.sha256sum); + else if (*pm.sha256sum != *p->sha256sum) + { + // All the previous repositories that have checksum for this + // package have it the same (since they passed this test), so we + // can pick any to show to the user. + // + const string& r1 (rl.canonical_name ()); + const string& r2 (p->locations[0].repository.object_id ()); + + fail << "checksum mismatch for " << pm.name << " " << pm.version << + info << r1 << " has " << *pm.sha256sum << + info << r2 << " has " << *p->sha256sum << + info << "consider reporting this to the repository maintainers"; + } } } -- cgit v1.1