diff options
21 files changed, 298 insertions, 85 deletions
diff --git a/bpkg/pkg-verify.cxx b/bpkg/pkg-verify.cxx index 4dea4fa..c20a393 100644 --- a/bpkg/pkg-verify.cxx +++ b/bpkg/pkg-verify.cxx @@ -224,7 +224,7 @@ namespace bpkg package_manifest m (pkg_verify (o, a, o.ignore_unknown (), - o.deep () /* expand_values */, + o.deep () /* expand_values */, o.deep () /* complete_depends */, !o.silent ())); diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 0c2ed89..fd47198 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -263,10 +263,47 @@ namespace bpkg return r; } + // Return contents of a file referenced by a *-file package manifest value. + // + static string + read_package_file (const path& f, + const string& name, + const dir_path& pkg, + const dir_path& repo, + const repository_location& rl, + const string& fragment) + { + path rp (pkg / f); + path fp (repo / rp); + + try + { + ifdstream is (fp); + string s (is.read_text ()); + + if (s.empty ()) + fail << name << " manifest value in " << pkg / manifest_file + << " references empty file " << rp << + info << "repository " << rl + << (!fragment.empty () ? " " + fragment : ""); + + return s; + } + catch (const io_error& e) + { + fail << "unable to read from " << rp << " referenced by " + << name << " manifest value in " << pkg / manifest_file << ": " + << e << + info << "repository " << rl + << (!fragment.empty () ? " " + fragment : "") << endf; + } + } + static rep_fetch_data rep_fetch_dir (const common_options& co, const repository_location& rl, - bool ignore_unknown) + bool iu, + bool ev) { assert (rl.absolute ()); @@ -276,23 +313,41 @@ namespace bpkg fr.repositories = parse_repository_manifests<dir_repository_manifests> ( rd / repositories_file, - ignore_unknown, + iu, rl, string () /* fragment */); dir_package_manifests pms ( parse_directory_manifests<dir_package_manifests> ( rd / packages_file, - ignore_unknown, + iu, rl, string () /* fragment */)); fr.packages = parse_package_manifests (co, rd, move (pms), - ignore_unknown, + iu, rl, - string () /* fragment */); + empty_string /* fragment */); + + // Expand file-referencing package manifest values. + // + if (ev) + { + for (package_manifest& m: fr.packages) + m.load_files ( + [&m, &rd, &rl] (const string& n, const path& p) + { + return read_package_file (p, + n, + path_cast<dir_path> (*m.location), + rd, + rl, + empty_string /* fragment */); + }, + iu); + } return rep_fetch_data {{move (fr)}, nullopt /* cert_pem */, @@ -303,7 +358,8 @@ namespace bpkg rep_fetch_git (const common_options& co, const dir_path* conf, const repository_location& rl, - bool ignore_unknown) + bool iu, + bool ev) { if (conf != nullptr && conf->empty ()) conf = dir_exists (bpkg_dir) ? ¤t_dir : nullptr; @@ -388,7 +444,7 @@ namespace bpkg // fr.repositories = parse_repository_manifests<git_repository_manifests> ( td / repositories_file, - ignore_unknown, + iu, rl, fr.friendly_name); @@ -397,11 +453,23 @@ namespace bpkg git_package_manifests pms ( parse_directory_manifests<git_package_manifests> ( td / packages_file, - ignore_unknown, + iu, rl, fr.friendly_name)); - // Checkout submodules, if required. + // Checkout submodules on the first call. + // + bool cs (true); + auto checkout_submodules = [&co, &rl, &td, &cs] () + { + if (cs) + { + git_checkout_submodules (co, rl, td); + cs = false; + } + }; + + // Checkout submodules to parse package manifests, if required. // for (const package_manifest& sm: pms) { @@ -409,7 +477,7 @@ namespace bpkg if (!exists (d) || empty (d)) { - git_checkout_submodules (co, rl, td); + checkout_submodules (); break; } } @@ -419,9 +487,41 @@ namespace bpkg fr.packages = parse_package_manifests (co, td, move (pms), - ignore_unknown, + iu, rl, fr.friendly_name); + + // Expand file-referencing package manifest values checking out + // submodules, if required. + // + if (ev) + { + for (package_manifest& m: fr.packages) + m.load_files ( + [&m, &td, &rl, &fr, &checkout_submodules] (const string& n, + const path& p) + { + // Note that this doesn't work for symlinks on Windows where git + // normally creates filesystem-agnostic symlinks that are + // indistinguishable from regular files (see fixup_worktree() + // for details). It seems like the only way to deal with that is + // to unconditionally checkout submodules on Windows. Let's not + // pessimize things for now (if someone really wants this to + // work, they can always enable real symlinks in git). + // + if (!exists (td / *m.location / p)) + checkout_submodules (); + + return read_package_file (p, + n, + path_cast<dir_path> (*m.location), + td, + rl, + fr.friendly_name); + }, + iu); + } + np += fr.packages.size (); r.fragments.push_back (move (fr)); @@ -452,13 +552,14 @@ namespace bpkg const dir_path* conf, const repository_location& rl, const optional<string>& dt, - bool iu) + bool iu, + bool ev) { switch (rl.type ()) { case repository_type::pkg: return rep_fetch_pkg (co, conf, rl, dt, iu); - case repository_type::dir: return rep_fetch_dir (co, rl, iu); - case repository_type::git: return rep_fetch_git (co, conf, rl, iu); + case repository_type::dir: return rep_fetch_dir (co, rl, iu, ev); + case repository_type::git: return rep_fetch_git (co, conf, rl, iu, ev); } assert (false); // Can't be here. @@ -469,9 +570,10 @@ namespace bpkg rep_fetch (const common_options& co, const dir_path* conf, const repository_location& rl, - bool iu) + bool iu, + bool ev) { - return rep_fetch (co, conf, rl, nullopt /* dependent_trust */, iu); + return rep_fetch (co, conf, rl, nullopt /* dependent_trust */, iu, ev); } // Return an existing repository fragment or create a new one. Update the @@ -949,7 +1051,12 @@ namespace bpkg // repository sets. // rep_fetch_data rfd ( - rep_fetch (co, &conf, rl, dependent_trust, true /* ignore_unknow */)); + rep_fetch (co, + &conf, + rl, + dependent_trust, + true /* ignore_unknow */, + false /* expand_values */)); // Save for subsequent certificate authentication for repository use by // its dependents. diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx index 24d37d3..1ed6a22 100644 --- a/bpkg/rep-fetch.hxx +++ b/bpkg/rep-fetch.hxx @@ -47,11 +47,18 @@ namespace bpkg shared_ptr<const bpkg::certificate> certificate; // Authenticated. }; + // If requested, expand the file-referencing package manifest values + // (description, changes, etc), setting them to the contents of files they + // refer to and set the potentially absent description-type value to the + // effective description type (see libbpkg/manifest.hxx). Note that for pkg + // repositories such values are expanded at the repository creation time. + // rep_fetch_data rep_fetch (const common_options&, const dir_path* conf, const repository_location&, - bool ignore_unknown); + bool ignore_unknown, + bool expand_values); // Add (or update) repository locations to the configuration and fetch // them. If shallow is true, then don't fetch their prerequisite and/or diff --git a/bpkg/rep-info.cli b/bpkg/rep-info.cli index 0e10778..916209f 100644 --- a/bpkg/rep-info.cli +++ b/bpkg/rep-info.cli @@ -90,8 +90,15 @@ namespace bpkg "Instead of printing the information in the human-readable form, dump it (to \cb{stdout}) as manifest(s). Normally you would use this option in combination with \cb{--packages|-p} or \cb{--repositories|-r} to only - dump one of the manifests. See also \cb{--repositories-file} and - \cb{--packages-file}." + dump one of the manifests. If the \cb{--deep} option is specified, + then in the resulting packages manifest the \cb{*-file} values are + replaced with the contents of the referenced files. See also + \cb{--repositories-file} and \cb{--packages-file}." + } + + bool --deep + { + "Verify files referenced by the \cb{*-file} manifest values." } path --repositories-file diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx index 8caabfd..1d3733d 100644 --- a/bpkg/rep-info.cxx +++ b/bpkg/rep-info.cxx @@ -55,7 +55,8 @@ namespace bpkg rep_fetch (o, o.directory_specified () && d.empty () ? nullptr : &d, rl, - !o.manifest () /* ignore_unknow */)); + !o.manifest () /* ignore_unknow */, + o.deep () /* expand_values */)); // Now print. // diff --git a/tests/common/git/init b/tests/common/git/init index 5153175..9519779 100755 --- a/tests/common/git/init +++ b/tests/common/git/init @@ -65,6 +65,8 @@ rm -f libfox.git/.gitmodules rm -f -r libfox.git/libbar rm -f -r libbar.git/.git +rm -f libbar.git/.gitmodules +rm -f -r libbar.git/extras rm -f -r style.git/.git rm -f -r style.git/basic @@ -85,7 +87,7 @@ git -C style-basic.git commit -am 'Create' sleep 1 # Make sure that master commits are older than stable commits. git -C style-basic.git branch stable git -C style-basic.git checkout stable -touch style-basic.git/README +echo "TODO" >style-basic.git/README cat <<EOF >style-basic.git/repositories.manifest : 1 email: user@example.com @@ -112,6 +114,7 @@ git -C style-basic.git checkout master # git -C libbar.git init git -C libbar.git add '*' +git -C libbar.git submodule add -b stable ../style-basic.git extras git -C libbar.git commit -am 'Create' # Create master branch for libfoo.git, adding style.git and libbar.git as diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar Binary files differindex 027112c..b63a7b5 100644 --- a/tests/common/git/state0/libbar.tar +++ b/tests/common/git/state0/libbar.tar diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar Binary files differindex 96a9f5b..200ebf5 100644 --- a/tests/common/git/state0/libfoo.tar +++ b/tests/common/git/state0/libfoo.tar diff --git a/tests/common/git/state0/libfox.tar b/tests/common/git/state0/libfox.tar Binary files differindex fc2c391..df77539 100644 --- a/tests/common/git/state0/libfox.tar +++ b/tests/common/git/state0/libfox.tar diff --git a/tests/common/git/state0/links.tar b/tests/common/git/state0/links.tar Binary files differindex 33c5dbf..25c4089 100644 --- a/tests/common/git/state0/links.tar +++ b/tests/common/git/state0/links.tar diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar Binary files differindex 8b57bd0..7223c52 100644 --- a/tests/common/git/state0/style-basic.tar +++ b/tests/common/git/state0/style-basic.tar diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar Binary files differindex 56e29f0..93007d5 100644 --- a/tests/common/git/state0/style.tar +++ b/tests/common/git/state0/style.tar diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar Binary files differindex 7acf277..d46b25c 100644 --- a/tests/common/git/state1/libbaz.tar +++ b/tests/common/git/state1/libbaz.tar diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar Binary files differindex 532e974..683f5ab 100644 --- a/tests/common/git/state1/libfoo.tar +++ b/tests/common/git/state1/libfoo.tar diff --git a/tests/common/git/state1/libfox.tar b/tests/common/git/state1/libfox.tar Binary files differindex ec49a86..49c4752 100644 --- a/tests/common/git/state1/libfox.tar +++ b/tests/common/git/state1/libfox.tar diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar Binary files differindex 1946606..5ab346d 100644 --- a/tests/common/git/state1/style-basic.tar +++ b/tests/common/git/state1/style-basic.tar diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar Binary files differindex 769b6d5..71be292 100644 --- a/tests/common/git/state1/style.tar +++ b/tests/common/git/state1/style.tar diff --git a/tests/pkg-checkout.testscript b/tests/pkg-checkout.testscript index a284f6b..86ae341 100644 --- a/tests/pkg-checkout.testscript +++ b/tests/pkg-checkout.testscript @@ -55,7 +55,8 @@ else $rep_add "$rep/libbar.git#master"; $rep_fetch; - $* libmbar/1.0.0 2>>EOE != 0 + $* libmbar/1.0.0 2>>~%EOE% != 0 + %.+ error: no configured package satisfies dependency on style-basic >= 1.0.0 warning: repository state is now broken info: run 'bpkg rep-fetch' to repair @@ -78,7 +79,9 @@ else $pkg_configure style-basic; - $* libmbar/1.0.0 2>>EOE; + $* libmbar/1.0.0 2>>~%EOE%; + checking out libmbar/1.0.0 + %.+ distributing libmbar/1.0.0 checked out libmbar/1.0.0 EOE diff --git a/tests/rep-fetch-git-refname.testscript b/tests/rep-fetch-git-refname.testscript index 71305ea..37fdcba 100644 --- a/tests/rep-fetch-git-refname.testscript +++ b/tests/rep-fetch-git-refname.testscript @@ -62,6 +62,7 @@ warn1 = '%.{0}'; warn2 = '%.{0}'; warn3 = '%.{0}'; + warn4 = '%.{0}'; if ($git_protocol == 'local' || \ $git_protocol == 'https-smart' || \ @@ -72,6 +73,7 @@ warn1 = "$warn_ref_hist for submodule 'doc/style'$reason_dumb$warn_dumb" warn2 = "$warn_repo_hist for submodule 'doc/style/basic'$reason_dumb$warn_dumb" warn3 = "$warn_ref_hist for submodule 'libbar'$reason_dumb$warn_dumb" + warn4 = "$warn_ref_hist for submodule 'libbar/extras'$reason_dumb$warn_dumb" end; $pkg_checkout libfoo/1.0.0 2>>~"%EOE%"; @@ -88,6 +90,9 @@ %fetching submodule 'libbar' from .+libbar\.git% $warn3 %submodule path 'libbar': checked out .+% + %fetching submodule 'libbar/extras' from .+style-basic\.git+% + $warn4 + %submodule path 'libbar/extras': checked out .+% distributing libfoo/1.0.0 checked out libfoo/1.0.0 EOE diff --git a/tests/rep-fetch.testscript b/tests/rep-fetch.testscript index bd67bc1..8e7011a 100644 --- a/tests/rep-fetch.testscript +++ b/tests/rep-fetch.testscript @@ -792,6 +792,10 @@ else %fetching submodule 'libbar' from .+libbar\.git% %warning: .+%{0,2} %submodule path 'libbar': checked out '.+'% + %querying .+style-basic\.git%? + %fetching submodule 'libbar/extras' from .+style-basic\.git% + %warning: .+%{0,2} + %submodule path 'libbar/extras': checked out '.+'% 2 package(s) in 1 repository(s) EOE } diff --git a/tests/rep-info.testscript b/tests/rep-info.testscript index f72d9d4..1a28dd9 100644 --- a/tests/rep-info.testscript +++ b/tests/rep-info.testscript @@ -44,12 +44,14 @@ end test.options += --auth all --trust-yes +posix = ($cxx.target.class != 'windows') + : no-location : $* 2>>EOE != 0 -error: repository location argument expected - info: run 'bpkg help rep-info' for more information -EOE + error: repository location argument expected + info: run 'bpkg help rep-info' for more information + EOE : default : @@ -57,24 +59,24 @@ EOE : unsigned : $* $rep/testing >>"EOO" - pkg:build2.org/rep-info/testing ($rep/testing) - prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) - complement pkg:build2.org/rep-info/stable ($rep/stable) + pkg:build2.org/rep-info/testing ($rep/testing) + prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) + complement pkg:build2.org/rep-info/stable ($rep/stable) - foo/1 - EOO + foo/1 + EOO : signed : $* $rep/signed >>"EOO" - pkg:build2.org/rep-info/signed ($rep/signed) - CN=build2.org/O=Code Synthesis/info@build2.org - $cert_fp - prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) - complement pkg:build2.org/rep-info/stable ($rep/stable) - - foo/1 - EOO + pkg:build2.org/rep-info/signed ($rep/signed) + CN=build2.org/O=Code Synthesis/info@build2.org + $cert_fp + prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) + complement pkg:build2.org/rep-info/stable ($rep/stable) + + foo/1 + EOO } : name @@ -88,24 +90,96 @@ $* --name $rep/testing >"pkg:build2.org/rep-info/testing ($rep/testing)" : list : - $* $rep/testing >>EOO + $* $rep/testing >>EOO - foo/1 - EOO + foo/1 + EOO : manifest : - $* --manifest $rep/testing >>EOO - : 1 - name: foo - version: 1 - summary: The "Foo" utility - license: MIT - url: http://www.example.org/foo - email: foo-users@example.org - location: foo-1.tar.gz - sha256sum: fee330a362a4f87ff42a954aa305b6446d541b7b60000ebcd2fbf68f2b1ae58e - EOO + { + test.arguments += --manifest + + : basic + : + $* $rep/testing >>EOO + : 1 + name: foo + version: 1 + summary: The "Foo" utility + license: MIT + url: http://www.example.org/foo + email: foo-users@example.org + location: foo-1.tar.gz + sha256sum: fee330a362a4f87ff42a954aa305b6446d541b7b60000ebcd2fbf68f2b1ae58e + EOO + + : deep + : + { + test.arguments += --deep + + : dir + : + { + tar ($posix ? : --force-local) -xf $src/git/libbar.tar &state0/***; + + $* --type dir "state0/libbar.git" >>~%EOO%d; + : 1 + name: libbar + version: 1.0.0 + summary: libbar + license: MIT + description: \ + TODO + + \ + description-type: text/plain + %.+ + EOO + + rm state0/libbar.git/libbar/README; + + $* --type dir "state0/libbar.git" 2>>/~%EOE% != 0 + %error: unable to read from libbar/README referenced by description-file manifest value in libbar/manifest: .+% + % info: repository .+% + EOE + } + + : git + : + : Enable on Windows if/when we make *-file values expansion to properly + : work for symlinks on Windows (see rep_fetch_git() for details). + : + if ($git_supported && $posix) + { + $* "$rep_git/state0/libbar.git#master" >>~%EOO%d 2>! + : 1 + name: libbar + version: 1.0.0 + summary: libbar + license: MIT + description: \ + TODO + + \ + description-type: text/plain + %.+ + : + name: libmbar + version: 1.0.0 + summary: libmbar + license: MIT + description: \ + TODO + + \ + description-type: text/plain + %.+ + EOO + } + } + } : to-file : @@ -113,9 +187,9 @@ $* --name $rep/testing >"pkg:build2.org/rep-info/testing ($rep/testing)" : no-manifest : $* --packages-file m $rep/testing 2>>EOE != 0 - error: --packages-file specified without --manifest - info: run 'bpkg help rep-info' for more information - EOE + error: --packages-file specified without --manifest + info: run 'bpkg help rep-info' for more information + EOE : manifest : @@ -147,23 +221,23 @@ $* --name $rep/testing >"pkg:build2.org/rep-info/testing ($rep/testing)" : list : $* $rep/testing >>"EOO" - prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) - complement pkg:build2.org/rep-info/stable ($rep/stable) - EOO + prerequisite pkg:build2.org/foo/testing ($rep_root/foo/testing) + complement pkg:build2.org/rep-info/stable ($rep/stable) + EOO : manifest : $* --manifest $rep/testing >>EOO - : 1 - location: ../../foo/testing - type: pkg - role: prerequisite - : - location: ../stable - type: pkg - role: complement - : - EOO + : 1 + location: ../../foo/testing + type: pkg + role: prerequisite + : + location: ../stable + type: pkg + role: complement + : + EOO : to-file : @@ -171,9 +245,9 @@ $* --name $rep/testing >"pkg:build2.org/rep-info/testing ($rep/testing)" : no-manifest : $* --repositories-file m $rep/testing 2>>EOE != 0 - error: --repositories-file specified without --manifest - info: run 'bpkg help rep-info' for more information - EOE + error: --repositories-file specified without --manifest + info: run 'bpkg help rep-info' for more information + EOE : manifest : @@ -223,10 +297,10 @@ else : Version module is enabled for the project. : $* "$rep/style-basic.git#master" >>~%EOO% - %git:.+style-basic#master .+style-basic.git#master% + %git:.+style-basic#master .+style-basic.git#master% - %style-basic/1\.1\.0-a\.0\.\d+\..+% - EOO + %style-basic/1\.1\.0-a\.0\.\d+\..+% + EOO : manifest-lists : @@ -234,12 +308,12 @@ else : repository root. : $* "$rep/libbar.git#master" >>~%EOO% - %git:.+libbar#master .+libbar.git#master% - %prerequisite git:.+style-basic#stable .+style-basic.git#stable% + %git:.+libbar#master .+libbar.git#master% + %prerequisite git:.+style-basic#stable .+style-basic.git#stable% - libbar/1.0.0 - libmbar/1.0.0 - EOO + libbar/1.0.0 + libmbar/1.0.0 + EOO : repository-manifests : @@ -248,9 +322,9 @@ else : master branch. : $* -r --manifest "$rep/style-basic.git#stable,master" >>~%EOO% - : 1 - email: user@example.com - EOO + : 1 + email: user@example.com + EOO : complete-dependency : @@ -260,6 +334,7 @@ else version: 1.0.0 summary: libbar license: MIT + description-file: README url: http://example.org/libbar email: pkg@example.org depends: style-basic >= 1.0.0 @@ -270,6 +345,7 @@ else version: 1.0.0 summary: libmbar license: MIT + description-file: README url: http://example.org/libmbar email: pkg@example.org depends: style-basic >= 1.0.0 |