aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2023-03-01 16:13:53 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2023-03-01 16:13:53 +0200
commit35bbf1d79dfcc2cdf5be5f457639550f06a51bc5 (patch)
tree9c9e48528f77a3f38d5317bffdf673508534d7ae
parentba7d2df9b8c834f2f4e3d2715f94503347c4630e (diff)
WIP (use installation manifest)
-rw-r--r--bpkg/system-package-manager-debian.cxx205
-rw-r--r--bpkg/system-package-manager.cxx235
-rw-r--r--bpkg/system-package-manager.hxx48
-rw-r--r--bpkg/utility.txx3
4 files changed, 414 insertions, 77 deletions
diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index ba0db91..07548f8 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -1848,7 +1848,7 @@ namespace bpkg
void system_package_manager_debian::
generate (const packages& pkgs,
const packages& deps,
- const strings&,
+ const strings& vars,
const package_manifest& pm,
const string& pt,
const small_vector<language, 1>& langs,
@@ -1956,6 +1956,83 @@ namespace bpkg
}
}
+ // We override every config.install.* variable in order not to pick
+ // anything configured. Note that we add some more in the rules file
+ // below.
+ //
+ // We make use of the <project> substitution since in the recursive mode
+ // we may be installing multiple projects. Note that the <private>
+ // directory component is automatically removed if this functionality is
+ // not enabled. One side-effect of using <project> is that we will be
+ // using the bpkg package name instead of the main Debian package name.
+ // But perhaps that's correct: on Debian it's usually the source package
+ // name, which is the same. To keep things consistent we use the bpkg
+ // package name for <private> as well.
+ //
+ // @@ Some libraries install what looks like architecture-specific
+ // configuration files to /usr/include/$(DEB_HOST_MULTIARCH). Maybe we
+ // should invent something like config.install.include_arch to support
+ // this distinction?
+ //
+ // NOTE: make sure to update .install files below if changing anyting
+ // here.
+ //
+ // Note: we need to quote values that contain `$` so that they don't get
+ // expanded as build2 variables in the installed_entries() call.
+ //
+ strings config {
+ "config.install.root=/usr/",
+ "config.install.data_root=root/",
+ "config.install.exec_root=root/",
+
+ "config.install.bin=exec_root/bin/",
+ "config.install.sbin=exec_root/sbin/",
+
+ // On Debian shared libraries should not be executable. Also,
+ // libexec/ is the same as lib/ (note that executables that get
+ // installed there will still have the executable bit set).
+ //
+ "config.install.lib='exec_root/lib/$(DEB_HOST_MULTIARCH)/<private>/'",
+ "config.install.lib.mode=644",
+ "config.install.libexec=lib/<project>/",
+ "config.install.pkgconfig=lib/pkgconfig/",
+
+ "config.install.etc=data_root/etc/",
+ "config.install.include=data_root/include/<private>/",
+ "config.install.share=data_root/share/",
+ "config.install.data=share/<private>/<project>/",
+
+ "config.install.doc=share/doc/<private>/<project>/",
+ "config.install.legal=doc/",
+ "config.install.man=share/man/",
+ "config.install.man1=man/man1/",
+ "config.install.man2=man/man2/",
+ "config.install.man3=man/man3/",
+ "config.install.man4=man/man4/",
+ "config.install.man5=man/man5/",
+ "config.install.man6=man/man6/",
+ "config.install.man7=man/man7/",
+ "config.install.man8=man/man8/"};
+
+ config.push_back ("config.install.private=" +
+ (priv ? pn.string () : "[null]"));
+
+ // Add user-specified configuration variables last to allow them to
+ // override anything.
+ //
+ for (const string& v: vars)
+ config.push_back (v);
+
+ // Get the map of files that will end up in the binary packages.
+ //
+ // Note that we are passing quoted values with $(DEB_HOST_MULTIARCH) which
+ // will be treated literally.
+ //
+ installed_entry_map ies (installed_entries (*ops_, pkgs, config));
+
+ if (ies.empty ())
+ fail << "specified package(s) do not install any files";
+
// Start assembling the package "source" directory.
//
// It's hard to predict all the files that will be generated (and
@@ -2504,7 +2581,12 @@ namespace bpkg
// (probably because stderr redirected to pipe). @@ No, there is
// progress. Maybe just keep, doesn't seem harmful. Or pass ours.
//
- os << "b := " << search_b (*ops_).effect_string () << '\n'
+ // Note: should be consistent with the invocation in installed_entries()
+ // above.
+ //
+ os << "b := " << search_b (*ops_).effect_string ();
+ for (const string& o: ops_->build_option ()) os << ' ' << o;
+ os << '\n'
<< '\n'
<< "parallel := $(filter parallel=%,$(DEB_BUILD_OPTIONS))" << '\n'
<< "ifneq ($(parallel),)" << '\n'
@@ -2517,70 +2599,47 @@ namespace bpkg
<< "endif" << '\n'
<< '\n';
- // Note that we override every config.install.* variable in order not to
- // pick anything configured.
- //
- // We make use of the <project> substitution since in the recursive mode
- // we may be installing multiple projects. Note that the <private>
- // directory component is automatically removed if this functionality is
- // not enabled. One side-effect of using <project> is that we will be
- // using the bpkg package name instead of the main Debian package name.
- // But perhaps that's correct: on Debian it's usually the source package
- // name, which is the same. To keep things consistent we use the bpkg
- // package name for <private> as well.
- //
- // @@ Some libraries install what looks like architecture-specific
- // configuration files to /usr/include/$(DEB_HOST_MULTIARCH). Maybe
- // we should invent something like config.install.include_arch to
- // support this distinction?
- //
- // NOTE: make sure to update .install files below if changing anyting
- // here.
- //
- os << "config := config.install.chroot=$(DESTDIR)/" << '\n'
- << "config += 'config.install.sudo=[null]'" << '\n'
-
- << "config += config.install.root=/usr/" << '\n'
- << "config += config.install.data_root=root/" << '\n'
- << "config += config.install.exec_root=root/" << '\n'
-
- << "config += config.install.bin=exec_root/bin/" << '\n'
- << "config += config.install.sbin=exec_root/sbin/" << '\n'
-
- // On Debian shared libraries should not be executable. Also,
- // libexec/ is the same as lib/ (note that executables that get
- // installed there will still have the executable bit set).
- //
- << "config += 'config.install.lib=exec_root/lib/$(DEB_HOST_MULTIARCH)/<private>/'" << '\n'
- << "config += config.install.lib.mode=644" << '\n'
- << "config += 'config.install.libexec=lib/<project>/'" << '\n'
- << "config += config.install.pkgconfig=lib/pkgconfig/" << '\n'
-
- << "config += config.install.etc=data_root/etc/" << '\n'
- << "config += 'config.install.include=data_root/include/<private>/'" << '\n'
- << "config += config.install.share=data_root/share/" << '\n'
- << "config += 'config.install.data=share/<private>/<project>/'" << '\n'
-
- << "config += 'config.install.doc=share/doc/<private>/<project>/'" << '\n'
- << "config += config.install.legal=doc/" << '\n'
- << "config += config.install.man=share/man/" << '\n'
- << "config += config.install.man1=man/man1/" << '\n'
- << "config += config.install.man2=man/man2/" << '\n'
- << "config += config.install.man3=man/man3/" << '\n'
- << "config += config.install.man4=man/man4/" << '\n'
- << "config += config.install.man5=man/man5/" << '\n'
- << "config += config.install.man6=man/man6/" << '\n'
- << "config += config.install.man7=man/man7/" << '\n'
- << "config += config.install.man8=man/man8/" << '\n'
-
- << "config += 'config.install.private="
- << (priv ? pn.string () : "[null]") << "'" << '\n';
+ // Configuration variables.
+ //
+ // Note: we need to quote values that contain `<>`, `[]`, since they
+ // will be passed through shell. For simplicity, let's just quote
+ // everything.
+ //
+ os << "config := config.install.chroot='$(DESTDIR)/'" << '\n'
+ << "config += config.install.sudo='[null]'" << '\n';
// If this is a C-based language, add rpath for private installation.
//
if (priv && (lang ("c") || lang ("c++") || lang ("cc")))
- os << "config += config.bin.rpath=/usr/lib/$(DEB_HOST_MULTIARCH)/"
- << pn << "/" << '\n';
+ os << "config += config.bin.rpath='/usr/lib/$(DEB_HOST_MULTIARCH)/"
+ << pn << "/'" << '\n';
+
+ // Keep last to allow user-specified configuration variables to override
+ // anything.
+ //
+ for (const string& c: config)
+ {
+ // Quote the value unless already quoted (see above). Presense of
+ // potentially-quoted user variables complicates things a bit (can
+ // be partially quoted, double-quoted, etc).
+ //
+ size_t p (c.find_first_of ("=+ \t")); // End of name.
+ if (p != string::npos)
+ {
+ p = c.find_first_not_of ("=+ \t", p); // Beginning of value.
+ if (p != string::npos)
+ {
+ if (c.find_first_of ("'\"", p) == string::npos) // Not quoted.
+ {
+ os << "config += " << string (c, 0, p) << '\''
+ << string (c, p) << "'\n";
+ continue;
+ }
+ }
+ }
+
+ os << "config += " << c << '\n';
+ }
os << '\n';
@@ -2687,11 +2746,6 @@ namespace bpkg
// variable, we can end up with multiple different directories (bundled
// package).
//
- string privdir (priv ? pn.string () + '/' : "");
- string libdir ("usr/lib/${DEB_HOST_MULTIARCH}/" + privdir);
- string incdir ("usr/include/" + privdir);
- string docdir ("usr/share/doc/" + privdir);
-
path main_install (deb / (st.main + ".install"));
try
{
@@ -2700,17 +2754,16 @@ namespace bpkg
// The main package contains everything that doesn't go to another
// package.
//
- os//<< "usr/bin/*" << '\n'
- //<< "usr/sbin/*" << '\n'
+ if (ies.contains ("/usr/bin/")) os << "usr/bin/*" << '\n';
+ if (ies.contains ("/usr/sbin/")) os << "usr/sbin/*" << '\n';
- << libdir << '\n'
+ if (ies.contains ("/usr/lib/$(DEB_HOST_MULTIARCH)/"))
+ os << "usr/lib/${DEB_HOST_MULTIARCH}/*" << '\n';
- << incdir << '\n'
+ if (ies.contains ("/usr/include/")) os << "usr/include/*" << '\n';
- << docdir << "*" << '\n'
- //<< "/usr/share/man/man1"
-
- ;
+ if (ies.contains ("/usr/share/doc/")) os << "usr/share/doc/*" << '\n';
+ if (ies.contains ("/usr/share/man/")) os << "usr/share/man/*" << '\n';
os.close ();
}
@@ -2719,8 +2772,6 @@ namespace bpkg
fail << "unable to write to " << main_install << ": " << e;
}
- return;
-
// Run dpkg-buildpackage.
//
// Note that there doesn't seem to be any way to control its verbosity or
diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx
index a04c213..aba5be1 100644
--- a/bpkg/system-package-manager.cxx
+++ b/bpkg/system-package-manager.cxx
@@ -7,6 +7,7 @@
#include <libbutl/regex.hxx>
#include <libbutl/semantic-version.hxx>
+#include <libbutl/json/parser.hxx>
#include <bpkg/package.hxx>
#include <bpkg/package-odb.hxx>
@@ -643,4 +644,238 @@ namespace bpkg
return r;
}
+
+ auto system_package_manager::
+ installed_entries (const common_options& co,
+ const packages& pkgs,
+ const strings& vars) -> installed_entry_map
+ {
+ process_path pp (search_b (co));
+
+ // Note that we don't use start_b() here since we want to be consistent
+ // with how things will be run when building the package.
+ //
+ cstrings args {
+ pp.recall_string (),
+ "--quiet", // Note: implies --no-progress.
+ "--dry-run"};
+
+ // Pass our --jobs value, if any.
+ //
+ string jobs;
+ if (size_t n = co.jobs_specified () ? co.jobs () : 0)
+ {
+ jobs = to_string (n);
+ args.push_back ("--jobs");
+ args.push_back (jobs.c_str ());
+ }
+
+ // Pass any --build-option.
+ //
+ for (const string& o: co.build_option ()) args.push_back (o.c_str ());
+
+ // Configuration variables.
+ //
+ for (const string& v: vars) args.push_back (v.c_str ());
+ args.push_back ("!config.install.manifest=-");
+
+ // Package directories to install.
+ //
+ strings dirs;
+ for (const package& p: pkgs) dirs.push_back (p.out_root.representation ());
+ args.push_back ("install:");
+ for (const string& d: dirs) args.push_back (d.c_str ());
+
+ args.push_back (nullptr);
+
+ installed_entry_map r;
+ try
+ {
+ if (verb >= 2)
+ print_process (args);
+ else if (verb == 1)
+ text << "determining filesystem entries that would be installed...";
+
+ // Redirect stdout to a pipe.
+ //
+ process pr (pp,
+ args,
+ 0 /* stdin */,
+ -1 /* stdout */,
+ 2 /* stderr */);
+ try
+ {
+ ifdstream is (move (pr.in_ofd), fdstream_mode::skip);
+
+ json::parser p (is,
+ args[0] /* input_name */,
+ true /* multi_value */,
+ "\n" /* value_separators */);
+
+ using event = json::event;
+
+ // Note: recursive lambda.
+ //
+ auto parse_entry = [&r, &p] (const auto& parse_entry) -> void
+ {
+ optional<event> e (p.next ());
+
+ // @@ This is really ugly, need to add next_expect() helpers to JSON
+ // parser (similar to libstudxml).
+
+ if (*e != event::begin_object)
+ fail << "entry object expected";
+
+ // type
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "type")
+ fail << "type member expected";
+
+ if (!(e = p.next ()) || *e != event::string)
+ fail << "type member string value expected";
+
+ string t (p.value ()); // Note: value invalidated after p.next().
+
+ if (t == "target")
+ {
+ // name
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "name")
+ fail << "name member expected";
+
+ if (!(e = p.next ()) || *e != event::string)
+ fail << "name member string value expected";
+
+ // entries
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "entries")
+ fail << "entries member expected";
+
+ if (!(e = p.next ()) || *e != event::begin_array)
+ fail << "entries member array value expected";
+
+ while ((e = p.peek ()) && *e != event::end_array)
+ parse_entry (parse_entry);
+
+ if (!(e = p.next ()) || *e != event::end_array)
+ fail << "entries member array value end expected";
+ }
+ else if (t == "file" || t == "symlink" || t == "directory")
+ {
+ // path
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "path")
+ fail << "path member expected";
+
+ if (!(e = p.next ()) || *e != event::string)
+ fail << "path member string value expected";
+
+ path ep (p.value ()); // Assuming normalized.
+
+ if (t == "file" || t == "directory")
+ {
+ // mode
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "mode")
+ fail << "mode member expected";
+
+ if (!(e = p.next ()) || *e != event::string)
+ fail << "mode member string value expected";
+
+ string em (p.value ());
+
+ if (t == "file")
+ {
+ auto p (
+ r.emplace (
+ move (ep), installed_entry {move (em), nullptr}));
+
+ if (!p.second)
+ fail << p.first->first << " is installed multiple times";
+ }
+ }
+ else
+ {
+ // target
+ //
+ if (!(e = p.next ()) || *e != event::name || p.name () != "target")
+ fail << "target member expected";
+
+ if (!(e = p.next ()) || *e != event::string)
+ fail << "target member string value expected";
+
+ path et (p.value ());
+ if (et.relative ())
+ {
+ et = ep.directory () / et;
+ et.normalize ();
+ }
+
+ auto i (r.find (et));
+ if (i == r.end ())
+ fail << "symlink " << ep << " target " << et << " does not "
+ << "refer to previously installed entry";
+
+ auto p (r.emplace (move (ep), installed_entry {"", &*i}));
+
+ if (!p.second)
+ fail << p.first->first << " is installed multiple times";
+ }
+ }
+ else
+ fail << "unknown entry type '" << t << "'";
+
+ if (!(e = p.next ()) || *e != event::end_object)
+ fail << "entry object end expected";
+ };
+
+ while (p.peek ()) // More values.
+ {
+ parse_entry (parse_entry);
+
+ if (p.next ()) // Consume value-terminating nullopt.
+ fail << "unexpected data after entry object";
+ }
+
+ is.close ();
+ }
+ catch (const json::invalid_json_input& e)
+ {
+ if (pr.wait ())
+ fail << "invalid " << args[0] << " json input: " << e;
+
+ // Fall through.
+ }
+ catch (const io_error& e)
+ {
+ if (pr.wait ())
+ fail << "unable to read " << args[0] << " output: " << e;
+
+ // Fall through.
+ }
+
+ if (!pr.wait ())
+ {
+ diag_record dr (fail);
+ dr << args[0] << " exited with non-zero code";
+
+ if (verb < 2)
+ {
+ dr << info << "command line: ";
+ print_process (dr, args);
+ }
+ }
+ }
+ catch (const process_error& e)
+ {
+ error << "unable to execute " << args[0] << ": " << e;
+
+ if (e.child)
+ exit (1);
+
+ throw failed ();
+ }
+
+ return r;
+ }
}
diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx
index 28b0c17..0fd219c 100644
--- a/bpkg/system-package-manager.hxx
+++ b/bpkg/system-package-manager.hxx
@@ -7,6 +7,8 @@
#include <bpkg/types.hxx>
#include <bpkg/utility.hxx>
+#include <libbutl/path-map.hxx>
+
#include <bpkg/package.hxx>
#include <bpkg/common-options.hxx>
#include <bpkg/host-os-release.hxx>
@@ -329,6 +331,52 @@ namespace bpkg
const string& name_id,
const string& version_id,
const vector<string>& like_ids);
+
+ // Return the map of filesystem entries (files and symlinks) that would be
+ // installed for the specified packages with the specified configuration
+ // variables.
+ //
+ // In essence, this function runs:
+ //
+ // b --dry-run --quiet <vars> !config.install.manifest=- install: <pkgs>
+ //
+ // And converts the printed installation manifest into the path map.
+ //
+ // Note that this function prints an appropriate progress indicator since
+ // even in the dry-run mode it may take some time (see the --dry-run
+ // option documentation for details).
+ //
+ struct installed_entry
+ {
+ string mode; // Empty if symlink.
+ const pair<const path, installed_entry>* target; // Target if symlink.
+ };
+
+ class installed_entry_map: public butl::path_map<installed_entry>
+ {
+ public:
+ // Return true if there are filesystem entries in the specified
+ // directory or its subdirectories.
+ //
+ bool
+ contains (const dir_path& d)
+ {
+ auto p (find_sub (d));
+ return p.first != p.second;
+ }
+
+ bool
+ contains (string d)
+ {
+ return contains (dir_path (move (d)));
+ }
+ };
+
+ installed_entry_map
+ installed_entries (const common_options&,
+ const packages& pkgs,
+ const strings& vars);
+
protected:
optional<bool> progress_; // --[no]-progress (see also stderr_term)
optional<size_t> fetch_timeout_; // --fetch-timeout
diff --git a/bpkg/utility.txx b/bpkg/utility.txx
index 0f88d53..3fe1d72 100644
--- a/bpkg/utility.txx
+++ b/bpkg/utility.txx
@@ -21,6 +21,9 @@ namespace bpkg
{
small_vector<const char*, 1> ops;
+ // NOTE: see custom versions in system_package_manager* if adding
+ // anything new here (search for search_b()).
+
// Map verbosity level. If we are running quiet or at level 1,
// then run build2 quiet. Otherwise, run it at the same level
// as us.