From 7b06ee81ab0e8a2199c4dce07ec67282c4f52f62 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 4 Apr 2023 08:05:21 +0200 Subject: Add support for installation filtering (GH issue #147) --- libbuild2/cc/functions.cxx | 4 +- libbuild2/filesystem.hxx | 2 + libbuild2/install/functions.cxx | 67 +++++++++ libbuild2/install/init.cxx | 76 +++++++++- libbuild2/install/operation.cxx | 324 ++++++++++++++++++++++++++++++++++++---- libbuild2/install/operation.hxx | 37 +++-- libbuild2/install/rule.cxx | 66 +++++--- libbuild2/install/rule.hxx | 13 +- libbuild2/utility.hxx | 1 + 9 files changed, 524 insertions(+), 66 deletions(-) (limited to 'libbuild2') diff --git a/libbuild2/cc/functions.cxx b/libbuild2/cc/functions.cxx index 94900ee..9d408af 100644 --- a/libbuild2/cc/functions.cxx +++ b/libbuild2/cc/functions.cxx @@ -52,7 +52,7 @@ namespace build2 // if (bs->ctx.phase != run_phase::match && bs->ctx.phase != run_phase::execute) - fail << f.name << " can only be called during execution"; + fail << f.name << " can only be called from recipe"; const module* m (rs->find_module (d.x)); @@ -131,7 +131,7 @@ namespace build2 if (bs->ctx.phase != run_phase::match && // See above. bs->ctx.phase != run_phase::execute) - fail << f.name << " can only be called during execution"; + fail << f.name << " can only be called from recipe"; const module* m (rs->find_module (d.x)); diff --git a/libbuild2/filesystem.hxx b/libbuild2/filesystem.hxx index 2998cec..7b45a08 100644 --- a/libbuild2/filesystem.hxx +++ b/libbuild2/filesystem.hxx @@ -22,6 +22,8 @@ // namespace build2 { + using butl::entry_type; + using butl::auto_rmfile; using butl::auto_rmdir; diff --git a/libbuild2/install/functions.cxx b/libbuild2/install/functions.cxx index 9f5fa44..d62b578 100644 --- a/libbuild2/install/functions.cxx +++ b/libbuild2/install/functions.cxx @@ -5,6 +5,7 @@ #include #include +#include namespace build2 { @@ -17,6 +18,8 @@ namespace build2 // $install.resolve([, ]) // + // @@ TODO: add overload to call resolve_file(). + // // Resolve potentially relative install.* value to an absolute and // normalized directory based on (other) install.* values visible from // the calling scope. @@ -80,6 +83,70 @@ namespace build2 move (dir), rel_base ? move (*rel_base) : dir_path ()); }; + + // @@ TODO: add $install.chroot(). + + // $install.filter([, ]) + // + // Apply filters from config.install.filter and return true if the + // specified filesystem entry should be installed/uninstalled. Note that + // the entry is specified as an absolute and normalized installation + // path (so not $path($>) but $install.resolve($>)). + // + // The type argument can be one of `regular`, `directory`, or `symlink`. + // If unspecified, either `directory` or `regular` is assumed, based on + // whether path is syntactially a directory (ends with a directory + // separator). + // + // Note that this function is not pure and can only be called from a + // install or uninstall operation recipe. + // + f.insert (".filter", false) += [] (const scope* s, + path p, + optional ot) + { + if (s == nullptr) + fail << "install.filter() called out of scope" << endf; + + context& ctx (s->ctx); + + if (ctx.phase != run_phase::match && + ctx.phase != run_phase::execute) + fail << "install.filter() can only be called from recipe"; + + if (ctx.current_inner_oif != &op_install && + ctx.current_inner_oif != &op_uninstall) + fail << "install.filter() can only be called during install/uninstall"; + + entry_type t; + if (ot) + { + string v (convert (move (*ot))); + + if (v == "regular") t = entry_type::regular; + else if (v == "directory") t = entry_type::directory; + else if (v == "symlink") t = entry_type::symlink; + else throw invalid_argument ("unknown type '" + v + '\''); + } + else + t = p.to_directory () ? entry_type::directory : entry_type::regular; + + // Split into directory and leaf. + // + dir_path d; + if (t == entry_type::directory) + { + d = path_cast (move (p)); + p = path (); // No leaf. + } + else + { + d = p.directory (); + p.make_leaf (); + } + + return context_data::filter (*s->root_scope (), d, p, t); + }; } } } diff --git a/libbuild2/install/init.cxx b/libbuild2/install/init.cxx index f0402ca..69f578b 100644 --- a/libbuild2/install/init.cxx +++ b/libbuild2/install/init.cxx @@ -421,9 +421,10 @@ namespace build2 using config::lookup_config; using config::specified_config; - // Note: ignore config.install.{scope,manifest} (see below). + // Note: ignore config.install.{scope,filter,manifest} (see below). // - bool s (specified_config (rs, "install", {"scope", "manifest"})); + bool s (specified_config ( + rs, "install", {"scope", "filter", "manifest"})); // Adjust module priority so that the (numerous) config.install.* // values are saved at the end of config.build. @@ -460,6 +461,77 @@ namespace build2 config::unsave_variable (rs, v); } + // config.install.filter + // + // Installation filterting. The value of this variable is a list of + // key-value pairs that specify the filesystem entries to include or + // exclude from the installation. For example, the following filters + // will omit installing headers and static libraries (notice the + // quoting of the wildcard). + // + // !config.install.filter='include/@false "*.a"@false' + // + // The key in each pair is a file or directory path or a path wildcard + // pattern. If a key is relative and contains a directory component or + // is a directory, then it is treated relative to the corresponding + // config.install.* location. Otherwise (simple path, normally a + // pattern), it is matched against the leaf of any path. Note that if + // an absolute path is specified, it should be without the + // config.install.chroot prefix. + // + // The value in each pair is either true (include) or false (exclude). + // The filters are evaluated in the order specified and the first + // match that is found determines the outcome. If no match is found, + // the default is to include. For a directory, while false means + // exclude all the sub-paths inside this directory, true does not mean + // that all the sub-paths will be included wholesale. Rather, the + // matched component of the sub-path is treated as included with the + // rest of the components matched against the following + // sub-filters. For example: + // + // !config.install.filter=' + // include/x86_64-linux-gnu/@true + // include/x86_64-linux-gnu/details/@false + // include/@false' + // + // The true or false value may be followed by comma and the `symlink` + // modifier to only apply to symlink filesystem entries. For example: + // + // !config.install.filter='"*.so"@false,symlink' + // + // Note that this mechanism only affects what gets physically copied + // to the installation directory without affecting what gets built for + // install or the view of what gets installed at the buildfile level. + // For example, given the `include/@false *.a@false` filters, static + // libraries will still be built (unless arranged not to with + // config.bin.lib) and the pkg-config files will still end up with -I + // options pointing to the header installation directory. Note also + // that this mechanism applies to both install and uninstall + // operations. + // + // If you are familiar with the Debian or Fedora packaging, this + // mechanism is somewhat similar to (and can be used for a similar + // purpose as) the Debian's .install files and Fedora's %files spec + // file sections that are used to split the installation into multiple + // binary packages. + // + // Note: can only be specified as a global override. + // + { + auto& v (vp.insert ("config.install.filter")); + + // If specified, verify it is a global override. + // + if (lookup l = rs[v]) + { + if (!l.belongs (rs.global_scope ())) + fail << "config.install.filter must be a global override" << + info << "specify !config.install.filter=..."; + } + + config::unsave_variable (rs, v); + } + // config.install.manifest // // Installation manifest. Valid values are a file path or `-` to dump diff --git a/libbuild2/install/operation.cxx b/libbuild2/install/operation.cxx index 95b4381..1b63bc0 100644 --- a/libbuild2/install/operation.cxx +++ b/libbuild2/install/operation.cxx @@ -19,10 +19,255 @@ namespace build2 { namespace install { + bool context_data:: + filter (const scope& rs, + const dir_path& base, + const path& leaf, + entry_type type) + { + assert (type != entry_type::unknown && + (type == entry_type::directory) == leaf.empty ()); + + context& ctx (rs.ctx); + + auto& d (*static_cast (ctx.current_inner_odata.get ())); + + if (d.filters == nullptr || d.filters->empty ()) + return true; + + tracer trace ("install::context_data::filter"); + + // Parse, resolve, and apply each filter in order. + // + // If redoing all this work for every entry proves too slow, we can + // consider some form of caching (e.g., on the per-project basis). + // + size_t limit (0); // See below. + + for (const pair& kv: *d.filters) + { + path k; + try + { + k = path (kv.first); + + if (k.absolute ()) + k.normalize (); + } + catch (const invalid_path&) + { + fail << "invalid path '" << kv.first << "' in config.install.filter " + << "value"; + } + + bool v; + { + const string& s (kv.second); + + size_t p (s.find (',')); + + if (s.compare (0, p, "true") == 0) + v = true; + else if (s.compare (0, p, "false") == 0) + v = false; + else + fail << "expected true or false instead of '" << string (s, 0, p) + << "' in config.install.filter value"; + + if (p != string::npos) + { + if (s.compare (p + 1, string::npos, "symlink") == 0) + { + if (type != entry_type::symlink) + continue; + } + else + fail << "unknown modifier '" << string (s, p + 1) << "' in " + << "config.install.filter value"; + } + } + + // @@ TODO (see below for all the corner cases). Note that in a sense + // we already have the match file in any subdirectory support via + // simple patterns so perhaps this is not worth the trouble. Or we + // could support some limited form (e.g., `**` should be in the + // last component). But it may still be tricky to determine if + // it is a sub-filter. + // + if (path_pattern_recursive (k)) + fail << "recursive wildcard pattern '" << kv.first << "' in " + << "config.install.filter value"; + + if (k.simple () && !k.to_directory ()) + { + // Simple name/pattern matched against the leaf. + // + // @@ What if it is `**`? + // + if (path_pattern (k)) + { + if (!path_match (leaf, k)) + continue; + } + else + { + if (k != leaf) + continue; + } + } + else + { + // Split into directory and leaf. + // + // @@ What if leaf is `**`? + // + dir_path d; + if (k.to_directory ()) + { + d = path_cast (move (k)); + k = path (); // No leaf. + } + else + { + d = k.directory (); + k.make_leaf (); + } + + // Resolve relative directory. + // + // Note that this resolution is potentially project-specific (that + // is, different projects may have different install.* locaitons). + // + // Note that if the first component is/contains a wildcard (e.g., + // `*/`), then the resulution will fail, which feels correct (what + // does */ mean?). + // + if (d.relative ()) + { + // @@ Strictly speaking, this should be base, not root scope. + // + d = resolve_dir (rs, move (d)); + } + + // Return the number of path components in the path. + // + auto path_comp = [] (const path& p) + { + size_t n (0); + for (auto i (p.begin ()); i != p.end (); ++i) + ++n; + return n; + }; + + // We need the sub() semantics but which uses pattern match instead + // of equality for the prefix. Looks like chopping off the path and + // calling path_match() on that is the best we can do. + // + // @@ Assumes no `**` components. + // + auto path_sub = [&path_comp] (const dir_path& ent, + const dir_path& pat, + size_t n = 0) + { + if (n == 0) + n = path_comp (pat); + + dir_path p; + for (auto i (ent.begin ()); n != 0 && i != ent.end (); --n, ++i) + p.combine (*i, i.separator ()); + + return path_match (p, pat); + }; + + // The following checks should continue on no match and fall through + // to return. + // + if (k.empty ()) // Directory. + { + // Directories have special semantics. + // + // Consider this sequence of filters: + // + // include/x86_64-linux-gnu/@true + // include/x86_64-linux-gnu/details/@false + // include/@false + // + // It seems the semantics we want is that only subcomponent + // filters should apply. Maybe remember the latest matched + // directory as a current limit? But perhaps we don't need to + // remember the directory itself but the number of path + // components? + // + // I guess for patterns we will use the actual matched directory, + // not the pattern, to calculate the limit? @@ Because we + // currently don't support `**`, we for now can count components + // in the pattern. + + // Check if this is a sub-filter. + // + size_t n (path_comp (d)); + if (n <= limit) + continue; + + if (path_pattern (d)) + { + if (!path_sub (base, d, n)) + continue; + } + else + { + if (!base.sub (d)) + continue; + } + + if (v) + { + limit = n; + continue; // Continue looking for sub-filters. + } + } + else + { + if (path_pattern (d)) + { + if (!path_sub (base, d)) + continue; + } + else + { + if (!base.sub (d)) + continue; + } + + if (path_pattern (k)) + { + // @@ Does not handle `**`. + // + if (!path_match (leaf, k)) + continue; + } + else + { + if (k != leaf) + continue; + } + } + } + + l4 ([&]{trace << (base / leaf) + << (v ? " included by " : " excluded by ") + << kv.first << '@' << kv.second;}); + return v; + } + + return true; + } + #ifndef BUILD2_BOOTSTRAP - install_context_data:: - install_context_data (const path* mf) - : manifest_name (mf), + context_data:: + context_data (const install::filters* fs, const path* mf) + : filters (fs), + manifest_name (mf), manifest_os (mf != nullptr ? open_file_or_stdout (manifest_name, manifest_ofs) : manifest_ofs), @@ -38,7 +283,7 @@ namespace build2 } static path - relocatable_path (install_context_data& d, const target& t, path p) + relocatable_path (context_data& d, const target& t, path p) { // This is both inefficient (re-detecting relocatable manifest for every // path) and a bit dirty (if multiple projects are being installed with @@ -101,7 +346,7 @@ namespace build2 // symlinks belong to tragets, directories do not. // static void - manifest_flush_target (install_context_data& d, const target* tgt) + manifest_flush_target (context_data& d, const target* tgt) { if (d.manifest_target != nullptr) { @@ -163,14 +408,13 @@ namespace build2 d.manifest_target = tgt; } - void install_context_data:: + void context_data:: manifest_install_d (context& ctx, const target& tgt, const dir_path& dir, const string& mode) { - auto& d ( - *static_cast (ctx.current_inner_odata.get ())); + auto& d (*static_cast (ctx.current_inner_odata.get ())); if (d.manifest_name.path != nullptr) { @@ -200,15 +444,14 @@ namespace build2 } } - void install_context_data:: + void context_data:: manifest_install_f (context& ctx, const target& tgt, const dir_path& dir, const path& name, const string& mode) { - auto& d ( - *static_cast (ctx.current_inner_odata.get ())); + auto& d (*static_cast (ctx.current_inner_odata.get ())); if (d.manifest_name.path != nullptr) { @@ -220,15 +463,14 @@ namespace build2 } } - void install_context_data:: + void context_data:: manifest_install_l (context& ctx, const target& tgt, const path& link_target, const dir_path& dir, const path& link) { - auto& d ( - *static_cast (ctx.current_inner_odata.get ())); + auto& d (*static_cast (ctx.current_inner_odata.get ())); if (d.manifest_name.path != nullptr) { @@ -243,8 +485,7 @@ namespace build2 static void manifest_close (context& ctx) { - auto& d ( - *static_cast (ctx.current_inner_odata.get ())); + auto& d (*static_cast (ctx.current_inner_odata.get ())); if (d.manifest_name.path != nullptr) { @@ -271,12 +512,13 @@ namespace build2 } } #else - install_context_data:: - install_context_data (const path*) + context_data:: + context_data (const install::filters* fs, const path*) + : filters (fs) { } - void install_context_data:: + void context_data:: manifest_install_d (context&, const target&, const dir_path&, @@ -284,7 +526,7 @@ namespace build2 { } - void install_context_data:: + void context_data:: manifest_install_f (context&, const target&, const dir_path&, @@ -293,7 +535,7 @@ namespace build2 { } - void install_context_data:: + void context_data:: manifest_install_l (context&, const target&, const path&, @@ -341,20 +583,48 @@ namespace build2 if (inner) { - // See if we need to write the installation manifest. + // See if we need to filter and/or write the installation manifest. // // Note: go straight for the public variable pool. // - const variable& var (*ctx.var_pool.find ("config.install.manifest")); - const path* mf (cast_null (ctx.global_scope[var])); + const filters* fs ( + cast_null ( + ctx.global_scope[*ctx.var_pool.find ("config.install.filter")])); + + const path* mf ( + cast_null ( + ctx.global_scope[*ctx.var_pool.find ("config.install.manifest")])); // Note that we cannot calculate whether the manifest should use // relocatable (relative) paths once here since we don't know the // value of config.install.root. ctx.current_inner_odata = context::current_data_ptr ( - new install_context_data (mf), - [] (void* p) {delete static_cast (p);}); + new context_data (fs, mf), + [] (void* p) {delete static_cast (p);}); + } + } + + static void + uninstall_pre (context& ctx, + const values& params, + bool inner, + const location& l) + { + // Note: a subset of install_pre(). + // + if (!params.empty ()) + fail (l) << "unexpected parameters for operation uninstall"; + + if (inner) + { + const filters* fs ( + cast_null ( + ctx.global_scope[*ctx.var_pool.find ("config.install.filter")])); + + ctx.current_inner_odata = context::current_data_ptr ( + new context_data (fs, nullptr), + [] (void* p) {delete static_cast (p);}); } } @@ -412,7 +682,7 @@ namespace build2 0 /* concurrency */, // Run serially &pre_uninstall, nullptr, - nullptr, + &uninstall_pre, nullptr, nullptr, nullptr diff --git a/libbuild2/install/operation.hxx b/libbuild2/install/operation.hxx index 4983976..d71d8f3 100644 --- a/libbuild2/install/operation.hxx +++ b/libbuild2/install/operation.hxx @@ -4,15 +4,15 @@ #ifndef LIBBUILD2_INSTALL_OPERATION_HXX #define LIBBUILD2_INSTALL_OPERATION_HXX -#include -#include - #ifndef BUILD2_BOOTSTRAP # include #endif +#include +#include + #include -#include // auto_rmfile +#include // auto_rmfile, entry_type namespace build2 { @@ -22,10 +22,27 @@ namespace build2 extern const operation_info op_uninstall; extern const operation_info op_update_for_install; - // Set as context::current_inner_odata during the install inner operation. + using filters = vector>; + + // Set as context::current_inner_odata during the install/uninstall inner + // operations. // - struct install_context_data + struct context_data { + // Filters. + // + const install::filters* filters; + + // If entry type is a directory, then leaf must be empty. + // + static bool + filter (const scope& rs, + const dir_path& base, + const path& leaf, + entry_type); + + // Manifest. + // #ifndef BUILD2_BOOTSTRAP path manifest_file; // Absolute and normalized, empty if `-`. path_name manifest_name; // Original path/name. @@ -43,9 +60,6 @@ namespace build2 vector manifest_target_entries; #endif - explicit - install_context_data (const path* manifest); - // The following manifest_install_[dfl]() functions correspond to (and // are called from) file_rule::install_[dfl](). @@ -74,6 +88,11 @@ namespace build2 const path& link_target, const dir_path& dir, const path& link); + + // Constructor. + // + explicit + context_data (const install::filters*, const path* manifest); }; } } diff --git a/libbuild2/install/rule.cxx b/libbuild2/install/rule.cxx index 5a8242b..f8e3e05 100644 --- a/libbuild2/install/rule.cxx +++ b/libbuild2/install/rule.cxx @@ -790,6 +790,8 @@ namespace build2 const file& t, uint16_t verbosity) { + assert (d.absolute ()); + context& ctx (rs.ctx); // Here is the problem: if this is a dry-run, then we will keep showing @@ -805,7 +807,8 @@ namespace build2 // Note that this also means we won't have the directory entries in the // manifest created with dry-run. Probably not a big deal. // - if (ctx.dry_run) + if (ctx.dry_run || + !context_data::filter (rs, d, path (), entry_type::directory)) return; dir_path chd (chroot_path (rs, d)); @@ -870,7 +873,7 @@ namespace build2 pp, args, verb >= verbosity ? 1 : verb_never /* finish_verbosity */); - install_context_data::manifest_install_d (ctx, t, d, *base.dir_mode); + context_data::manifest_install_d (ctx, t, d, *base.dir_mode); } void file_rule:: @@ -881,8 +884,15 @@ namespace build2 const path& f, uint16_t verbosity) { + assert (name.empty () || name.simple ()); + context& ctx (rs.ctx); + const path& leaf (name.empty () ? f.leaf () : name); + + if (!context_data::filter (rs, base.dir, leaf, entry_type::regular)) + return; + path relf (relative (f)); dir_path chd (chroot_path (rs, base.dir)); @@ -934,12 +944,7 @@ namespace build2 pp, args, verb >= verbosity ? 1 : verb_never /* finish_verbosity */); - install_context_data::manifest_install_f ( - ctx, - t, - base.dir, - name.empty () ? f.leaf () : name, - *base.mode); + context_data::manifest_install_f (ctx, t, base.dir, leaf, *base.mode); } void file_rule:: @@ -950,8 +955,13 @@ namespace build2 const path& link_target, uint16_t verbosity) { + assert (link.simple () && !link.empty ()); + context& ctx (rs.ctx); + if (!context_data::filter (rs, base.dir, link, entry_type::symlink)) + return; + if (link_target.absolute () && cast_false (rs["install.relocatable"])) { @@ -1033,12 +1043,11 @@ namespace build2 } #endif - install_context_data::manifest_install_l ( - ctx, - target, - link_target, - base.dir, - link); + context_data::manifest_install_l (ctx, + target, + link_target, + base.dir, + link); } target_state file_rule:: @@ -1158,9 +1167,14 @@ namespace build2 const dir_path& d, uint16_t verbosity) { + assert (d.absolute ()); + + context& ctx (rs.ctx); + // See install_d() for the rationale. // - if (rs.ctx.dry_run) + if (ctx.dry_run || + !context_data::filter (rs, d, path (), entry_type::directory)) return false; dir_path chd (chroot_path (rs, d)); @@ -1241,10 +1255,10 @@ namespace build2 } process pr (run_start (pp, args, - 0 /* stdin */, - 1 /* stdout */, - diag_buffer::pipe (rs.ctx) /* stderr */)); - diag_buffer dbuf (rs.ctx, args[0], pr); + 0 /* stdin */, + 1 /* stdout */, + diag_buffer::pipe (ctx) /* stderr */)); + diag_buffer dbuf (ctx, args[0], pr); dbuf.read (); r = run_finish_code ( dbuf, @@ -1338,10 +1352,15 @@ namespace build2 const path& name, uint16_t verbosity) { - assert (t != nullptr || !name.empty ()); + assert (name.empty () ? t != nullptr : name.simple ()); + + const path& leaf (name.empty () ? t->path ().leaf () : name); + + if (!context_data::filter (rs, base.dir, leaf, entry_type::regular)) + return false; dir_path chd (chroot_path (rs, base.dir)); - path f (chd / (name.empty () ? t->path ().leaf () : name)); + path f (chd / leaf); try { @@ -1380,6 +1399,11 @@ namespace build2 const path& /*link_target*/, uint16_t verbosity) { + assert (link.simple () && !link.empty ()); + + if (!context_data::filter (rs, base.dir, link, entry_type::symlink)) + return false; + dir_path chd (chroot_path (rs, base.dir)); path f (chd / link); diff --git a/libbuild2/install/rule.hxx b/libbuild2/install/rule.hxx index b0dabe5..b319071 100644 --- a/libbuild2/install/rule.hxx +++ b/libbuild2/install/rule.hxx @@ -205,6 +205,8 @@ namespace build2 // install / # if is empty // install / # if is not empty // + // Note that should be a simple path. + // static void install_f (const scope& rs, const install_dir& base, @@ -221,8 +223,9 @@ namespace build2 // // ln -s / // - // Note that must not be absolute if relocatable - // installation is requested (config.install.relocatable). + // Note that should be a simple path. Note that + // must not be absolute if relocatable installation is requested + // (config.install.relocatable). // // Note that the target argument only specifies which target this // symlink "belongs" to. @@ -267,9 +270,9 @@ namespace build2 // // uninstall -d // - // We try to remove all the directories between base and dir but not base - // itself unless base == dir. Return false if nothing has been removed - // (i.e., the directories do not exist or are not empty). + // We try to remove all the directories between base and dir but not + // base itself unless base == dir. Return false if nothing has been + // removed (i.e., the directories do not exist or are not empty). // static bool uninstall_d (const scope& rs, diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index 2cb0738..1e22e7e 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -100,6 +100,7 @@ namespace build2 // // using butl::path_pattern; + using butl::path_match; // Perform process-wide initializations/adjustments/workarounds. Should be // called once early in main(). In particular, besides other things, this -- cgit v1.1