diff options
46 files changed, 3608 insertions, 455 deletions
diff --git a/build2/b.cxx b/build2/b.cxx index 9ffb6d1..0b4ec3a 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -7,7 +7,7 @@ #include <exception> // terminate(), set_terminate(), terminate_handler #include <libbutl/pager.hxx> -#include <libbutl/fdstream.hxx> // stderr_fd(), fdterm() +#include <libbutl/fdstream.hxx> // stderr_fd(), fdterm(), std*_fdmode() #include <libbutl/backtrace.hxx> // backtrace() #ifndef BUILD2_BOOTSTRAP @@ -249,8 +249,6 @@ main (int argc, char* argv[]) tracer trace ("main"); - init_process (); - int r (0); b_options ops; scheduler sched; @@ -261,6 +259,26 @@ main (int argc, char* argv[]) try { + // Note that the standard stream descriptors can potentially be in the + // non-blocking mode, which the C++ streams are not suited for and which + // are not fully supported by butl::iofdstreams. Using such descriptors + // may lead to various weird failures (see GH issue #417 for the + // reproducer). Thus, we just turn such descriptors into the blocking mode + // at the beginning of the program execution. + // + try + { + stdin_fdmode (fdstream_mode::blocking); + stdout_fdmode (fdstream_mode::blocking); + stderr_fdmode (fdstream_mode::blocking); + } + catch (const io_error& e) + { + fail << "unable to turn standard streams into blocking mode: " << e; + } + + init_process (); + // Parse the command line. // b_cmdline cmdl (parse_b_cmdline (trace, argc, argv, ops)); @@ -1541,8 +1559,13 @@ main (int argc, char* argv[]) if (dump_match_pre) dump (ctx, a); - if (mif->execute != nullptr && !ctx.match_only) - mif->execute (mparams, a, tgs, diag, true /* progress */); + if (mif->execute != nullptr) + { + if (!ctx.match_only) + mif->execute (mparams, a, tgs, diag, true /* progress */); + else if (mif->execute == &perform_execute) + perform_post_operation_callbacks (ctx, a, tgs, false /*failed*/); + } } if (pre_oif->operation_post != nullptr) @@ -1585,8 +1608,13 @@ main (int argc, char* argv[]) if (dump_match) dump (ctx, a); - if (mif->execute != nullptr && !ctx.match_only) - mif->execute (mparams, a, tgs, diag, true /* progress */); + if (mif->execute != nullptr) + { + if (!ctx.match_only) + mif->execute (mparams, a, tgs, diag, true /* progress */); + else if (mif->execute == &perform_execute) + perform_post_operation_callbacks (ctx, a, tgs, false /*failed*/); + } } if (oif->operation_post != nullptr) @@ -1629,8 +1657,13 @@ main (int argc, char* argv[]) if (dump_match_post) dump (ctx, a); - if (mif->execute != nullptr && !ctx.match_only) - mif->execute (mparams, a, tgs, diag, true /* progress */); + if (mif->execute != nullptr) + { + if (!ctx.match_only) + mif->execute (mparams, a, tgs, diag, true /* progress */); + else if (mif->execute == &perform_execute) + perform_post_operation_callbacks (ctx, a, tgs, false /*failed*/); + } } if (post_oif->operation_post != nullptr) diff --git a/doc/manual.cli b/doc/manual.cli index 4c55374..1e489e9 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -2093,7 +2093,7 @@ If we forget to adjust the \c{missing-name} test, then this is what we could expect to see when running the tests: \ -b test +$ b test c++ hello/cxx{hello} -> hello/obje{hello} ld hello/exe{hello} test hello/exe{hello} + hello/testscript{testscript} @@ -6700,8 +6700,8 @@ quickly re-run a previously failed test), it can also be persisted in subset of tests by default. For example: \ -b test config.test=foo/exe{driver} # Only test foo/exe{driver} target. -b test config.test=bar/baz # Only run bar/baz testscript test. +$ b test config.test=foo/exe{driver} # Only test foo/exe{driver} target. +$ b test config.test=bar/baz # Only run bar/baz testscript test. \ The \c{config.test} variable contains a list of \c{@}-separated pairs with the @@ -6712,14 +6712,14 @@ name. Otherwise \- an id path. The targets are resolved relative to the root scope where the \c{config.test} value is set. For example: \ -b test config.test=foo/exe{driver}@bar +$ b test config.test=foo/exe{driver}@bar \ To specify multiple id paths for the same target we can use the pair generation syntax: \ -b test config.test=foo/exe{driver}@{bar baz} +$ b test config.test=foo/exe{driver}@{bar baz} \ If no targets are specified (only id paths), then all the targets are tested @@ -6741,9 +6741,9 @@ and the right hand side \- for individual tests. The zero value clears the previously set timeout. For example: \ -b test config.test.timeout=20 # Test operation. -b test config.test.timeout=20/5 # Test operation and individual tests. -b test config.test.timeout=/5 # Individual tests. +$ b test config.test.timeout=20 # Test operation. +$ b test config.test.timeout=20/5 # Test operation and individual tests. +$ b test config.test.timeout=/5 # Individual tests. \ The test timeout can be specified on multiple nested root scopes. For example, @@ -6759,7 +6759,7 @@ specifying the \c{config.test.runner} variable. Its value has the \c{<path> [<options>]} form. For example: \ -b test config.test.runner=\"valgrind -q\" +$ b test config.test.runner=\"valgrind -q\" \ When the runner program is specified, commands of simple and Testscript tests @@ -7648,6 +7648,12 @@ config.cc.reprocess cc.reprocess config.cc.pkgconfig.sysroot + +config.cc.compiledb +config.cc.compiledb.name +config.cc.compiledb.filter +config.cc.compiledb.filter.input +config.cc.compiledb.filter.output \ Note that the compiler mode options are \"cross-hinted\" between \c{config.c} @@ -8054,6 +8060,336 @@ As a result, it should only be used for dealing with issues in third-party installation} should be used instead.| +\h#cc-compiledb|Compilation Database| + +The \c{cc}-based modules provide support for generating and maintaining the +\l{https://clang.llvm.org/docs/JSONCompilationDatabase.html JSON Compilation +Database} which can be used by other tools (static analyzers, language +servers, IDEs, etc) to understand how a codebase is compiled. \"Maintaining\" +in the previous sentence means that if new source files get added to the +project or old ones removed, or if any compilation options change, then the +corresponding entries in the compilation database will be automatically +updated when you update your project. This helps maintain the database in sync +with the project state. + +The generation of compilation databases and their configuration are controlled +with a number of \c{config.cc.compiledb.*} variables. The +\c{config.cc.compiledb} variable provides a simplified interface that enables +the generation of one database per project with the resulting database +containing entries for all the source and object files. The rest of the +variables provide a more flexible interface that allows you to generate +multiple databases in different locations as well as filter the entries that +end up in each database. + +Let's start with the simplified interface as provided by +\c{config.cc.compiledb}. The value of this configuration variable is a single +\ci{name} or a \ci{name} and \ci{path} pair in the \c{\i{name}[@\i{path}]} +form. + +The \ci{name} part is the compilation database name that can be used to refer +to it in filters (see below). If \ci{path} is absent or is (syntactically) a +directory, then \ci{name} is also used to derive the compilation database file +by appending the \c{.json} extension to it. + +If \ci{path} is absent, then the compilation database is placed into the +top-level amalgamation that loads any \c{cc}-based module. Otherwise, the +database is placed into the specified location. + +The special \c{-} name is interpreted as an instruction to dump the database +to \c{stdout}. + +Let's see some examples of using \c{config.cc.compiledb} to handle a few +common scenarios. Here we will use \l{bdep(1)} to create amalgamations +(configurations) and configure (initialize) one or more projects. We will +assume we have \c{hello} and \c{libhello} as if created like this: + +\ +$ bdep new -t exe hello +$ bdep new -t lib libhello +\ + +The most common scenario is likely having a compilation database per +project: + +\ +$ cd libhello +$ bdep config create ../build-gcc @gcc cc config.cxx=g++ +$ bdep init @gcc config.cc.compiledb=libhello +$ cd .. + +$ cd hello +$ bdep config add ../build-gcc @gcc +$ bdep init @gcc config.cc.compiledb=hello +$ cd .. + +$ b hello/ libhello/ +\ + +\N|Or if you prefer to create/add configuration as part of \c{init} (notice +the \c{--} separator): + +\ +$ bdep init -C ../build-gcc @gcc cc config.cxx=g++ -- \\ + config.cc.compiledb=libhello + +$ bdep init -A ../build-gcc @gcc config.cc.compiledb=hello +\ + +| + +After the update (the last command), we will have \c{hello.json} and +\c{libhello.json} in \c{build-gcc/} which contain the compilation command +lines for each project. + +\N|Only source files that are compiled end up being added to the compilation +database. + +To illustrate this point, let's assume our \c{hello} project imports and links +\c{libhello}. And instead of updating both as in the above example, we will +first update only \c{hello}: + +\ +$ b hello/ +\ + +In this case \c{libhello.json} will still be generated but it will only +contain a subset of the expected entries \- only those that were caused to be +compiled by \c{hello}. The missing entries can be added by updating +\c{libhello}: + +\ +$ b libhello/ +\ + +| + +In the above setup it feels natural to call each database after the project +and place them into the output directory. However, some consumers, such as +IDEs and LSP servers, may not handle this setup well. Specifically, they may +only recognize the canonical \c{compile_commands.json} file as the compilation +database, opening all other files as generic JSON. They may also assume the +directory where this file resides to be the project source directory root. To +accommodate these assumptions we can instead place each database into the +project's source directory and call it \c{compile_commands.json}: + +\ +$ bdep init @gcc config.cc.compiledb=libhello@./compile_commands.json + +$ bdep init @gcc config.cc.compiledb=hello@./compile_commands.json +\ + +To facilitate this use-case, \c{config.cc.compiledb} supports another +shortcut: if we specify just \ci{name} and it contains a directory component, +then it is interpreted as \ci{path} rather than \ci{name}. In this case +\ci{name} is taken to be the name of the last directory component in \ci{path} +(which would typically be a project or package name). And if \ci{path} is a +directory, then the database file name is taken to be +\c{compile_commands.json}. Or, in other words, the following: + +\ +config.cc.compiledb=.../<dir>/ +\ + +Is equivalent to: + +\ +config.cc.compiledb=<dir>@.../<dir>/compile_commands.json +\ + +This shortcut allows us to simplify the above \c{init} commands to read: + +\ +$ bdep init @gcc config.cc.compiledb=./ + +$ bdep init @gcc config.cc.compiledb=./ +\ + +Note also that in this case it will be your responsibility to remove the +database files if and when necessary. \N{\l{bdep-new(1)} adds +\c{compile_commands.json} to \c{.gitignore} it generates.} + +If instead of having a separate database for each project we wanted to place +all the entries into a single database (and in the output directory), then the +relevant commands would change as follows: + +\ +$ bdep init @gcc config.cc.compiledb=compiledb + +$ bdep init @gcc config.cc.compiledb=compiledb +\ + +This would give us a single \c{build-gcc/compiledb.json} that contains the +compilation command lines for both projects. + +In the above example only \c{hello} and \c{libhello} will end up in the +database, but not any of their dependencies. What if we wanted entries for +everything in \c{build-gcc/}? In this case, we should enable the compilation +database for the entire configuration rather than for individual projects: + +\ +$ bdep config create ../build-gcc @gcc cc \\ + config.cxx=g++ \\ + config.cc.compiledb=compiledb +$ bdep init @gcc + +$ bdep config add ../build-gcc @gcc +$ bdep init @gcc +\ + +If multiple linked configurations are involved, then we would often want +projects initialized in different configurations share the compilation +database. The representative scenario here is a tool, such as a source code +generator, which is initialized in the host configuration, and its runtime +library plus tests/examples, which are initialized in the target +configuration. Let's assume that in our example \c{hello} is the tool and +\c{libhello} is the runtime library and both are part of the same project. +This is how we can arrange for them to share the compilation database: + +\ +$ bdep config create @host ../host-gcc --type host cc config.cxx=g++ +$ bdep config create @target ../build-gcc cc config.cxx=g++ + +$ bdep init @host -d hello config.cc.compiledb=hello@../build-gcc/ +$ bdep init @target -d libhello config.cc.compiledb=hello + +$ bdep update @host @target +\ + +With this setup the \c{hello.json} database in \c{build-gcc/} will contain +entries for both \c{hello} and \c{libhello}. + +If instead of configuring and maintaining the compilation database in a file +you want to dump it somewhere once, the recommended approach is to write it +to \c{stdout}. For example: + +\ +$ b -n hello/ libhello/ config.cc.compiledb=- >/tmp/compiledb.json +\ + +Note that writing to \c{stdout} forces recompilation of all the targets that +would be updated in order to make sure their entries end up in the database. +If you don't want the actual recompilation, then you can use the dry run mode +(\c{-n} option above). + +\N|If your projects are spread across multiple linked configurations and you +would like to get compilation command lines for all of them, then use the +global override for \c{config.cc.compiledb}: + +\ +$ b '!config.cc.compiledb=-' ... +\ + +As mentioned earlier, the entries that will end up in such a database are +determined by what gets updated.| + +Let's now turn to the rest of the \c{config.cc.compiledb.*} configuration +variables that provide a lower-level but more flexible interface. The +following listing shows their synopsis: + +\ +config.cc.compiledb.name = <name>[@<path>]... +config.cc.compiledb.filter = [<name>@]<bool>... +config.cc.compiledb.filter.input = [<name>@]<target-type>... +config.cc.compiledb.filter.output = [<name>@]<target-type>... +\ + +The \c{config.cc.compiledb.name} variable specifies the name and location of +one or more compilation databases. The semantics of the +\c{\i{name}[@\i{path}]} pair is the same as in \c{config.cc.compiledb} +discussed above, except that if \ci{path} is absent, then the database is +placed into the project being configured rather than into the top-level +amalgamation. + +Also, unlike \c{config.cc.compiledb}, this variable does not automatically +enable writing to the specified databases. Instead, this is the job of +\c{config.cc.compiledb.filter}. Splitting this logic into two steps allows us +to configure the database name/location in one place, typically an outer +amalgamation, and then enable writing to it in other places, typically +specific subprojects. + +The \c{config.cc.compiledb.filter.{input,output\}} variables allow us to +filter the entries that end up in the databases based on the input (\c{c{\}}, +\c{cxx{\}}, etc) and output (\c{obja{\}}, \c{objs{\}}, etc) target types. + +Note that in all three \c{.filter} variables the values are examined in the +reverse order and the first entry that matches determines the outcome. +Entries without \ci{name} apply to all databases and the target types are +matched taking into account inheritance (so \c{target{\}} will match any type) +and groups (so \c{obj{\}} will match any \c{obj[eas]{\}}). If no target type +filter (input or output) is specified, then no corresponding target filtering +is performed. + +\N|The \c{config.cc.compiledb=<name>} semantics can be expressed as the +following set of lower-level variables: + +\ +config.cc.compiledb.name = <name>@../path/to/amalgamation/ +config.cc.compiledb.filter += <name>@true +config.cc.compiledb.filter.input += <name>@target +config.cc.compiledb.filter.output += <name>@target +\ + +The last three assignments only apply if the corresponding variable is not set +to a custom value for this project.| + +Let's look at a few examples of using these lower-level configuration +variables. The common use for the output target filtering is getting rid of +\c{obja{\}} or \c{objs{\}} entries in libraries. Unless configured otherwise, +when we build a library we end up with both static and shared variants. And +this means that each source file for the library is compiled twice, once to +produce \c{obja{\}} that goes to the static library and once -- \c{objs{\}}. +And that, in turn, means that we will end up with two compilation database +entries for each such source file. If we don't want that for some reason (for +instance, because the consumer of the database does not handle this well), +then we can filter one of them out. For example, below is how we can +initialize \c{libhello} to achieve this (notice that we also include +\c{obje{\}} to keep object files for executables, such as tests): + +\ +$ bdep init @gcc \\ + config.cc.compiledb=libhello \\ + config.cc.compiledb.filter.output='obje objs' +\ + +As an example of the input target type filtering, below is how we can keep +entries only for the C and C++ source files, filtering out everything else +(assembler, Objective-C/C++), for instance, because the consumer of our +database does not recognize them: + +\ +$ bdep init @gcc \\ + config.cc.compiledb=libhello \\ + config.cc.compiledb.filter.input='c cxx' +\ + +As an example of a more advanced configuration, consider a compilation +database for a project that use C++ modules. To know how such a project is +compiled we not only need to know how its own source files are compiled, but +also how to compile all the module interfaces that it consumes, including from +other projects, transitively. One way to set this up would be to enable +writing entries of the \c{bmi{\}} output target type to any database in the +amalgamation: + +\ +$ bdep config create ../build-gcc @gcc cc \\ + config.cxx=g++ \\ + config.cc.compiledb.filter=true \\ + config.cc.compiledb.filter.output=bmi \\ + + +$ bdep init @gcc config.cc.compiledb=libhello + +$ bdep init @gcc config.cc.compiledb=hello +\ + +With this setup \c{libhello.json} and \c{hello.json} will contain module +interface entries from all the dependencies. + +\N|When debugging complex compilation database setups it can be helpful to +increase diagnostics verbosity to level 6 in order to get a trace of filtering +decisions (the relevant lines will contain the \c{compiledb} keyword).| + + \h#cc-gcc|GCC Compiler Toolchain| The GCC compiler id is \c{gcc}. @@ -8298,11 +8634,11 @@ c m S h \ -The \c{m{\}} target type represents an Objective-C source file, see \l{c-objc +The \c{m{\}} target type represents an Objective-C source file, see \l{#c-objc Objective-C Compilation} for details. The \c{S{\}} target type represents an Assembler with C Preprocessor file, see -\l{c-as-cpp Assembler with C Preprocessor Compilation} for details. +\l{#c-as-cpp Assembler with C Preprocessor Compilation} for details. \h2#c-target-types-c|\c{c{\}}, \c{h{\}}| @@ -8549,7 +8885,7 @@ mxx \ The \c{mm{\}} target type represents an Objective-C++ source file, see -\l{cxx-objcxx Objective-C++ Compilation} for details. +\l{#cxx-objcxx Objective-C++ Compilation} for details. \h2#cxx-target-types-cxx|\c{cxx{\}}, \c{hxx{\}}, \c{ixx{\}}, \c{txx{\}}, \c{mxx{\}}| diff --git a/doc/testscript.cli b/doc/testscript.cli index c539903..18dac41 100644 --- a/doc/testscript.cli +++ b/doc/testscript.cli @@ -2873,7 +2873,9 @@ supports a small subset of primaries and doesn't support compound expressions, negations, logical OR and (explicit) AND operators, and the \c{-type} primary values other than \c{f}, \c{d}, and \c{l}. It, however, supports the \c{-mindepth} and \c{-maxdepth} primaries which are not specified by POSIX but -are supported by the major \c{find} utility implementations. +are supported by the major \c{find} utility implementations. Likewise, it +dereferences symlinks in <start-path> if it ends with a trailing directory +separator. The following primaries are supported: diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli index 5aea034..9639477 100644 --- a/libbuild2/build/script/builtin.cli +++ b/libbuild2/build/script/builtin.cli @@ -103,9 +103,10 @@ namespace build2 // Dynamic target extraction options. // // This functionality is enabled with the --dyn-target option. Only - // the make format is supported, where the listed targets are added as - // ad hoc group members (unless already specified as static members). - // This functionality is not available in the byproduct mode. + // the `make` and `lines` formats are supported (see above), where the + // listed targets are added as ad hoc group members (unless already + // specified as static members). This functionality is not available + // in the byproduct mode. // string --target-what; // Target kind, e.g., "source". diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index 95ba89f..29a26b5 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -25,6 +25,7 @@ #include <libbuild2/cc/target.hxx> // h #include <libbuild2/cc/module.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using std::exit; using std::strlen; @@ -1181,6 +1182,11 @@ namespace build2 fsdir_rule::perform_update_direct (a, *dir); } + // Use the subset of the depdb checks to detect changes to the + // compilation database entry. + // + bool compiledb_changed (false); + // Note: the leading '@' is reserved for the module map prefix (see // extract_modules()) and no other line must start with it. // @@ -1198,8 +1204,14 @@ namespace build2 // but only in what it targets, then the checksum will still change. // if (dd.expect (cast<string> (rs[x_checksum])) != nullptr) + { l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); + // The checksum includes the absolute compiler path. + // + compiledb_changed = true; + } + // Then the compiler environment checksum. // if (dd.expect (env_checksum) != nullptr) @@ -1263,7 +1275,17 @@ namespace build2 append_sys_hdr_options (cs); // Extra system header dirs (last). if (dd.expect (cs.string ()) != nullptr) + { l4 ([&]{trace << "options mismatch forcing update of " << t;}); + + // Note that this doesn't include any of the "plumbing" options + // like -x, -c, -o, etc. In the unlikely event that there are + // changes in this area that also affect the semantics of the + // compilation database (options reordering doesn't, for example), + // then we can resort to incrementing the rule version. + // + compiledb_changed = true; + } } // Finally the source file. @@ -1273,7 +1295,10 @@ namespace build2 assert (!p.empty ()); // Sanity check. if (dd.expect (p) != nullptr) + { l4 ([&]{trace << "source file mismatch forcing update of " << t;}); + compiledb_changed = true; + } } // If any of the above checks resulted in a mismatch (different @@ -1296,6 +1321,14 @@ namespace build2 u = dd.mtime > mt; } + // Confirm the entry in the compilation database, if any. + // + if (compiledb::match (bs, t, tp, src, compiledb_changed) && !u) + { + l4 ([&]{trace << "compilation database forcing update of " << t;}); + u = true; + } + // If updating for any of the above reasons, treat it as if doesn't // exist. // @@ -1950,8 +1983,8 @@ namespace build2 // // @@ Should we print the pid we are talking to? It gets hard to // follow once things get nested. But if all our diag will include - // some kind of id (chain, thread?), then this will not be strictly - // necessary. + // some kind of id (dependency chain, thread?), then this will not + // be strictly necessary. // diag_record dr (text); for (size_t i (0); i != batch_n; ++i) @@ -2628,8 +2661,8 @@ namespace build2 // @@ MODHDR: Should we print the pid we are talking to? It gets hard to // follow once things get nested. But if all our diag will - // include some kind of id (chain, thread?), then this will - // not be strictly necessary. + // include some kind of id (dependency chain, thread?), then + // this will not be strictly necessary. // if (verb >= 3) text << " > " << rq; @@ -3148,7 +3181,7 @@ namespace build2 } hk.file = move (fp); - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); slock l (hc.header_map_mutex); auto i (hc.header_map.find (hk)); @@ -3201,7 +3234,7 @@ namespace build2 // path has changed (header has been remapped). // if (!e || r.second) - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); const file* f; { @@ -3214,6 +3247,8 @@ namespace build2 { //cache_cls.fetch_add (1, memory_order_relaxed); + // @@ TMP cleanup. + // #if 0 assert (r.first == f); #else @@ -4167,9 +4202,10 @@ namespace build2 if (l->empty ()) // Done, nothing changed. { // If modules are enabled, then we keep the preprocessed output - // around (see apply() for details). + // around (see apply() for details). Unless reprocessing was + // requested. // - if (modules) + if (modules && !reprocess) { result.first = ctx.fcache->create_existing (t.path () + pext); result.second = true; @@ -7351,8 +7387,11 @@ namespace build2 // apply()). For named modules there may be no obj*{} if this is a // sidebuild (obj*{} is already in the library binary). // - path relm; + const path* abso (nullptr); + const path* absm (nullptr); path relo; + path relm; + switch (ut) { case unit_type::module_header: @@ -7362,12 +7401,18 @@ namespace build2 case unit_type::module_impl_part: { if (const file* o = find_adhoc_member<file> (t, tts.obj)) - relo = relative (o->path ()); + { + abso = &o->path (); + relo = relative (*abso); + } break; } default: - relo = relative (tp); + { + abso = &tp; + relo = relative (tp); + } } // Build the command line. @@ -7397,6 +7442,9 @@ namespace build2 small_vector<string, 2> header_args; // Header unit options storage. small_vector<string, 2> module_args; // Module options storage. + // NOTE: see a note in apply() on the compilation database implications + // if changing anything below. + // switch (cclass) { case compiler_class::msvc: @@ -7531,6 +7579,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); args.push_back ("/ifcOutput"); @@ -7744,6 +7793,9 @@ namespace build2 // Output module file is specified in the mapping file, the // same as input. // + // We set neither relm nor absm since they are not on the + // command line. + // if (ut == unit_type::module_header) // No obj, -c implied. break; @@ -7772,6 +7824,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); // Without this option Clang's .pcm will reference source @@ -7883,6 +7936,15 @@ namespace build2 else if (verb == 2) print_process (args); + // Insert or update the entry in the compilation database, if any. + // + compiledb::execute ( + bs, + t, tp, s, *sp, + cpath, args, + relo, abso != nullptr ? *abso : empty_path, + relm, absm != nullptr ? *absm : empty_path); + // If we have the (partially) preprocessed output, switch to that. // // But we remember the original source/position to restore later. diff --git a/libbuild2/cc/compiledb.cxx b/libbuild2/cc/compiledb.cxx new file mode 100644 index 0000000..ccf08a9 --- /dev/null +++ b/libbuild2/cc/compiledb.cxx @@ -0,0 +1,1100 @@ +// file : libbuild2/cc/compiledb.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <libbuild2/cc/compiledb.hxx> + +#include <cstring> // strlen() +#include <iostream> // cout + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/parser.hxx> +#endif + +#include <libbuild2/filesystem.hxx> +#include <libbuild2/diagnostics.hxx> + +#include <libbuild2/cc/module.hxx> + +#include <libbuild2/cc/target.hxx> +#include <libbuild2/bin/target.hxx> + +using namespace std; + +namespace build2 +{ + namespace cc + { + compiledb_set compiledbs; + + // compiledb + // + compiledb:: + ~compiledb () + { + } + + // Return true if this entry should be written to the database with the + // specified name. + // + static bool + filter (const scope& rs, + const core_module& m, + const string& name, + const file& ot, const file& it) + { + tracer trace ("cc::compiledb_filter"); + + bool r (true); + const char* w (nullptr); // Why r is false. + + // First check if writing to this database is enabled. + // + // No filter means not enabled. + // + if (m.cdb_filter_ == nullptr) + { + r = false; + w = "no database name filter"; + } + else + { + // Iterate in reverse (so that later values override earlier) and take + // the first name match. + // + r = false; + for (const pair<optional<string>, bool>& p: + reverse_iterate (*m.cdb_filter_)) + { + if (!p.first || *p.first == name) + { + r = p.second; + break; + } + } + + if (!r) + w = "no match in database name filter"; + } + + // Verify the name is known in this amalgamation. Note that without + // this check we may end up writing to unrelated databases in other + // amalgamations (think linked configurations). + // + if (r) + { + r = false; + for (const core_module* pm (&m); + pm != nullptr; + pm = pm->outer_module_) + { + const strings& ns (pm->cdb_names_); + + if (find (ns.begin (), ns.end (), name) != ns.end ()) + { + r = true; + break; + } + } + + if (!r) + w = "database name unknown in amalgamation"; + } + + // Filter based on the output target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_output_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_output_->empty ()) + { + r = false; + w = "empty output target type filter"; + } + else + { + const target_type& ott (ot.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_output_)) + { + if (p.first && *p.first != name) + continue; + + using namespace bin; + + const string& n (p.second); + + if (ott.name == n || n == "target") + { + r = true; + } + // + // Handle obj/bmi/hbmi{} groups ad hoc. + // + else if (n == "obj") + { + r = ott.is_a<obje> () || ott.is_a<objs> () || ott.is_a<obja> (); + } + else if (n == "bmi") + { + r = ott.is_a<bmie> () || ott.is_a<bmis> () || ott.is_a<bmia> (); + } + else if (n == "hbmi") + { + r = ott.is_a<hbmie> () || ott.is_a<hbmis> () || ott.is_a<hbmia> (); + } + else + { + // Handle the commonly-used, well-known targets directly (see + // note in core_config_init() for why we cannot pre-lookup + // them). + // + const target_type* tt ( + n == "obje" ? &obje::static_type : + n == "objs" ? &objs::static_type : + n == "obja" ? &obja::static_type : + n == "bmie" ? &bmie::static_type : + n == "bmis" ? &bmis::static_type : + n == "bmia" ? &bmia::static_type : + n == "hbmie" ? &hbmie::static_type : + n == "hbmis" ? &hbmis::static_type : + n == "hbmia" ? &hbmia::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.output value"; + + r = ott.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in output target type filter"; + } + } + + // Filter based on the input target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_input_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_input_->empty ()) + { + r = false; + w = "empty input target type filter"; + } + else + { + const target_type& itt (it.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_input_)) + { + if (p.first && *p.first != name) + continue; + + const string& n (p.second); + + if (itt.name == n || n == "target") + r = true; + else + { + // The same optimization as above. Note: cxx{}, etc., are in the + // cxx module so we have to look them up. + // + const target_type* tt ( + n == "c" ? &c::static_type : + n == "m" ? &m::static_type : + n == "S" ? &m::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.input value"; + + r = itt.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in input target type filter"; + } + } + + l6 ([&] + { + if (r) + trace << "keep " << ot << " in " << name; + else + trace << "omit " << ot << " from " << name << ": " << w; + }); + + return r; + } + + bool compiledb:: + match (const scope& bs, + const file& ot, const path_type& op, + const file& it, + bool changed) + { + if (compiledbs.empty ()) + return false; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + bool u (false); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + u = db->match (ot, op, changed) || u; + } + + return u; + } + + void compiledb:: + execute (const scope& bs, + const file& ot, const path_type& op, + const file& it, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + if (compiledbs.empty ()) + return; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + assert (relo.empty () == abso.empty () && + relm.empty () == absm.empty ()); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + db->execute (ot, op, it, ip, cpath, args, relo, abso, relm, absm); + } + } + + void + compiledb_pre (context& ctx, action a, const action_targets&) + { + // Note: won't be registered if compiledbs is empty. + + // Note: may be called directly with empty action_targets. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_pre"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->pre (ctx); + } + + void + compiledb_post (context& ctx, + action a, + const action_targets& ts, + bool failed) + { + // Note: won't be registered if compiledbs is empty. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_post"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx + << ", failed: " << failed;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->post (ctx, ts, failed); + } + +#ifndef BUILD2_BOOTSTRAP + + namespace json = butl::json; + + // compiledb_stdout + // + compiledb_stdout:: + compiledb_stdout (string n) + : compiledb (move (n), path_type ()), + state_ (state::init), + nesting_ (0), + js_ (cout, 0 /* indentation */, "" /* multi_value_separator */) + { + } + + void compiledb_stdout:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // The module context (used to build build system modules) poses a + // problem: we can receive its callbacks before the main context's or + // nested in the pre/post calls of the main context (or both, in + // fact). Plus there may be multiple pre/post sequences corresponding to + // the module context of both kinds. The three distinct cases are: + // + // 1. Module is loaded as part of the initial buildfile load (e.g., from + // root.build) -- in this case we will observe module pre/post before + // the main context's pre/post. + // + // In fact, to be precise, we will only observe them if cc is loaded + // before such a module. + // + // 2. Module is loaded via the interrupting load (e.g., from a directory + // buildfile that is loaded implicitly during match) -- in this case + // we will observe pre/post calls nested into the main context's + // pre/post. + // + // 3. The module context is used to build an ad hoc C++ recipe -- in + // this case we also get nested calls like in (2) since this happens + // during the recipe's match(). + // + // One thing to keep in mind (and which we rely upon quite a bit below) + // is that the main context's post will always be last (within any given + // operation; there could be another for the subsequent operation in a + // batch). + // + // Handling the nested case is relatively straightforward: we can keep + // track and ignore all the nested calls. + // + // The before case is where things get complicated. We could "take" the + // first module pre call and then wait until the main post, unless we + // see a module post call with failed=true, in which case there will be + // no further pre/post calls. There is, however, a nuance: the module is + // loaded and build for any operation, not just update, which means that + // if the main operation is not update (say, it's clean), we won't see + // any of the main context's pre/post calls. + // + // The way we are going to resolve this problem is different for the + // stdout and file implementations: + // + // For stdout we will just say that it should only be used with the + // update operation. There is really no good reason to use it with + // anything else anyway. See compiledb_stdout::post() for additional + // details. + // + // For file we will rely on its persistence and simply close and reopen + // the database for each pre/post sequence, the same way as if they were + // separate operations in a batch. + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::init) // First pre() call. + { + state_ = state::empty; + cout << "[\n"; + } + } + + bool compiledb_stdout:: + match (const file&, const path_type&, bool) + { + return true; + } + + static inline const char* + rel_to_abs (const char* a, + const string& rs, const string& as, + string& buf) + { + if (size_t rn = rs.size ()) + { + size_t an (strlen (a)); + + if (an >= rn && rs.compare (0, rn, a, rn) == 0) + { + if (an == rn) + return as.c_str (); + + buf = as; + buf.append (a + rn, an - rn); + + return buf.c_str (); + } + } + + return nullptr; + } + + void compiledb_stdout:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::full: + { + cout << ",\n"; + break; + } + case state::empty: + { + state_ = state::full; + break; + } + case state::failed: + return; + case state::init: + assert (false); + return; + } + + try + { + // Duplicate what we have in the file implementation (instead of + // factoring it out to something common) in case here we need to + // adjust things (change order, omit some values; for example to + // accommodate broken consumers). We have this freedom here but not + // there. + // + js_.begin_object (); + { + js_.member ("output", op.string ()); + js_.member ("file", ip.string ()); + + js_.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js_.value (r); + } + } + js_.end_array (); + + js_.member ("directory", work.string ()); + } + js_.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + } + + void compiledb_stdout:: + post (context& ctx, const action_targets&, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + bool mctx (ctx.module_context == &ctx); + + switch (state_) + { + case state::empty: + case state::full: + { + // If this is a module context's post, wait for the main context's + // post (last) unless the module load failed (in which case there + // will be no main pre/post). + // + // Note that there is no easy way to diagnose the case where we + // won't get the main pre/post calls. Instead, we will just produce + // invalid JSON (array won't be closed). In a somewhat hackish way, + // this actually makes the `b [-n] clean update` sequence work: we + // will take the pre() call from clean and the main post() from + // update. + // + if (mctx && !failed) + return; + + if (state_ == state::full) + cout << '\n'; + + cout << "]\n"; + break; + } + case state::failed: + return; + case state::init: + assert (false); + } + + state_ = state::init; + } + + // compiledb_file + // + compiledb_file:: + compiledb_file (string n, path_type p) + : compiledb (move (n), move (p)), + state_ (state::closed), + nesting_ (0) + { + } + + void compiledb_file:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // See compiledb_stdout::pre() for background on dealing with the module + // context. Here are some file-specific nuances: + // + // We are going to load the database on the first pre call and flush + // (but not close) it on the matching post. Flushing means that we will + // update the file but still keep the in-memory state, in case there is + // another pre/post session coming. This is both a performance + // optimization but also the way we handle prunning no longer present + // entries, which gets tricky across multiple pre/post sessions (see + // post() for details). + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::closed) // First pre() call. + { + // Load the contents of the file if it exists, marking all the entries + // as (presumed) absent. + // + if (exists (path)) + { + uint64_t line (1); + try + { + ifdstream ifs (path, ifdstream::badbit); + + // Parse the top-level array manually (see post() for the expected + // format). + // + auto throw_invalid_input = [] (const string& d) + { + throw json::invalid_json_input ("", 0, 1, 0, d); + }; + + enum {first, second, next, last, end} s (first); + + for (string l; !eof (getline (ifs, l)); line++) + { + switch (s) + { + case first: + { + if (l != "[") + throw_invalid_input ("beginning of array expected"); + + s = second; + continue; + } + case second: + { + if (l == "]") + { + s = end; + continue; + } + + s = next; + } + // Fall through. + case next: + { + if (!l.empty () && l.back () == ',') + l.pop_back (); + else + s = last; + + break; + } + case last: + { + if (l != "]") + throw_invalid_input ("end of array expected"); + + s = end; + continue; + } + case end: + { + throw_invalid_input ("junk after end of array"); + } + } + + // Parse just the output target path, which must come first. + // + json::parser jp (l, "" /* name */); + + jp.next_expect (json::event::begin_object); + string op (move (jp.next_expect_member_string ("output"))); + + auto r (db_.emplace (move (op), entry {entry_status::absent, l})); + if (!r.second) + throw_invalid_input ( + "duplicate output value '" + r.first->first + '\''); + } + + if (s != end) + throw_invalid_input ("corrupt input text"); + } + catch (const json::invalid_json_input& e) + { + state_ = state::failed; + + location l (path, line, e.column); + fail (l) << "invalid compilation database json input: " << e << + info << "remove this file if it was produced by a different tool"; + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to read " << path << ": " << e; + } + } + + absent_ = db_.size (); + changed_ = false; + + state_ = state::open; + } + } + + bool compiledb_file:: + match (const file&, const path_type& op, bool changed) + { + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return false; + case state::closed: + assert (false); + return false; + } + + // Mark an existing entry as present or changed. And if one does not + // exist, then (for now) as missing. + // + auto i (db_.find (op.string ())); + + if (i != db_.end ()) + { + entry& e (i->second); + + // Note: we can end up with present entries via the module context + // (see post() below). And we can see changed entries in a subsequent + // nested module context. + // + switch (e.status) + { + case entry_status::present: + case entry_status::changed: + assert (!changed); + break; + case entry_status::absent: + { + e.status = changed ? entry_status::changed : entry_status::present; + + absent_--; + changed_ = changed_ || (e.status == entry_status::changed); + break; + } + case entry_status::missing: + assert (false); + } + + return false; + } + else + { + db_.emplace (op.string (), entry {entry_status::missing, string ()}); + + changed_ = true; + + return true; + } + } + + void compiledb_file:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + auto i (db_.find (op.string ())); + + // We should have had the match() call before execute(). + // + assert (i != db_.end () && i->second.status != entry_status::absent); + + entry& e (i->second); + + if (e.status == entry_status::present) // Present and unchanged. + return; + + // The entry is either missing or changed. + // + try + { + e.json.clear (); + json::buffer_serializer js (e.json, 0 /* indentation */); + + js.begin_object (); + { + js.member ("output", op.string ()); // Note: must come first. + js.member ("file", ip.string ()); + + js.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js.value (r); + } + } + js.end_array (); + + js.member ("directory", work.string ()); + } + js.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + + e.status = entry_status::changed; + } + + void compiledb_file:: + post (context& ctx, const action_targets& ts, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + bool mctx (ctx.module_context == &ctx); + + tracer trace ("cc::compiledb_file::post"); + + // See if we need to update the file. + // + if (changed_) + l6 ([&]{trace << "updating due to missing/changed entries: " << path;}); + + // Don't prune the stale entries if the operation failed since we may + // not have gotten to execute some of them. + // + // And if this is a module context's post, then also don't prune the + // stale entries, instead waiting for the main context's post (if there + // will be one; this means we will only prune on update). + // + // Actually, this pruning business is even trickier than that: if we + // are not updating the entire project (say, rather only a subdirectory + // or even a specific target), then we will naturally not get any + // match/execute calls for targets of this project that don't get pulled + // into this build. Which means that we cannot just prune entries that + // we did not match/execute. It feels the correct semantics is to only + // prune the entries if they are in a subdirectory of the dir{} targets + // which we are building. + // + // What do we do about the module context, where we always update a + // specific libs{}? We could use its directory instead but that may lead + // to undesirable results. For example, if there are unit tests in the + // same directory, we will end up dropping their entries. It feels like + // the correct approach is to just ignore module context's entries + // entirely. If someone wants to prune the compilation database of a + // module, they will just need to update it directly (i.e., via the main + // context). Note that we cannot apply the same "simplification" to the + // changed entries since we will only observe the change once. + // + bool absent (false); + + if (!failed && !mctx && absent_ != 0) + { + // Pre-scan the entries and drop the appropriate absent ones. + // + for (auto i (db_.begin ()); i != db_.end (); ) + { + const entry& e (i->second); + + if (e.status == entry_status::absent) + { + // Absent entries should be rare enough during the normal + // development that we don't need to bother with caching the + // directories. + // + bool a (false); + for (const action_target& at: ts) + { + const target& t (at.as<target> ()); + if (t.is_a<dir> ()) + { + const string& p (i->first); + const string& d (t.out_dir ().string ()); + + if (path_traits::sub (p.c_str (), p.size (), + d.c_str (), d.size ())) + { + // Remove this entry from the in-memory state so that it + // matches the file state. + // + i = db_.erase (i); + --absent_; + a = absent = true; + break; + } + } + } + + if (a) + continue; + } + + ++i; + } + } + + if (absent) + l6 ([&]{trace << "updating due to absent entries: " << path;}); + + try + { + auto_rmfile rm; + ofdstream ofs; + + bool u (changed_ || absent); // Update the file. + + if (u) + { + rm = auto_rmfile (path); + ofs.open (path); + + // We parse the top-level array manually (see pre() above) and the + // expected format is as follows: + // + // [ + // {"output":...}, + // ... + // {"output":...} + // ] + // + ofs.write ("[\n", 2); + } + + // Iterate over the entries resetting their status and writing them to + // the file if necessary. + // + bool first (true); + for (auto& p: db_) + { + entry& e (p.second); + + // First sort out the status also skipping appropriate entries. + // + switch (e.status) + { + case entry_status::absent: + { + // This is an absent entry that we should keep (see pre-scan + // above). + // + break; + } + case entry_status::missing: + { + // This should only happen if this operation has failed (see + // also below) or we are in the match-only mode. + // + assert (failed || ctx.match_only); + continue; + } + case entry_status::present: + case entry_status::changed: + { + // This is tricky: if this is a module context, then we don't + // want to mark the entries as absent since they will then get + // dropped by the main operation context. + // + if (mctx) + e.status = entry_status::present; + else + { + // Note: this is necessary for things to work across multiple + // operations in a batch. + // + e.status = entry_status::absent; + absent_++; + } + } + } + + if (u) + { + if (first) + first = false; + else + ofs.write (",\n", 2); + + ofs.write (e.json.c_str (), e.json.size ()); + } + } + + if (u) + { + ofs.write (first ? "]\n" : "\n]\n", first ? 2 : 3); + + ofs.close (); + rm.cancel (); + } + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to write to " << path << ": " << e; + } + + // If this operation has failed, then our state may not be accurate + // (e.g., entries with missing status) but we also don't expect any + // further pre calls. Let's change out state to failed as a sanity + // check. + // + if (failed) + state_ = state::failed; + else + changed_ = false; + + // Note: keep in the open state (see pre() for details). + } + +#endif // BUILD2_BOOTSTRAP + } +} diff --git a/libbuild2/cc/compiledb.hxx b/libbuild2/cc/compiledb.hxx new file mode 100644 index 0000000..edfd1ee --- /dev/null +++ b/libbuild2/cc/compiledb.hxx @@ -0,0 +1,226 @@ +// file : libbuild2/cc/compiledb.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_CC_COMPILEDB_HXX +#define LIBBUILD2_CC_COMPILEDB_HXX + +#include <unordered_map> + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/serializer.hxx> +#endif + +#include <libbuild2/types.hxx> +#include <libbuild2/utility.hxx> + +#include <libbuild2/target.hxx> +#include <libbuild2/action.hxx> +#include <libbuild2/context.hxx> + +namespace build2 +{ + namespace cc + { + using compiledb_name_filter = vector<pair<optional<string>, bool>>; + using compiledb_type_filter = vector<pair<optional<string>, string>>; + + class compiledb + { + public: + // Match callback where we confirm an entry in the database and also + // signal whether it has changes (based on change tracking in depdb). + // Return true to force compilation of this target and thus make sure + // the below execute() is called (unless something before that failed). + // + // Besides noticing changes, this callback is also necessary to notice + // and delete entries that should no longer be in the database (e.g., a + // source file was removed from the project). + // + // Note that output is either obj*{}, bmi*{}, of hbmi*{}. + // + static bool + match (const scope& bs, + const file& output, const path& output_path, + const file& input, + bool changed); + + // Execute callback where we insert or update an entry in the database. + // + // The {relo, abso}, and {relm, absm} pairs are used to "untranslate" + // relative paths to absolute. Specifically, any argument that has rel? + // as a prefix has this prefix replaced with the corresponding abs?. + // Note that this means we won't be able to handle old MSVC and + // clang-cl, which don't support the `/F?: <path>` form, only + // `/F?<path>`. Oh, well. Note also that either relo or relm (but not + // both) could be empty if unused. + // + // Note also that we assume the source file is always absolute and is + // the last argument. + // + static void + execute (const scope& bs, + const file& output, const path& output_path, + const file& input, const path& input_path, + const process_path& cpath, const cstrings& args, + const path& relo, const path& abso, + const path& relm, const path& absm); + + public: + using path_type = build2::path; + + string name; + path_type path; + + // The path is expected to be absolute and normalized or empty if the + // name is `-` (stdout). + // + compiledb (string n, path_type p) + : name (move (n)), path (move (p)) + { + } + + virtual void + pre (context&) = 0; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) = 0; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) = 0; + + virtual void + post (context&, const action_targets&, bool failed) = 0; + + virtual + ~compiledb (); + }; + + using compiledb_set = vector<unique_ptr<compiledb>>; + + // Populated by core_config_init() during serial load. + // + extern compiledb_set compiledbs; + + // Context operation callbacks. + // + void + compiledb_pre (context&, action, const action_targets&); + + void + compiledb_post (context&, action, const action_targets&, bool failed); + +#ifndef BUILD2_BOOTSTRAP + + // Implementation that writes to stdout. + // + // Note that this implementation forces compilation of all the targets for + // which it is called to make sure their entries are in the database. So + // typically used in the dry run mode. + // + class compiledb_stdout: public compiledb + { + public: + // The path is expected to be empty. + // + explicit + compiledb_stdout (string name); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {init, empty, full, failed} state_; + size_t nesting_; + butl::json::stream_serializer js_; + }; + + // Implementation that maintains a file. + // + class compiledb_file: public compiledb + { + public: + compiledb_file (string name, path_type path); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {closed, open, failed} state_; + size_t nesting_; + + // We want to optimize the performance for the incremental update case + // where only a few files will be recompiled and most of the time there + // will be no change in the command line, which means we won't need to + // rewrite the file. + // + // As a result, our in-memory representation is a hashmap (we could have + // thousands of entries) of absolute and normalized output file paths + // (stored as strings for lookup efficiency) to their serialized JSON + // text lines plus the status: absent, present, changed, or missing + // (entry should be there but is not). This way we don't waste + // (completely) parsing (and re-serializing) each line knowing that we + // won't need to touch most of them. + // + // In fact, we could have gone even further and used a sorted vector + // since insertions will be rare in this case. But we will need to + // lookup every entry on each update, so it's unclear this is a win. + // + enum class entry_status {absent, present, changed, missing}; + + struct entry + { + entry_status status; + string json; + }; + + using map_type = std::unordered_map<string, entry>; + map_type db_; + + // Number/presence of various entries in the database (used to determine + // whether we need to update the file without iterating over all the + // entries). + // + size_t absent_; // Number of absent entries. + bool changed_; // Presence of changed or missing entries. + }; + +#endif // BUILD2_BOOTSTRAP + } +} + +#endif // LIBBUILD2_CC_COMPILEDB_HXX diff --git a/libbuild2/cc/guess.cxx b/libbuild2/cc/guess.cxx index 5ae6fb2..2d079f0 100644 --- a/libbuild2/cc/guess.cxx +++ b/libbuild2/cc/guess.cxx @@ -949,9 +949,15 @@ namespace build2 // there is nothing like -m32/-m64 or /MACHINE). Targeting // 64-bit seems like as good of a default as any. // - fb = ((dir_path (mi->msvc_dir) /= "bin") /= "Hostx64") /= - "x64"; - + fb = dir_path (mi->msvc_dir) /= "bin"; + +#if defined(_M_ARM64) || defined(__aarch64__) + fb /= "HostARM64"; + fb /= "ARM64"; +#else + fb /= "Hostx64"; + fb /= "x64"; +#endif search_info = info_ptr ( new msvc_info (move (*mi)), msvc_info_deleter); } @@ -1629,15 +1635,25 @@ namespace build2 // Seeing that we only do 64-bit on Windows, let's always use 64-bit // MSVC tools (link.exe, etc). In case of the Platform SDK, it's unclear - // what the CPU signifies (host, target, both). + // what the CPU signifies (host, target, both). It appears to be host. // - r = (((dir_path (mi.msvc_dir) /= "bin") /= "Hostx64") /= cpu). - representation (); + r = (((dir_path (mi.msvc_dir) /= "bin") /= +#if defined(_M_ARM64) || defined(__aarch64__) + "HostARM64" +#else + "Hostx64" +#endif + ) /= cpu).representation (); r += path::traits_type::path_separator; - r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= cpu). - representation (); + r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= +#if defined(_M_ARM64) || defined(__aarch64__) + "arm64" +#else + "x64" +#endif + ).representation (); return r; } @@ -1878,7 +1894,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - "", + "", // Checksum to be calculated from signature. move (t), move (ot), move (cpat), @@ -2128,7 +2144,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - move (gr.checksum), // Calculated on whole -v output. + "", // Checksum calculated on whole -v output. move (t), move (ot), move (pat), @@ -2898,7 +2914,7 @@ namespace build2 move (ver), move (var_ver), move (gr.signature), - move (gr.checksum), // Calculated on whole -v output. + "", // Checksum calculated on whole -v output. move (t), move (ot), move (cpat), @@ -3216,7 +3232,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - "", + "", // Checksum to be calculated from signature. move (t), move (ot), move (pat), @@ -3370,6 +3386,8 @@ namespace build2 cs.append (gr.type_signature); } + cs.append (r.path.effect_string ()); + r.checksum = cs.string (); // Derive binutils pattern unless this has already been done by the diff --git a/libbuild2/cc/guess.hxx b/libbuild2/cc/guess.hxx index 7cbbd87..dfa8aa2 100644 --- a/libbuild2/cc/guess.hxx +++ b/libbuild2/cc/guess.hxx @@ -160,6 +160,9 @@ namespace build2 // checksum will still change. This is currently the case for all the // compilers that we support. // + // And we assume that the checksum incorporates the absolute compiler + // path. This is used to detect compilation database changes. + // // The target is the compiler's traget architecture triplet. Note that // unlike all the preceding fields, this one takes into account the // compile options (e.g., -m32). diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx index e124450..d691bc5 100644 --- a/libbuild2/cc/init.cxx +++ b/libbuild2/cc/init.cxx @@ -10,8 +10,10 @@ #include <libbuild2/config/utility.hxx> +#include <libbuild2/cc/module.hxx> #include <libbuild2/cc/target.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using namespace std; using namespace butl; @@ -23,7 +25,7 @@ namespace build2 // Scope operation callback that cleans up module sidebuilds. // static target_state - clean_module_sidebuilds (action, const scope& rs, const dir&) + clean_module_sidebuilds (const scope& rs) { context& ctx (rs.ctx); @@ -67,6 +69,131 @@ namespace build2 return target_state::unchanged; } + // Scope operation callback that cleans up compilation databases. + // + static target_state + clean_compiledb (const scope& rs) + { + context& ctx (rs.ctx); + + target_state r (target_state::unchanged); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + const path& p (db->path); + + if (p.empty () || + ctx.scopes.find_out (p.directory ()).root_scope () != &rs) + continue; + + if (rmfile (ctx, p)) + r = target_state::changed; + } + + return r; + } + + // Scope operation callback for cleaning module sidebuilds and compilation + // databases. + // + static target_state + clean_callback (action, const scope& rs, const dir&) + { + target_state r (clean_module_sidebuilds (rs)); + + if (!compiledbs.empty ()) + r |= clean_compiledb (rs); + + return r; + } + + // Detect if just <name> in the <name>[@<path>] form is actually <path>. + // We assume it is <path> and not <name> if it contains a directory + // component or is the special directory name (`.`/`..`) . If that's the + // case, return canonicalized name representing <path>. See the call site + // in core_config_init() below for background. + // + static optional<name> + compiledb_name_to_path (const name& n) + { + if (n.directory ()) + return n; + + if (n.file ()) + { + if (!n.dir.empty () || + path_traits::find_separator (n.value) != string::npos) + { + name r (n); + r.canonicalize (); + return r; + } + else if (n.value == "." || n.value == "..") + { + return name (dir_path (n.value)); + } + } + + return nullopt; + } + + // Custom save function that completes relative paths in the + // config.cc.compiledb and config.cc.compiledb.name values. + // + static pair<names_view, const char*> + save_compiledb_name (const scope&, + const value& v, + const value*, + names& storage) + { + const names& ns (v.as<names> ()); // Value is untyped. + + // Detect and handle the case where just <name> is actually <path>. + // + if (ns.size () == 1) + { + const name& n (ns.back ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + storage.push_back (move (tn)); + return make_pair (names_view (storage), "="); + } + } + + if (find_if (ns.begin (), ns.end (), + [] (const name& n) {return n.pair;}) == ns.end ()) + { + return make_pair (names_view (ns), "="); + } + + storage = ns; + for (auto i (storage.begin ()); i != storage.end (); ++i) + { + if (i->pair) + { + name& n (*++i); + + if (!n.directory ()) + n.canonicalize (); + + if (n.dir.relative ()) + n.dir.complete (); + + n.dir.normalize (); + } + } + + return make_pair (names_view (storage), "="); + } + bool core_vars_init (scope& rs, scope&, @@ -107,6 +234,22 @@ namespace build2 vp.insert<abs_dir_path> ("config.cc.pkgconfig.sysroot"); + // Compilation database. + // + // See the manual for the semantics. + // + // config.cc.compiledb -- <name>[@<path>]|<path> (untyped) + // config.cc.compiledb.name -- <name>[@<path>]... (untyped) + // config.cc.compiledb.filter -- [<name>@]<bool>... + // config.cc.compiledb.filter.input -- [<name>@]<target-type>... + // config.cc.compiledb.filter.output -- [<name>@]<target-type>... + // + vp.insert ("config.cc.compiledb"); + vp.insert ("config.cc.compiledb.name"); + vp.insert<compiledb_name_filter> ("config.cc.compiledb.filter"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.input"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.output"); + vp.insert<strings> ("cc.poptions"); vp.insert<strings> ("cc.coptions"); vp.insert<strings> ("cc.loptions"); @@ -192,16 +335,6 @@ namespace build2 // vp.insert<bool> ("cc.serialize"); - // Register scope operation callback. - // - // It feels natural to clean up sidebuilds as a post operation but that - // prevents the (otherwise-empty) out root directory to be cleaned up - // (via the standard fsdir{} chain). - // - rs.operation_callbacks.emplace ( - perform_clean_id, - scope::operation_callback {&clean_module_sidebuilds, nullptr /*post*/}); - return true; } @@ -292,6 +425,8 @@ namespace build2 assert (first); + context& ctx (rs.ctx); + // Load cc.core.guess. // load_module (rs, rs, "cc.core.guess", loc); @@ -312,7 +447,6 @@ namespace build2 // // @@ Same nonsense as in module. // - // rs.assign ("cc.poptions") += cast_null<strings> ( lookup_config (rs, "config.cc.poptions", nullptr)); @@ -363,21 +497,16 @@ namespace build2 if (!cast_false<bool> (rs["bin.config.loaded"])) { // Prepare configuration hints (pretend it belongs to root scope). - // They are only used on the first load of bin.config so we only - // populate them on our first load. // variable_map h (rs); - if (first) - { - // Note that all these variables have already been registered. - // - h.assign ("config.bin.target") = - cast<target_triplet> (rs["cc.target"]).representation (); + // Note that all these variables have already been registered. + // + h.assign ("config.bin.target") = + cast<target_triplet> (rs["cc.target"]).representation (); - if (auto l = extra.hints["config.bin.pattern"]) - h.assign ("config.bin.pattern") = cast<string> (l); - } + if (auto l = extra.hints["config.bin.pattern"]) + h.assign ("config.bin.pattern") = cast<string> (l); init_module (rs, rs, "bin.config", loc, false /* optional */, h); } @@ -386,7 +515,6 @@ namespace build2 // ourselves since the target can come from the configuration and not // our hint). // - if (first) { const auto& ct (cast<target_triplet> (rs["cc.target"])); const auto& bt (cast<target_triplet> (rs["bin.target"])); @@ -416,6 +544,447 @@ namespace build2 if (tsys == "mingw32") load_module (rs, rs, "bin.rc.config", loc); + // Find the innermost outer core_module, if any. + // + const core_module* om (nullptr); + for (const scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if ((om = s->find_module<core_module> (core_module::name)) != nullptr) + break; + } + + auto& m (extra.set_module (new core_module (om))); + + // config.cc.compiledb.* + // + { + // For config.cc.compiledb and config.cc.compiledb.name we only + // consider a value in this root scope (if it's inherited from the + // outer scope, then that's where it will be handled). One special + // case is when it's specified on a scope that doesn't load the cc + // module (including, ultimately, the global scope for a global + // override). We handle it by assuming the value belongs to the + // outermost amalgamation that loads the cc module. + // + // Note: cache the result. + // + auto find_outermost = + [&rs, o = optional<pair<scope*, core_module*>> ()] () mutable + { + if (!o) + { + o = pair<scope*, core_module*> (&rs, nullptr); + for (scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if (auto* m = s->find_module<core_module> (core_module::name)) + { + o->first = s; + o->second = m; + } + } + } + + return *o; + }; + + auto belongs = [&rs, &find_outermost] (const lookup& l) + { + return l.belongs (rs) || find_outermost ().first == &rs; + }; + + // Add compilation databases specified in ns as <name>[@<path>] pairs, + // appending their names to cdb_names. If <path> is absent, then place + // the database into the base directory. Return the last added name. + // + auto add_cdbs = [&ctx, + &loc, + &trace] (strings& cdb_names, + const names& ns, + const dir_path& base) -> const string& + { + // Check that names and paths match. Return false if this entry + // already exist. + // + // Note that before we also checked that the same paths are not used + // across contexts. But, actually, there doesn't seem to be anything + // wrong with that and this can actually be useful, for example, + // when developing build system modules. + // + auto check = [&loc] (const string& n, const path& p) + { + for (const unique_ptr<compiledb>& db: compiledbs) + { + bool nm (db->name == n); + bool pm (db->path == p); + + if (nm != pm) + fail (loc) << "inconsistent compilation database names/paths" << + info << p << " is called " << n << + info << db->path << " is called " << db->name; + + if (nm) + return false; + } + + return true; + }; + + const string* r (&empty_string); + + bool reg (false); + size_t j (compiledbs.size ()); // First newly added database. + for (auto i (ns.begin ()); i != ns.end (); ++i) + { + // Each element has the <name>[@<path>] form. + // + // The special `-` <name> signifies stdout. + // + // If <path> is absent, then the file is called <name>.json and + // placed into the output directory of the amalgamation or project + // root scope (passed as the base argument). + // + // If <path> is (syntactically) a directory, then the file path is + // <path>/<name>.json. + // + if (!i->simple () || i->empty ()) + fail (loc) << "invalid compilation database name '" << *i << "'"; + + // Don't allow names that have (or are) directory components. + // + if (compiledb_name_to_path (*i)) + fail (loc) << "directory component in compilation database name '" + << *i << "'"; + + string n (i->value); + + path p; + if (i->pair) + { + ++i; + + if (n == "-") + fail (loc) << "compilation database path specified for stdout " + << "name"; + try + { + if (i->directory ()) + p = i->dir / n + ".json"; + else if (i->file ()) + { + if (i->dir.empty ()) + p = path (i->value); + else + p = i->dir / i->value; + } + else + throw invalid_path (""); + + if (p.relative ()) + p.complete (); + + p.normalize (); + } + catch (const invalid_path&) + { + fail (loc) << "invalid compilation database path '" << *i + << "'"; + } + } + else if (n != "-") + { + p = base / n + ".json"; + } + + if (check (n, p)) + { + reg = compiledbs.empty (); // First time. + +#ifdef BUILD2_BOOTSTRAP + fail (loc) << "compilation database requested during bootstrap"; +#else + if (n == "-") + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_stdout (n))); + else + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_file (n, move (p)))); +#endif + } + + // We may end up with duplicates via the config.cc.compiledb + // logic. + // + auto k (find (cdb_names.begin (), cdb_names.end (), n)); + + if (k == cdb_names.end ()) + { + cdb_names.push_back (move (n)); + r = &cdb_names.back (); + } + else + r = &*k; + } + + // Register context operation callback for compiledb generation. + // + // We have two complications here: + // + // 1. We could be performing all this from the load phase that + // interrupted the match phase, which means the point where the + // pre callback would have been called is already gone (but the + // post callback will still be called). This will happen if we, + // say, import a project that has a compilation database from a + // project that doesn't. + // + // (Note that if you think that this can be solved by simply + // always registering the callbacks, regardless of whether we + // have any databases or not, consider a slightly different + // scenario where we import a project that loads the cc module + // from a project that does not). + // + // What we are going to do in this case is simply call the pre + // callback manually. + // + // 2. We could again be performing all this from the load phase that + // interrupted the match phase, but this time the pre callback + // has already been called, which means there will be no pre() + // call for the newly added database(s). This will happen if we, + // say, import a project that has a compilation database from a + // project that also has one. + // + // Again, what we are going to do in this case is simply call the + // pre callback for the new database(s) manually. + // + if (reg) + ctx.operation_callbacks.emplace ( + perform_update_id, + context::operation_callback {&compiledb_pre, &compiledb_post}); + + if (ctx.load_generation > 1) + { + action a (ctx.current_action ()); + + if (a.inner_action () == perform_update_id) + { + if (reg) // Case #1. + { + l6 ([&]{trace << "direct compiledb_pre for context " << &ctx;}); + compiledb_pre (ctx, a, action_targets {}); + } + else // Case #2. + { + size_t n (compiledbs.size ()); + + if (j != n) + { + l6 ([&]{trace << "additional compiledb for context " << &ctx;}); + + for (; j != n; ++j) + compiledbs[j]->pre (ctx); + } + } + } + } + + return *r; + }; + + lookup l; + + // config.cc.compiledb + // + // The semantics of this value is as follows: + // + // Location: outermost amalgamation that loads the cc module. + // Name filter: enable from this scope unless specified explicitly. + // Type filter: enable from this scope unless specified explicitly. + // + // Note: save omitted. + // + optional<string> enable_filter; + + l = lookup_config (rs, "config.cc.compiledb", 0, &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb specified on " << rs;}); + + const names& ns (cast<names> (l)); + + // Make sure it's one name/path. + // + size_t n (ns.size ()); + if (n == 0 || n != (ns.front ().pair ? 2 : 1)) + fail (loc) << "invalid compilation database name '" << ns << "'"; + + // Detect and translate just <name> which is actually <path> to the + // <name>@<path> form: + // + // - The <name> part is the name of the directory where the database + // file will reside (typically project/repository or package + // name). + // + // - If <path> is a directory, then the database name is + // compile_commands.json. + // + names tns; + if (n == 1) + { + const name& n (ns.front ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + // Note: the add_cdbs() call below completes and normalizes the + // path but we need to do it earlier in order to be able to + // derive the name (the last component can be `.`/`..`). + // + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + if (!exists (tn.dir)) + fail (loc) << "compilation database directory " << tn.dir + << " does not exist"; + + if (tn.value.empty ()) + tn.value = "compile_commands.json"; + + tns.push_back (name (tn.dir.leaf ().string ())); + tns.back ().pair = '@'; + tns.push_back (move (tn)); + } + } + + // We inject the database directly into the outer amalgamation's + // module, as-if config.cc.compiledb.name was specified in its + // scope. Unless there isn't one, in which case it's us. + // + pair<scope*, core_module*> p (find_outermost ()); + + // Save the name for the name filter below. + // + enable_filter = add_cdbs ( + (p.second != nullptr ? *p.second : m).cdb_names_, + tns.empty () ? ns : tns, + p.first->out_path ()); + } + + // config.cc.compiledb.name + // + // Note: save omitted. + // + l = lookup_config (rs, + "config.cc.compiledb.name", + 0, + &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb.name specified on " << rs;}); + + add_cdbs (m.cdb_names_, cast<names> (l), rs.out_path ()); + } + + // config.cc.compiledb.filter + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_ = &cast<compiledb_name_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_ != nullptr) + m.cdb_filter_storage_ = *om->cdb_filter_; + + m.cdb_filter_storage_.emplace_back (*enable_filter, true); + m.cdb_filter_ = &m.cdb_filter_storage_; + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_ = om->cdb_filter_; + } + + // config.cc.compiledb.filter.input + // config.cc.compiledb.filter.output + // + // Note that filtering happens before we take into account the change + // status, which means for larger projects there would be a lot of + // targets to filter even during the incremental update. So it feels + // it would have been better to pre-lookup the target types. However, + // the targets that would normally be used are registered by other + // modules (bin, c/cxx) and which haven't been loaded yet. So instead + // we try to optimize the lookup for the commonly used targets. + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter.input"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_input_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_input_ != nullptr) + { + m.cdb_filter_input_storage_ = *om->cdb_filter_input_; + m.cdb_filter_input_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_input_ = &m.cdb_filter_input_storage_; + } + else + m.cdb_filter_input_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_input_ = om->cdb_filter_input_; + } + + l = lookup_config (rs, "config.cc.compiledb.filter.output"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_output_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_output_ != nullptr) + { + m.cdb_filter_output_storage_ = *om->cdb_filter_output_; + m.cdb_filter_output_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_output_ = &m.cdb_filter_output_storage_; + } + else + m.cdb_filter_output_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_output_ = om->cdb_filter_output_; + } + } + + // Register scope operation callback for cleaning module sidebuilds and + // compilation databases. + // + // It feels natural to clean this stuff up as a post operation but that + // prevents the (otherwise-empty) out root directory to be cleaned up + // (via the standard fsdir{} chain). + // + rs.operation_callbacks.emplace ( + perform_clean_id, + scope::operation_callback {&clean_callback, nullptr /*post*/}); + return true; } diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx index 8c68b64..a669f37 100644 --- a/libbuild2/cc/link-rule.cxx +++ b/libbuild2/cc/link-rule.cxx @@ -503,8 +503,8 @@ namespace build2 return false; } - // We will only chain a C source if there is also an X source or we were - // explicitly told to. + // We will only synthesize a dependency for a C source if there is also + // an X source or we were explicitly told to. // if (r.seen_c && !r.seen_x && hint.empty ()) { @@ -915,8 +915,8 @@ namespace build2 const fsdir* dir (inject_fsdir (a, t)); // Process prerequisites, pass 1: search and match prerequisite - // libraries, search obj/bmi{} targets, and search targets we do rule - // chaining for. + // libraries, search obj/bmi{} targets, and search targets we do + // dependency synthesis for. // // Also clear the binless flag if we see any source or object files. // Note that if we don't see any this still doesn't mean the library is @@ -928,7 +928,7 @@ namespace build2 // compile::apply() to unmatch them and therefore not to hinder // parallelism (or mess up for-install'ness). // - // We also create obj/bmi{} chain targets because we need to add + // We also synthesize obj/bmi{} dependencies because we need to add // (similar to lib{}) all the bmi{} as prerequisites to all the other // obj/bmi{} that we are creating. Note that this doesn't mean that the // compile rule will actually treat them all as prerequisite targets. @@ -955,6 +955,20 @@ namespace build2 auto& pts (t.prerequisite_targets[a]); size_t start (pts.size ()); + // Compile rule options specified on lib/exe{} to propagate to obj/bmi{} + // during dependency synthesis (see below). + // + struct cr_options + { + const strings* c_p; // cc.poptions + const strings* x_p; // x.poptions + const strings* c_c; // cc.coptions + const strings* x_c; // x.coptions + + bool member; // Any value came from member as opposed to group. + }; + optional<cr_options> cr_ops; // Lookup lazily. + for (prerequisite_member p: group_prerequisite_members (a, t)) { // Note that we have to recognize update=match for *(update), not just @@ -1025,7 +1039,14 @@ namespace build2 continue; } - const target*& pt (pto); + const target*& pt (pto.target); + + // Auxiliary data in prerequisite_target: + // + // - for libraries it stores link flags (lflag_whole) + // - for synthesized obj/bmi{} it strores the group flag + // + uintptr_t& pd (pto.data); // Mark (2 bits): // @@ -1045,7 +1066,7 @@ namespace build2 { binless = binless && (mod ? user_binless : false); - // Rule chaining, part 1. + // Dependency synthesis, part 1. // // Which scope shall we use to resolve the root? Unlikely, but // possible, the prerequisite is from a different project @@ -1058,6 +1079,62 @@ namespace build2 // bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + // Lookup all the relevant binary-specific compile option values if + // this hasn't already been done. + // + if (!cr_ops) + { + bool m (false); + + auto lookup = [&bs, &t, &m] (const variable& var) -> const strings* + { + // We don't want to pick anything beyond target-specific + // values (but including target type/pattern-specific) since + // they will also be in effect for obj/bmi{} and setting them + // to the same values is a waste (and, in fact, it could be + // that when this feature is not used, different obj/bmi{} are + // compiled with different options). + // + // Note that we need to take into account target type/pattern- + // specific append/prepend since it could modify the scope + // value but only be applicable to lib/exe{}. Something like + // this: + // + // cc.poptions += -DFOO + // lib{*}: cc.poptions += -DBAR + // + // Then there is this nuance: if any value came from the member + // and not from the group, then we need to override the above + // group semantics. In particular, this allows us to do: + // + // liba{hello}: cxx.poptions += -DLIBHELLO_STATIC_BUILD + // libs{hello}: cxx.poptions += -DLIBHELLO_SHARED_BUILD + // + // Note also that the variables we are dealing with are not + // overridable. + // + pair<build2::lookup, size_t> p ( + t.lookup_original (var, + lookup_limit::target_type, + &bs)); + + const strings* r (cast_null<strings> (p.first)); + + if (r != nullptr) + m = m || p.second == 1; // Found in the target itself. + + return r; + }; + + cr_ops = cr_options {lookup (c_poptions), lookup (x_poptions), + lookup (c_coptions), lookup (x_coptions), + false}; + cr_ops->member = m; + } + + if (group && cr_ops->member) + group = false; + const target_type& rtt (mod ? (group ? bmi::static_type : tts.bmi) : (group ? obj::static_type : tts.obj)); @@ -1091,44 +1168,278 @@ namespace build2 // obj/bmi{} is always in the out tree. Note that currently it could // be the group -- we will pick a member in part 2 below. // + // Note: d is used below. + // pair<target&, ulock> r ( search_new_locked ( ctx, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope)); - // If we shouldn't clean obj{}, then it is fair to assume we - // shouldn't clean the source either (generated source will be in - // the same directory as obj{} and if not, well, go find yourself - // another build system ;-)). - // - if (skip (r.first)) - { - pt = nullptr; - continue; - } + const target& cpt (r.first); + bool locked (r.second.owns_lock ()); - // Either set of verify the bin.binless value on this bmi*{} target + // Either set or verify the bin.binless value on this bmi*{} target // (see config_data::b_binless for semantics). // if (mod) { - if (r.second.owns_lock ()) + if (locked) { if (user_binless) r.first.assign (b_binless) = true; } else { - lookup l (r.first[b_binless]); + lookup l (cpt[b_binless]); if (user_binless ? !cast_false<bool> (l) : l.defined ()) fail << "synthesized dependency for prerequisite " << p << " would be incompatible with existing target " - << r.first << + << cpt << info << "incompatible bin.binless value"; } } - pt = &r.first; + // Binary-specific compile options. + // + // Propagate compile rule options (*.poptions and *.coptions) + // specified on the binary to the obj/bmi{} targets that we + // synthesize. + // + // The semantics we are aiming for is as-if they were set on the + // obj/bmi{} target at the end of the buildfile (to be precise, at + // the end of loading all buildfiles for the scope). + // + // While ideally we would like to prevent sharing such obj/bmi{} + // between multiple binaries or for the user to specify any compile + // options explicitly, this is not easy to do (since once the + // options are set, we don't know who set them: same binary on the + // previous operation batch, another binary, or user). Instead, we + // are going to approximate this by making sure the options (or + // their absence) match. + // + // Note also that we don't touch user-specified obj/bmi{} + // prerequisites (neither set nor verify). In particular, this + // allows customizing compile options for specific translation + // units. + // + // NOTE: keep last since unlocks the lock. + // + { + // If we have any value, then either set them (locked) or make + // sure they all match (unlocked). + // + // Note that the case where one lib/exe{} specifies a value while + // the other doesn't specify any (and they share a synthesized + // dependency) will be racy to verify. Getting rid of this race + // will be difficult because in the verify case we can't say who + // set the value on obj/bmi{}. We could set some sort of a marker + // variable but then it means we would need to look it up for + // every lib/exe{} (whether they use this feature or not, and most + // won't). However, we can handle the common case based on the + // target being newly created (as opposed to being mentioned in + // the buildfile; see target_decl). + // + auto check = [&bs, &p, &d] (const variable& var, + const target& et, + const strings* e, + const target& st) + { + // If the expected value is NULL, then just make sure this + // variable is not set on the target. Otherwise, compare the + // result of the normal lookup and if it matches, then we assume + // it's good regardless of where it comes from (this covers all + // the corner cases which we cannot verify precisely; see above + // for details). + // + const strings* v; + size_t vd (0); + + if (e == nullptr) + { + v = cast_null<strings> (st.vars[var]); + + if (v == nullptr) + return; + } + else + { + // Optimize the lookup for the common case. + // + pair<lookup, size_t> p ( + st.lookup_original ( + var, + d.empty () ? &bs : nullptr)); + + v = cast_null<strings> (p.first); + + if (v != nullptr && *v == *e) + return; + + vd = p.second; + } + + diag_record dr (fail); + + dr << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << st << + info << "variable " << var << " value mismatch"; + + if (e == nullptr) // Expected to be absent. + { + dr << info << st << " value: "; to_stream_quoted (dr.os, *v); + dr << info << et << " value is absent"; + } + else if (v == nullptr) // Expected to be present. + { + dr << info << st << " value is absent"; + dr << info << et << " value: "; to_stream_quoted (dr.os, *e); + } + else // Expected to match. + { + dr << info << st << " value: "; to_stream_quoted (dr.os, *v); + dr << info << et << " value: "; to_stream_quoted (dr.os, *e); + + if (vd > 1) + { + if (const target* g = st.group) + dr << info << st << " value came from group " << *g; + } + } + }; + + auto set = [&r] (const variable& var, const strings& v) + { + // One nuance here is that target type/pattern-specific + // append/prepend/assign specified for obj/bmi{} will not be + // in effect for options specified on lib/exe{}. For example: + // + // obj{*}: cc.poptions = -DFOO + // lib{bar}: cc.poptions += -DBAR + // + // It doesn't seem there is anything sensible we can do about + // it automatically other than documenting this nuance and + // suggesting the user adds lib/exe{} to such a pattern. + // + r.first.assign (var) = v; + }; + + bool absent (false); + if (cr_ops->c_p != nullptr || cr_ops->x_p != nullptr || + cr_ops->c_c != nullptr || cr_ops->x_c != nullptr || + (absent = !operator>= (cpt.decl, target_decl::implied))) // VC14 + { + if (locked) + { + if (!absent) + { + if (cr_ops->c_p != nullptr) set (c_poptions, *cr_ops->c_p); + if (cr_ops->x_p != nullptr) set (x_poptions, *cr_ops->x_p); + if (cr_ops->c_c != nullptr) set (c_coptions, *cr_ops->c_c); + if (cr_ops->x_c != nullptr) set (x_coptions, *cr_ops->x_c); + } + + // @@ PERF: maybe pass the lock to search_new_locked() below? + // + r.second.unlock (); + locked = false; + } + else + { + if (absent && cpt.vars.empty ()) + ; // Optimize for the common case. + else + { + // If the values came from the group, then use the group in + // diagnostics. + // + const target& et (group ? *t.group : t); + + check (c_poptions, et, cr_ops->c_p, cpt); + check (x_poptions, et, cr_ops->x_p, cpt); + check (c_coptions, et, cr_ops->c_c, cpt); + check (x_coptions, et, cr_ops->x_c, cpt); + } + } + + // Verify member/group consistency. + // + // The check above doesn't quite work for libraries where the + // one that specified any options will likely end up + // synthesizing obja/objs{} targets (see the group logic above) + // while the one that didn't -- obj{}. So we also need to check + // obj{} vs obj[as]{} consistency if both are synthesized. + // + // (This is actually even hairier than that: sometimes, the + // obj{} library will race ahead in its matching and manage to + // create the obja/objs{} prerequisite. In which case we will + // end up failing the above check, which will be quite confusing + // and which is the reason we have added the "value came from + // group" info above). + // + // Implementing this verification in this racing environment is + // challanging, to put it mildly. So what we are going to do is, + // in case of a group, also enter the member (which we will be + // doing anyway shortly). If we were the ones who created it, + // then we have "staked out" our view of its variables and any + // subsequent attempts to enter it will trigger the above + // check. If, however, this target was already there, then we + // verify that it's consistent with the values we expect. + // + if (group) + { + const target_type& tt (mod ? tts.bmi : tts.obj); + + // @@ PERF: maybe we should stash the member in pd and avoid + // another search when we pick the member? (Though obj{} + // might also not be synthesized.) + + pair<target&, ulock> r ( + search_new_locked ( + ctx, tt, d, dir_path (), *cp.tk.name, nullptr, cp.scope)); + + if (r.second.owns_lock ()) + r.second.unlock (); + else + { + const target& cmt (r.first); + + if (!absent || + !operator>= (cmt.decl, target_decl::implied)) // VC14 + { + if (absent && cmt.vars.empty ()) + ; // Optimize for the common case. + else + { + check (c_poptions, cpt, cr_ops->c_p, cmt); + check (x_poptions, cpt, cr_ops->x_p, cmt); + check (c_coptions, cpt, cr_ops->c_c, cmt); + check (x_coptions, cpt, cr_ops->x_c, cmt); + } + } + } + } + } + } + + if (locked) + r.second.unlock (); + + // If we shouldn't clean obj{}, then it is fair to assume we + // shouldn't clean the source either (generated source will be in + // the same directory as obj{} and if not, well, go find yourself + // another build system ;-)). + // + // Note: should be done after setting/verifying variables since can + // be an operation batch. + // + if (skip (cpt)) + { + pt = nullptr; + continue; + } + + pt = &cpt; + pd = group ? 1 : 0; mk = mod ? 2 : 1; } else if (p.is_a<libx> () || @@ -1729,9 +2040,9 @@ namespace build2 } } - // Process prerequisites, pass 2: finish rule chaining but don't start - // matching anything yet since that may trigger recursive matching of - // bmi{} targets we haven't completed yet. Hairy, I know. + // Process prerequisites, pass 2: finish dependency synthesis but don't + // start matching anything yet since that may trigger recursive matching + // of bmi{} targets we haven't completed yet. Hairy, I know. // // Parallel prerequisites/prerequisite_targets loop. @@ -1759,14 +2070,14 @@ namespace build2 // Note that if this is a library not to be cleaned, we keep it // marked for completion (see the next phase). } - else if (mk == 1 || mk == 2) // Source/module chain. + else if (mk == 1 || mk == 2) // Synthesized source/module dependency. { bool mod (mk == 2); // p is_a x_mod mk = 1; const target& rt (*pt); - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + bool group (pd != 0); // Group's prerequisite (see pass 1). // If we have created a obj/bmi{} target group, pick one of its // members; the rest would be primarily concerned with it. @@ -1812,7 +2123,7 @@ namespace build2 } // Add our lib*{} (see the export.* machinery for details) and - // bmi*{} (both original and chained; see module search logic) + // bmi*{} (both original and synthesized; see module search logic) // prerequisites. // // Note that we don't resolve lib{} to liba{}/libs{} here @@ -1833,7 +2144,8 @@ namespace build2 size_t j (start); for (prerequisite_member p: group_prerequisite_members (a, t)) { - const target* pt (pts[j++]); + const target* pt (pts[j].target); + uintptr_t& pd (pts[j++].data); if (pt == nullptr) // Note: ad hoc is taken care of. continue; @@ -1847,7 +2159,8 @@ namespace build2 { ps.push_back (p.as_prerequisite ()); } - else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. + else if (x_mod != nullptr && p.is_a (*x_mod)) // Synthesized + // module dependency. { // Searched during pass 1 but can be NULL or marked. // @@ -1857,7 +2170,7 @@ namespace build2 // was a group, then we would have picked up a member. So // here we may have to "unpick" it. // - bool group (j < i && !p.prerequisite.belongs (t)); + bool group (j < i && pd != 0); unmark (pt); ps.push_back (prerequisite (group ? *pt->group : *pt)); @@ -1997,7 +2310,8 @@ namespace build2 mark (pt, mk); } - // Process prerequisites, pass 3: match everything and verify chains. + // Process prerequisites, pass 3: match everything and verify synthesized + // dependencies. // // Wait with unlocked phase to allow phase switching. @@ -2051,7 +2365,9 @@ namespace build2 i = start; for (prerequisite_member p: group_prerequisite_members (a, t)) { - const target*& pt (pts[i++]); + const target*& pt (pts[i].target); + uintptr_t& pd (pts[i++].data); + // Skipped or not marked for completion. // @@ -2088,7 +2404,7 @@ namespace build2 if (&tp != &tp1) { - bool group (!p.prerequisite.belongs (t)); + bool group (pd != 0); // See pass 1. const target_type& rtt (mod ? (group ? bmi::static_type : tts.bmi) @@ -2193,12 +2509,19 @@ namespace build2 // appear after the preceding static library of which this binless // library is a dependency. // + // Note that we omit the duplicate suppression if we are linking the + // whole archive since the previous instance may not necessarily do + // the same (see GH issue #411; we could have complicated things and + // stored the flag in appended_libraries but it doesn't feel + // worthwhile in this case). + // // From the process_libraries() semantics we know that this callback // is always called and always after the options callbacks. // - appended_library* al (l != nullptr - ? &d.ls.append (*l, d.args.size ()) - : d.ls.append (ns, d.args.size ())); + appended_library* al ( + f & lflag_whole ? nullptr : + l != nullptr ? &d.ls.append (*l, d.args.size ()) : + d.ls.append (ns, d.args.size ())); if (al != nullptr && al->end != appended_library::npos) // Closed. { @@ -2397,13 +2720,13 @@ namespace build2 else { d.args.push_back ("-Wl,--whole-archive"); - d.args.push_back (move (p)); + d.args.push_back (move (p)); p.clear (); d.args.push_back ("-Wl,--no-whole-archive"); - goto done; } } - d.args.push_back (move (p)); + if (!p.empty ()) + d.args.push_back (move (p)); } if (d.cs != nullptr) @@ -3199,10 +3522,6 @@ namespace build2 // args.push_back ("/NOLOGO"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // For utility libraries use thin archives if possible. // // LLVM's lib replacement had the /LLVMLIBTHIN option at least from @@ -3269,7 +3588,16 @@ namespace build2 { // Are we using the compiler or the linker (e.g., link.exe) directly? // - bool ldc (tsys != "win32-msvc"); + bool ldc; + + if (tsys == "win32-msvc") + { + args.push_back ("/NOLOGO"); + ldc = false; + } + else + ldc = true; + if (ldc) { @@ -3679,6 +4007,20 @@ namespace build2 if (lt.shared_library () && (tsys == "win32-msvc" || tsys == "mingw32")) reli = relative (find_adhoc_member<libi> (t)->path ()); + if (tsys == "win32-msvc") + { + // Add /MACHINE unless there is a custom value (/MACHINE:ARM64EC). + // + // Note that we don't bother hashing it since to change its value one + // would have to use a different MSVC toolchain (which means things + // would be rebuilt from scratch anyway). + // + if (!find_option_prefix ("/MACHINE:", args, true)) + { + args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); + } + } + const process_path* ld (nullptr); if (lt.static_library ()) { @@ -3702,15 +4044,10 @@ namespace build2 // Using link.exe directly. // ld = &cast<process_path> (rs["bin.ld.path"]); - args.push_back ("/NOLOGO"); if (ot == otype::s) args.push_back ("/DLL"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // Unless explicitly enabled with /INCREMENTAL, disable incremental // linking (it is implicitly enabled if /DEBUG is specified). The // reason is the .ilk file: its name cannot be changed and if we diff --git a/libbuild2/cc/module.cxx b/libbuild2/cc/module.cxx index cf6c6e4..a3c64d9 100644 --- a/libbuild2/cc/module.cxx +++ b/libbuild2/cc/module.cxx @@ -22,6 +22,12 @@ namespace build2 { namespace cc { + // cc.core_module + // + const string core_module::name ("cc.core.config"); + + // x.config_module + // void config_module:: guess (scope& rs, const location& loc, const variable_map&) { @@ -891,6 +897,9 @@ namespace build2 config::save_environment (rs, xi.platform_environment); } + // x module + // + // Global cache of ad hoc importable headers. // // The key is a hash of the system header search directories diff --git a/libbuild2/cc/module.hxx b/libbuild2/cc/module.hxx index 4213516..d4e9a67 100644 --- a/libbuild2/cc/module.hxx +++ b/libbuild2/cc/module.hxx @@ -14,6 +14,8 @@ #include <libbuild2/cc/common.hxx> +#include <libbuild2/cc/compiledb.hxx> + #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> #include <libbuild2/cc/install-rule.hxx> @@ -27,6 +29,35 @@ namespace build2 { struct compiler_info; + // cc.core module + // + class core_module: public build2::module + { + public: + static const string name; + + explicit + core_module (const core_module* om) + : outer_module_ (om) + { + } + + public: + const core_module* outer_module_; + + strings cdb_names_; + + const compiledb_name_filter* cdb_filter_ = nullptr; + const compiledb_type_filter* cdb_filter_input_ = nullptr; + const compiledb_type_filter* cdb_filter_output_ = nullptr; + + compiledb_name_filter cdb_filter_storage_; + compiledb_type_filter cdb_filter_input_storage_; + compiledb_type_filter cdb_filter_output_storage_; + }; + + // x.config module + // class LIBBUILD2_CC_SYMEXPORT config_module: public build2::module, public config_data { @@ -92,13 +123,36 @@ namespace build2 // struct header_key { - path file; + // We used to use path comparison/hash which are case-insensitive on + // Windows. While this sounds right on the surface, the catch is that + // our target names are always case-sensitive, even on Windows. So + // what can happen is that the same header spelled in different case + // ends up with distinct targets but trying to occupy the same cache + // entry. See GH issue #390 for details. + // + // The conceptually correct way to fix this would be to actualize the + // header name. But that would be prohibitively expensive, especially + // on Windows. Plus the header may be generated and thus not yet + // exist. + // + // On the other hand, we can already end up with different targets + // mapped to the same filesystem entry in other situations (h{} vs + // hxx{} is the canonical example). And we are ok with that provided + // they have noop recipes. So it feels like the simplest solution here + // is to go along by having case-sensitive cache entries. This means + // that auto-generated headers will have to be included using the same + // spelling, but that seems like a sensible restriction (doing + // otherwise won't be portable). + // + path file; size_t hash; friend bool operator== (const header_key& x, const header_key& y) { - return x.file == y.file; // Note: hash was already compared. + // Note: hash was already compared. + // + return x.file.string () == y.file.string (); } }; @@ -130,6 +184,8 @@ namespace build2 msvc_library_search_dirs (const compiler_info&, scope&) const; }; + // x module + // class LIBBUILD2_CC_SYMEXPORT module: public build2::module, public virtual common, public link_rule, @@ -139,7 +195,6 @@ namespace build2 public predefs_rule { public: - explicit module (data&& d, const scope& rs) : common (move (d)), link_rule (move (d)), diff --git a/libbuild2/cc/msvc.cxx b/libbuild2/cc/msvc.cxx index d21969c..416df36 100644 --- a/libbuild2/cc/msvc.cxx +++ b/libbuild2/cc/msvc.cxx @@ -33,10 +33,10 @@ namespace build2 const char* msvc_cpu (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "x86" : - cpu == "x86_64" ? "x64" : + const char* m (cpu == "x86_64" ? "x64" : + cpu == "i386" || cpu == "i686" ? "x86" : + cpu == "aarch64" ? "arm64" : cpu == "arm" ? "arm" : - cpu == "arm64" ? "arm64" : nullptr); if (m == nullptr) @@ -51,10 +51,10 @@ namespace build2 const char* msvc_machine (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : - cpu == "x86_64" ? "/MACHINE:x64" : + const char* m (cpu == "x86_64" ? "/MACHINE:x64" : + cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : + cpu == "aarch64" ? "/MACHINE:ARM64" : cpu == "arm" ? "/MACHINE:ARM" : - cpu == "arm64" ? "/MACHINE:ARM64" : nullptr); if (m == nullptr) diff --git a/libbuild2/cc/pkgconfig.cxx b/libbuild2/cc/pkgconfig.cxx index 7e47534..79a38ea 100644 --- a/libbuild2/cc/pkgconfig.cxx +++ b/libbuild2/cc/pkgconfig.cxx @@ -1,6 +1,8 @@ // file : libbuild2/cc/pkgconfig.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file +#include <libbuild2/cc/pkgconfig.hxx> + #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> #include <libbuild2/context.hxx> @@ -18,7 +20,6 @@ #include <libbuild2/cc/utility.hxx> #include <libbuild2/cc/common.hxx> -#include <libbuild2/cc/pkgconfig.hxx> #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> @@ -707,6 +708,7 @@ namespace build2 cmp ("user32") || cmp ("userenv") || cmp ("uuid") || + cmp ("uxtheme") || cmp ("version") || cmp ("windowscodecs") || cmp ("winhttp") || diff --git a/libbuild2/cc/windows-manifest.cxx b/libbuild2/cc/windows-manifest.cxx index 14f4a53..ff77327 100644 --- a/libbuild2/cc/windows-manifest.cxx +++ b/libbuild2/cc/windows-manifest.cxx @@ -23,8 +23,9 @@ namespace build2 const char* windows_manifest_arch (const string& tcpu) { - const char* pa (tcpu == "i386" || tcpu == "i686" ? "x86" : - tcpu == "x86_64" ? "amd64" : + const char* pa (tcpu == "x86_64" ? "amd64" : + tcpu == "i386" || tcpu == "i686" ? "x86" : + tcpu == "aarch64" ? "arm64" : nullptr); if (pa == nullptr) diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx index 2f134c4..776299c 100644 --- a/libbuild2/config/init.cxx +++ b/libbuild2/config/init.cxx @@ -27,6 +27,7 @@ namespace build2 namespace config { static const file_rule file_rule_ (true /* check_type */); + static const noop_rule noop_rule_ (true /* exclude_group */); void functions (function_map&); // functions.cxx @@ -38,7 +39,10 @@ namespace build2 // the entire values. // static pair<names_view, const char*> - save_environment (const value& d, const value* b, names& storage) + save_environment (const scope&, + const value& d, + const value* b, + names& storage) { if (b == nullptr) return make_pair (reverse (d, storage, true /* reduce */), "="); @@ -730,7 +734,7 @@ namespace build2 // This allows a custom configure rule while doing nothing by default. // - rs.insert_rule<target> (configure_id, 0, "config.noop", noop_rule::instance); + rs.insert_rule<target> (configure_id, 0, "config.noop", noop_rule_); // We need this rule for out-of-any-project dependencies (for example, // libraries imported from /usr/lib). We are registering it on the diff --git a/libbuild2/config/module.cxx b/libbuild2/config/module.cxx index 713d30c..faae865 100644 --- a/libbuild2/config/module.cxx +++ b/libbuild2/config/module.cxx @@ -14,7 +14,7 @@ namespace build2 bool module:: save_variable (const variable& var, optional<uint64_t> flags, - save_variable_function* save) + save_variable_function* func) { const string& n (var.name); @@ -45,15 +45,18 @@ namespace build2 return false; } - sv.push_back (saved_variable {var, flags, save}); + sv.push_back (saved_variable {var, flags, func}); return true; } void module:: - save_variable (scope& rs, const variable& var, optional<uint64_t> flags) + save_variable (scope& rs, + const variable& var, + optional<uint64_t> flags, + save_variable_function* func) { if (module* m = rs.find_module<module> (module::name)) - m->save_variable (var, flags); + m->save_variable (var, flags, func); } void module:: diff --git a/libbuild2/config/module.hxx b/libbuild2/config/module.hxx index 8d3ff67..77109ce 100644 --- a/libbuild2/config/module.hxx +++ b/libbuild2/config/module.hxx @@ -22,20 +22,12 @@ namespace build2 namespace config { // An ordered list of build system modules each with an ordered list of - // config.* variables and their "save flags" (see save_variable()) that - // are used (as opposed to just being specified) in this configuration. - // Populated by the config utility functions (required(), optional()) and - // saved in the order populated. If flags are absent, then this variable - // was marked as "unsaved" (always transient). + // config.* variables and their save flags/function (see save_variable()) + // that are used (as opposed to just being specified) in this + // configuration. Populated by the config utility functions (required(), + // optional()) and saved in the order populated. If flags are absent, then + // this variable was marked as "unsaved" (always transient). // - // The optional save function can be used to implement custom variable - // saving, for example, as a difference appended to the base value. The - // second half of the result is the assignment operator to use. - // - using save_variable_function = - pair<names_view, const char*> (const value&, - const value* base, - names& storage); struct saved_variable { reference_wrapper<const variable> var; @@ -151,7 +143,10 @@ namespace build2 save_variable_function* = nullptr); static void - save_variable (scope&, const variable&, optional<uint64_t>); + save_variable (scope&, + const variable&, + optional<uint64_t>, + save_variable_function*); bool save_module (const char* name, int prio = 0); diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx index b1716cf..6e7ef18 100644 --- a/libbuild2/config/operation.cxx +++ b/libbuild2/config/operation.cxx @@ -558,7 +558,7 @@ namespace build2 storage.clear (); pair<names_view, const char*> p ( sv.save != nullptr - ? sv.save (v, base, storage) + ? sv.save (rs, v, base, storage) : make_pair (reverse (v, storage, true /* reduce */), "=")); // Might becomes empty after a custom save function had at it. diff --git a/libbuild2/config/utility.cxx b/libbuild2/config/utility.cxx index 6574367..aa0d5af 100644 --- a/libbuild2/config/utility.cxx +++ b/libbuild2/config/utility.cxx @@ -7,7 +7,14 @@ using namespace std; namespace build2 { - void (*config_save_variable) (scope&, const variable&, optional<uint64_t>); + void + (*config_save_variable) (scope&, + const variable&, + optional<uint64_t>, + pair<names_view, const char*> (*)(const scope&, + const value&, + const value*, + names&)); void (*config_save_environment) (scope&, const char*); void (*config_save_module) (scope&, const char*, int); const string& (*config_preprocess_create) (context&, @@ -21,7 +28,10 @@ namespace build2 namespace config { pair<lookup, bool> - lookup_config_impl (scope& rs, const variable& var, uint64_t sflags) + lookup_config_impl (scope& rs, + const variable& var, + uint64_t sflags, + save_variable_function* sfunc) { // This is a stripped-down version of the default value case. @@ -71,7 +81,7 @@ namespace build2 } if (l.defined ()) - save_variable (rs, var, sflags); + save_variable (rs, var, sflags, sfunc); return pair<lookup, bool> (l, n); } diff --git a/libbuild2/config/utility.hxx b/libbuild2/config/utility.hxx index 1e2ff53..3b67c6c 100644 --- a/libbuild2/config/utility.hxx +++ b/libbuild2/config/utility.hxx @@ -29,13 +29,22 @@ namespace build2 // disfigure hooks (for example, for second-level configuration). These are // accessed through the config module entry points (which are NULL for // transient configurations). Note also that the exact interpretation of the - // save flags and module order depends on the config module implementation - // (which may ignore them as not applicable). An implementation may also - // define custom save flags (for example, accessible through the config.save - // attribute). Such flags should start from 0x100000000. + // save flags/function and module order depends on the config module + // implementation (which may ignore them as not applicable). An + // implementation may also define custom save flags (for example, accessible + // through the config.save attribute). Such flags should start from + // 0x100000000. + // + // See below for the save function (last argument) semantics. // LIBBUILD2_SYMEXPORT extern void - (*config_save_variable) (scope&, const variable&, optional<uint64_t>); + (*config_save_variable) (scope&, + const variable&, + optional<uint64_t>, + pair<names_view, const char*> (*)(const scope&, + const value&, + const value*, + names&)); LIBBUILD2_SYMEXPORT extern void (*config_save_environment) (scope&, const char*); @@ -75,11 +84,27 @@ namespace build2 const uint64_t save_false_omitted = 0x08; // Treat false as undefined. const uint64_t save_base = 0x10; // Custom save with base. + // The optional save function can be used to implement custom variable + // saving, for example, as a difference appended to the base value or to + // complete relative paths (if abs_dir_path does not fit). The base + // argument is the value of this variable from the outer scope (if any) + // and is only calculated if the save_base flag is specified. The second + // half of the result is the assignment operator to use. + // + using save_variable_function = + pair<names_view, const char*> (const scope& rs, + const value&, + const value* base, + names& storage); + inline void - save_variable (scope& rs, const variable& var, uint64_t flags = 0) + save_variable (scope& rs, + const variable& var, + uint64_t flags = 0, + save_variable_function* func = nullptr) { if (config_save_variable != nullptr) - config_save_variable (rs, var, flags); + config_save_variable (rs, var, flags, func); } // Mark a variable as "unsaved" (always transient). @@ -91,7 +116,7 @@ namespace build2 unsave_variable (scope& rs, const variable& var) { if (config_save_variable != nullptr) - config_save_variable (rs, var, nullopt); + config_save_variable (rs, var, nullopt, nullptr); } // Mark an environment variable to be saved during hermetic configuration. @@ -256,35 +281,40 @@ namespace build2 lookup lookup_config (scope& rs, const variable&, - uint64_t save_flags = 0); + uint64_t save_flags = 0, + save_variable_function* = nullptr); lookup lookup_config (bool& new_value, scope& rs, const variable&, - uint64_t save_flags = 0); + uint64_t save_flags = 0, + save_variable_function* = nullptr); // Note that the variable is expected to have already been entered. // inline lookup lookup_config (scope& rs, const string& var, - uint64_t save_flags = 0) + uint64_t save_flags = 0, + save_variable_function* func = nullptr) { // Note: go straight for the public variable pool. // - return lookup_config (rs, rs.ctx.var_pool[var], save_flags); + return lookup_config (rs, rs.ctx.var_pool[var], save_flags, func); } inline lookup lookup_config (bool& new_value, scope& rs, const string& var, - uint64_t save_flags = 0) + uint64_t save_flags = 0, + save_variable_function* func = nullptr) { // Note: go straight for the public variable pool. // - return lookup_config (new_value, rs, rs.ctx.var_pool[var], save_flags); + return lookup_config ( + new_value, rs, rs.ctx.var_pool[var], save_flags, func); } // Lookup a config.* variable value and, if the value is undefined, set it diff --git a/libbuild2/config/utility.ixx b/libbuild2/config/utility.ixx index d8348bd..87f628c 100644 --- a/libbuild2/config/utility.ixx +++ b/libbuild2/config/utility.ixx @@ -6,25 +6,32 @@ namespace build2 namespace config { LIBBUILD2_SYMEXPORT pair<lookup, bool> - lookup_config_impl (scope&, const variable&, uint64_t); + lookup_config_impl (scope&, + const variable&, + uint64_t, + save_variable_function*); template <typename T> pair<lookup, bool> lookup_config_impl (scope&, const variable&, T&&, uint64_t, bool); inline lookup - lookup_config (scope& rs, const variable& var, uint64_t sflags) + lookup_config (scope& rs, + const variable& var, + uint64_t sflags, + save_variable_function* func) { - return lookup_config_impl (rs, var, sflags).first; + return lookup_config_impl (rs, var, sflags, func).first; } inline lookup lookup_config (bool& new_value, scope& rs, const variable& var, - uint64_t sflags) + uint64_t sflags, + save_variable_function* func) { - auto r (lookup_config_impl (rs, var, sflags)); + auto r (lookup_config_impl (rs, var, sflags, func)); new_value = new_value || r.second; return r.first; } diff --git a/libbuild2/context.hxx b/libbuild2/context.hxx index 33fc892..81ac970 100644 --- a/libbuild2/context.hxx +++ b/libbuild2/context.hxx @@ -178,34 +178,35 @@ namespace build2 // match - search prerequisites and match rules // execute - execute the matched rule // - // The build system starts with a "serial load" phase and then continues - // with parallel match and execute. Match, however, can be interrupted - // both with load and execute. + // The build system starts with a serial "initial load" phase and then + // continues with parallel match and execute. Match, however, can be + // interrupted both with load and execute. // - // Match can be interrupted with "exclusive load" in order to load - // additional buildfiles. Similarly, it can be interrupted with (parallel) - // execute in order to build targetd required to complete the match (for - // example, generated source code or source code generators themselves). + // Match can be interrupted with a (serial) "interrupting load" in order + // to load additional buildfiles. Similarly, it can be interrupted with + // (parallel) execute in order to build targetd required to complete the + // match (for example, generated source code or source code generators + // themselves). // // Such interruptions are performed by phase change that is protected by // phase_mutex (which is also used to synchronize the state changes // between phases). // - // Serial load can perform arbitrary changes to the build state. Exclusive - // load, however, can only perform "island appends". That is, it can - // create new "nodes" (variables, scopes, etc) but not (semantically) - // change already existing nodes or invalidate any references to such (the - // idea here is that one should be able to load additional buildfiles as - // long as they don't interfere with the existing build state). The - // "islands" are identified by the load_generation number (1 for the - // initial/serial load). It is incremented in case of a phase switch and - // can be stored in various "nodes" to verify modifications are only done - // "within the islands". Another example of invalidation would be - // insertion of a new scope "under" an existing target thus changing its - // scope hierarchy (and potentially even its base scope). This would be - // bad because we may have made decisions based on the original hierarchy, - // for example, we may have queried a variable which in the new hierarchy - // would "see" a new value from the newly inserted scope. + // Initial load can perform arbitrary changes to the build state. + // Interrupting load, however, can only perform what we call "island + // appends". That is, it can create new "nodes" (variables, scopes, etc) + // but not (semantically) change already existing nodes or invalidate any + // references to such (the idea here is that one should be able to load + // additional buildfiles as long as they don't interfere with the existing + // build state). The "islands" are identified by the load_generation + // number (1 for the initial load). It is incremented in case of a phase + // switch and can be stored in various "nodes" to verify modifications are + // only done "within the islands". Another example of invalidation would + // be insertion of a new scope "under" an existing target thus changing + // its scope hierarchy (and potentially even its base scope). This would + // be bad because we may have made decisions based on the original + // hierarchy, for example, we may have queried a variable which in the new + // hierarchy would "see" a new value from the newly inserted scope. // // The special load_generation value 0 indicates initialization before // anything has been loaded. Currently, it is changed to 1 at the end @@ -351,10 +352,52 @@ namespace build2 (current_mname.empty () && current_oname == mo)); }; + // Operation callbacks. + // + // An entity (module, core) can register a function that will be called + // when an action is executed on a set of targets. The pre callback is + // called before any recipes for the action are matched and the post -- + // after all have been executed. The post callback is called even if + // execution has failed. + // + // The callback should only be registered during the load phase. Note + // that it's registered for the inner action, meaning that it will be + // called for any outer action (which is discernible from the first + // argument of the callback). Note also that meta-operations other than + // perform never actually execute any recipes and it probably only makes + // sense to register these callbacks for the perform_* actions. + // + // Note that the callbacks will also be called when building a build + // system module or an ad hoc C++ recipe. See create_module_context() for + // details. + // + // Note also that if the callbacks are registered from a module load + // function, then there are nuances with interrupted load phases. See the + // compilation database handling in the cc module for details. + // + // See also scope::operation_callback. + // + struct operation_callback + { + using pre_callback = + void (context&, action, const action_targets&); + + using post_callback = + void (context&, action, const action_targets&, bool failed); + + function<pre_callback> pre; + function<post_callback> post; + }; + + using operation_callback_map = multimap<action_id, operation_callback>; + + operation_callback_map operation_callbacks; + // Meta/operation-specific context-global auxiliary data storage. // - // Note: cleared by current_[meta_]operation() below. Normally set by - // meta/operation-specific callbacks from [mate_]operation_info. + // Normally set by meta/operation-specific callbacks from + // [mata_]operation_info. The operation data is cleared by + // current_operation() below. // // Note also: watch out for MT-safety in the data itself. // @@ -759,6 +802,9 @@ namespace build2 // Set current meta-operation and operation. // + // Note that the context instance is not to be re-used between different + // meta-operations. + // void current_meta_operation (const meta_operation_info&); @@ -904,7 +950,7 @@ namespace build2 bool unlock_phase = false); void - wait (); + wait (bool work_queue = true); // Note: move-assignable to empty only. // diff --git a/libbuild2/context.ixx b/libbuild2/context.ixx index 6c8c428..2c62a45 100644 --- a/libbuild2/context.ixx +++ b/libbuild2/context.ixx @@ -26,8 +26,11 @@ namespace build2 inline wait_guard:: ~wait_guard () noexcept (false) { + // Don't work our own queue since we are most likely in stack unwinding + // causes by an exception. + // if (task_count != nullptr) - wait (); + wait (false); } inline wait_guard:: @@ -54,10 +57,15 @@ namespace build2 } inline void wait_guard:: - wait () + wait (bool wq) { phase_unlock u (phase ? ctx : nullptr, true /* delay */); - ctx->sched->wait (start_count, *task_count, u); + ctx->sched->wait (start_count, + *task_count, + u, + (wq + ? scheduler::work_queue::work_all + : scheduler::work_queue::work_none)); task_count = nullptr; } } diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index dd1c87b..5b00980 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -155,8 +155,12 @@ namespace build2 for (const dir_entry& e: dir_iterator (d, dir_iterator::no_follow)) { const path& n (e.path ()); + const string& s (n.string ()); - if (!n.empty () && n.string ().front () != '.') + if (s.compare (0, 4, ".git") != 0 && + s != ".bdep" && + s != ".bpkg" && + s != ".build2") try { if (e.type () == entry_type::directory) // Can throw. @@ -278,120 +282,134 @@ namespace build2 const location loc (pn); // Dummy location. action_targets ts {tgt}; - auto process_postponed = [&ctx, &mod] () { - if (!mod.postponed.list.empty ()) + auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); + ctx.match_only = match_only_level::all; + + auto process_postponed = [&ctx, &mod, &ts] (action a) { - // Re-grab the phase lock similar to perform_match(). - // - phase_lock l (ctx, run_phase::match); + if (!mod.postponed.list.empty ()) + { + auto eg ( + make_exception_guard ( + [&ctx, a, &ts] () + { + perform_post_operation_callbacks ( + ctx, a, ts, true /* failed */); + })); - // Note that we don't need to bother with the mutex since we do - // all of this serially. But we can end up with new elements at - // the end. - // - // Strictly speaking, to handle this correctly we would need to do - // multiple passes over this list and only give up when we cannot - // make any progress since earlier entries that we cannot resolve - // could be "fixed" by later entries. But this feels far-fetched - // and so let's wait for a real example before complicating this. - // - for (auto i (mod.postponed.list.begin ()); - i != mod.postponed.list.end (); - ++i) - rule::match_postponed (*i); - } - }; + // Re-grab the phase lock similar to perform_match(). + // + phase_lock l (ctx, run_phase::match); - auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); - ctx.match_only = match_only_level::all; + // Note that we don't need to bother with the mutex since we do + // all of this serially. But we can end up with new elements at + // the end. + // + // Strictly speaking, to handle this correctly we would need to + // do multiple passes over this list and only give up when we + // cannot make any progress since earlier entries that we cannot + // resolve could be "fixed" by later entries. But this feels + // far-fetched and so let's wait for a real example before + // complicating this. + // + for (auto i (mod.postponed.list.begin ()); + i != mod.postponed.list.end (); + ++i) + rule::match_postponed (*i); + } + }; - const operations& ops (rs.root_extra->operations); - for (operations::size_type id (default_id + 1); // Skip default_id. - id < ops.size (); - ++id) - { - if (const operation_info* oif = ops[id]) + const operations& ops (rs.root_extra->operations); + for (operations::size_type id (default_id + 1); // Skip default_id. + id < ops.size (); + ++id) { - // Skip aliases (e.g., update-for-install). In fact, one can argue - // the default update should be sufficient since it is assumed to - // update all prerequisites and we no longer support ad hoc stuff - // like test.input. Though here we are using the dist - // meta-operation, not perform. - // - if (oif->id != id) - continue; - - // Use standard (perform) match. - // - if (auto pp = oif->pre_operation) + if (const operation_info* oif = ops[id]) { - if (operation_id pid = pp (ctx, {}, dist_id, loc)) + // Skip aliases (e.g., update-for-install). In fact, one can + // argue the default update should be sufficient since it is + // assumed to update all prerequisites and we no longer support + // ad hoc stuff like test.input. Though here we are using the + // dist meta-operation, not perform. + // + if (oif->id != id) + continue; + + // Use standard (perform) match. + // + if (auto pp = oif->pre_operation) { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); + if (operation_id pid = pp (ctx, {}, dist_id, loc)) + { + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, false /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); - if (poif->operation_pre != nullptr) - poif->operation_pre (ctx, {}, true /* inner */, loc); + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, poif->id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (poif->operation_post != nullptr) - poif->operation_post (ctx, {}, true /* inner */); + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, false /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); + } } - } - ctx.current_operation (*oif, nullptr, false /* diag_noise */); + ctx.current_operation (*oif, nullptr, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, true /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, true /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, true /* inner */); - if (auto po = oif->post_operation) - { - if (operation_id pid = po (ctx, {}, dist_id)) + if (auto po = oif->post_operation) { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); + if (operation_id pid = po (ctx, {}, dist_id)) + { + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, false /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); - if (poif->operation_pre != nullptr) - poif->operation_pre (ctx, {}, true /* inner */, loc); + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, poif->id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (poif->operation_post != nullptr) - poif->operation_post (ctx, {}, true /* inner */); + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, false /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); + } } } } @@ -643,8 +661,8 @@ namespace build2 { l5 ([&]{trace << "bootstrap dist " << rs;}); - // Recursively enter/collect file targets in src_root ignoring those - // that start with a dot. + // Recursively enter/collect file targets in src_root ignoring the + // following ones: .git*, .bdep, .bpkg, and .build2. // // Note that, in particular, we also collect the symlinks which point // outside src_root (think of third-party project packaging with the diff --git a/libbuild2/dump.cxx b/libbuild2/dump.cxx index 9b7f5b1..9fcfca8 100644 --- a/libbuild2/dump.cxx +++ b/libbuild2/dump.cxx @@ -242,7 +242,8 @@ namespace build2 h_pair = true; } else if (t.is_a<map<optional<string>, string>> () || - t.is_a<vector<pair<optional<string>, string>>> ()) + t.is_a<vector<pair<optional<string>, string>>> () || + t.is_a<vector<pair<optional<string>, bool>>> ()) { h_array = true; h_pair = false; diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx index ecb5f80..f834d8c 100644 --- a/libbuild2/file.cxx +++ b/libbuild2/file.cxx @@ -2200,8 +2200,8 @@ namespace build2 const project_name& pn (project (iroot)); if (pn.empty ()) - fail (loc) << "project-local importation of target " << tgt - << " from an unnamed project"; + fail (loc) << "project-local importation of target '" << tgt + << "' from an unnamed project"; tgt.proj = pn; // Reduce to normal import. @@ -2219,7 +2219,7 @@ namespace build2 // maybe project-less does not make sense. // if (tgt.absolute ()) - fail (loc) << "absolute directory in imported target " << tgt; + fail (loc) << "absolute directory in imported target '" << tgt << "'"; // Get the project name and convert the target to unqualified. // @@ -2584,7 +2584,7 @@ namespace build2 src = p.second; if (out_root.empty ()) - fail (loc) << "no project for imported target " << tgt; + fail (loc) << "no project for imported target '" << tgt << "'"; } if (src) @@ -2855,7 +2855,8 @@ namespace build2 auto df = make_diag_frame ( [&tgt, &loc] (const diag_record& dr) { - dr << info (loc) << "while loading export stub for " << tgt; + dr << info (loc) << "while loading export stub for imported " + << "target '" << tgt << "'"; }); parser p (ctx); @@ -2866,7 +2867,7 @@ namespace build2 // assume the target is not exported. // if (v.empty () && !tgt.empty ()) - fail (loc) << "target " << tgt << " is not exported by project " + fail (loc) << "target '" << tgt << "' is not exported by project " << *proj; pair<names, const scope&> r (move (v), *root); @@ -2880,7 +2881,9 @@ namespace build2 } catch (const io_error& e) { - fail (loc) << "unable to read buildfile " << es << ": " << e << endf; + fail (loc) << "unable to read buildfile " << es << ": " << e << + info << "while loading export stub for imported target '" + << tgt << "'" << endf; } } else @@ -2903,7 +2906,7 @@ namespace build2 altn); if (!bf) fail << "no buildfile in " << src_base << " or parent directories " - << "for imported target " << tgt; + << "for imported target '" << tgt << "'"; if (!bf->empty ()) src_base = bf->directory (); @@ -2982,7 +2985,7 @@ namespace build2 // Validate the name. // if (tgt.qualified () && tgt.empty ()) - fail (loc) << "project-qualified empty name " << tgt; + fail (loc) << "importing project-qualified empty name '" << tgt << "'"; // If metadata is requested, delegate to import_direct() which will lookup // the target and verify the metadata was loaded. diff --git a/libbuild2/install/operation.cxx b/libbuild2/install/operation.cxx index ce5d24a..029a5f6 100644 --- a/libbuild2/install/operation.cxx +++ b/libbuild2/install/operation.cxx @@ -369,7 +369,10 @@ namespace build2 // files, there is not going to be much speedup from doing it in parallel. // There is also now the installation manifest, which relies on us // installing all the filesystem entries of a target serially. - + // + // Additionally, we stop on first error since there is no sense in + // continuing. + // const operation_info op_install { install_id, 0, @@ -379,7 +382,8 @@ namespace build2 "installed", "has nothing to install", // We cannot "be installed". execution_mode::first, - 0 /* concurrency */, // Run serially. + 0 /* concurrency */, // Run serially. + false /* keep_going */, // Stop on first error. &pre_install, nullptr, &install_pre, @@ -406,7 +410,8 @@ namespace build2 "uninstalled", "is not installed", execution_mode::last, - 0 /* concurrency */, // Run serially + 0 /* concurrency */, // Run serially. + false /* keep_going */, // Stop on first error. &pre_uninstall, nullptr, nullptr, @@ -427,6 +432,7 @@ namespace build2 op_update.name_done, op_update.mode, op_update.concurrency, + op_update.keep_going, op_update.pre_operation, op_update.post_operation, op_update.operation_pre, diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx index 1aaa38d..36a7ce5 100644 --- a/libbuild2/module.cxx +++ b/libbuild2/module.cxx @@ -96,18 +96,28 @@ namespace build2 nullopt)); /* module_context */ // We use the same context for building any nested modules that might be - // required while building modules. + // required while building modules. Note: this is also used to detect + // module building context. @@ Maybe we should invent special build.mode? // context& mctx (*(ctx.module_context = ctx.module_context_storage->get ())); mctx.module_context = &mctx; + // Copy over any operation callbacks. If a callback implementation does + // not wish to see module context's calls, it can filter them out based on + // the passed context. + // + // Note also that only the callbacks registered before we need to build + // the first module will be in effect. Probably good enough for now. + // + mctx.operation_callbacks = ctx.operation_callbacks; + // Setup the context to perform update. In a sense we have a long-running // perform meta-operation batch (indefinite, in fact, since we never call // the meta-operation's *_post() callbacks) in which we periodically // execute update operations. // // Note that we perform each build in a separate update operation. Failed - // that, if the same target is update twice (which may happen with ad hoc + // that, if the same target is updated twice (which may happen with ad hoc // recipes) we will see the old state. // if (mo_perform.meta_operation_pre != nullptr) @@ -470,7 +480,7 @@ namespace build2 // if (nested) { - // This could be initial or exclusive load. + // This could be initial or interrupting load. // // @@ TODO: see the ad hoc recipe case as a reference. // diff --git a/libbuild2/name.hxx b/libbuild2/name.hxx index f5cb2c5..c6aac45 100644 --- a/libbuild2/name.hxx +++ b/libbuild2/name.hxx @@ -136,7 +136,7 @@ namespace build2 // value to dir. Throw invalid_argument if value would become empty. May // also throw invalid_path. // - void + LIBBUILD2_SYMEXPORT void canonicalize (); }; diff --git a/libbuild2/operation.cxx b/libbuild2/operation.cxx index 6f88e38..76118ea 100644 --- a/libbuild2/operation.cxx +++ b/libbuild2/operation.cxx @@ -271,9 +271,12 @@ namespace build2 struct monitor_data { size_t incr; - string what; - atomic<timestamp::rep> time {timestamp_nonexistent_rep}; + string what1; + string what2; + size_t exec = 0; // Number of targets executed during match. + timestamp time = timestamp_nonexistent; } md; // Note: must outlive monitor_guard. + scheduler::monitor_guard mg; if (prog && show_progress (2 /* max_verb */)) @@ -282,39 +285,66 @@ namespace build2 // the up-to-date check on some projects (e.g., Boost). So we jump // through a few hoops to make sure we don't overindulge. // + // Note also that the higher the increment, the less accurate our + // executed during match number will be. + // md.incr = stderr_term // Scale depending on output type. - ? (ctx.sched->serial () ? 1 : 5) + ? (ctx.sched->serial () ? 1 : 2) : 100; - md.what = " targets to " + diag_do (ctx, a); + md.what1 = " targets to " + diag_do (ctx, a); + md.what2 = ' ' + diag_did (ctx, a) + " during match)"; mg = ctx.sched->monitor ( ctx.target_count, md.incr, - [&md] (size_t c) -> size_t + [&md, &ctx] (size_t p, size_t c) -> size_t { - size_t r (c + md.incr); + if (p > c) + md.exec += p - c; if (stderr_term) { - timestamp o (duration (md.time.load (memory_order_consume))); timestamp n (system_clock::now ()); - if (n - o < chrono::milliseconds (80)) - return r; + if (n - md.time < chrono::milliseconds (80)) + return md.incr; - md.time.store (n.time_since_epoch ().count (), - memory_order_release); + md.time = n; } diag_progress_lock pl; diag_progress = ' '; diag_progress += to_string (c); - diag_progress += md.what; + diag_progress += md.what1; + + if (md.exec != 0) + { + // Offset by the number of targets skipped. + // + size_t s (ctx.skip_count.load (memory_order_relaxed)); + + if (md.exec > s) + { + diag_progress += " ("; + diag_progress += to_string (md.exec - s); + diag_progress += md.what2; + } + } - return r; + return md.incr; }); } + // Call the pre operation callbacks. + // + // See a comment in perform_execute() for why we are doing it here + // (short answer: phase switches). + // + auto cs (ctx.operation_callbacks.equal_range (a)); + for (auto i (cs.first); i != cs.second; ++i) + if (const auto& f = i->second.pre) + f (ctx, a, ts); + // Start asynchronous matching of prerequisites keeping track of how // many we have started. Wait with unlocked phase to allow phase // switching. @@ -417,7 +447,11 @@ namespace build2 diag_progress.clear (); } - // We are now running serially. Re-examine targets that we have matched. + // We are now running serially. + // + + // Re-examine targets that we have matched and determine whether we have + // failed. // for (size_t j (0); j != n; ++j) { @@ -443,11 +477,8 @@ namespace build2 case target_state::postponed: { // We bailed before matching it (leave state in action_target as - // unknown). + // unknown for the structured result printing). // - if (verb != 0 && diag >= 1) - info << "not " << diag_did (a, t); - break; } case target_state::unknown: @@ -460,9 +491,6 @@ namespace build2 { // Things didn't go well for this target. // - if (verb != 0 && diag >= 1) - info << "failed to " << diag_do (a, t); - at.state = s; fail = true; break; @@ -472,6 +500,36 @@ namespace build2 } } + // Call the post operation callbacks if perform_execute() won't be + // called. + // + if (fail) + perform_post_operation_callbacks (ctx, a, ts, fail); + + // Re-examine targets that we have matched and print diagnostics. + // + if (verb != 0 && diag >= 1) + { + for (size_t j (0); j != n; ++j) + { + action_target& at (ts[j]); + const target& t (at.as<target> ()); + + if (at.state == target_state::failed) + { + // Things didn't go well for this target. + // + info << "failed to " << diag_do (a, t); + } + else if (j >= i || t.matched_state (a) == target_state::postponed) + { + // We bailed before matching it. + // + info << "not " << diag_did (a, t); + } + } + } + if (fail) throw failed (); @@ -602,40 +660,55 @@ namespace build2 switch (ctx.current_inner_oif->concurrency) { case 0: sched_tune = tune_guard (*ctx.sched, 1); break; // Run serially. - case 1: break; // Run as is. - default: assert (false); // Not supported. + case 1: break; // Run as is. + default: assert (false); // Not supported. } + // Override the keep_going flag if requested by the operation. + // + auto kgg = make_guard ([&ctx, o = ctx.keep_going] () + { + ctx.keep_going = o; + }); + if (!ctx.current_inner_oif->keep_going) + ctx.keep_going = false; + // Set the dry-run flag. // ctx.dry_run = ctx.dry_run_option; // Setup progress reporting if requested. // - string what; // Note: must outlive monitor_guard. + struct monitor_data + { + size_t init; + size_t incr; + string what; + } md; // Note: must outlive monitor_guard. + scheduler::monitor_guard mg; if (prog && show_progress (1 /* max_verb */)) { - size_t init (ctx.target_count.load (memory_order_relaxed)); - size_t incr (init > 100 ? init / 100 : 1); // 1%. + md.init = ctx.target_count.load (memory_order_relaxed); + md.incr = md.init > 100 ? md.init / 100 : 1; // 1%. - if (init != incr) + if (md.init != md.incr) { - what = "% of targets " + diag_did (ctx, a); + md.what = "% of targets " + diag_did (ctx, a); mg = ctx.sched->monitor ( ctx.target_count, - init - incr, - [init, incr, &what, &ctx] (size_t c) -> size_t + md.incr, + [&md, &ctx] (size_t, size_t c) -> size_t { - size_t p ((init - c) * 100 / init); + size_t p ((md.init - c) * 100 / md.init); size_t s (ctx.skip_count.load (memory_order_relaxed)); diag_progress_lock pl; diag_progress = ' '; diag_progress += to_string (p); - diag_progress += what; + diag_progress += md.what; if (s != 0) { @@ -644,11 +717,18 @@ namespace build2 diag_progress += " skipped)"; } - return c - incr; + return md.incr; }); } } + // Note that while this would seem like the natural place to call the + // pre operation callbacks, it is actually too late since during match + // we may switch to the execute phase and execute some recipes (think + // building a tool to generate some code). So we have to do this in + // perform_match() and then carefully make sure the post callbacks are + // called for all the exit paths (match failed, match_only, etc). + // In the 'last' execution mode run post hoc first. // if (ctx.current_mode == execution_mode::last) @@ -697,9 +777,44 @@ namespace build2 // We are now running serially. // - // Clear the dry-run flag. + // Re-examine all the targets and determine whether we have failed. // - ctx.dry_run = false; + for (action_target& at: ts) + { + const target& t (at.as<target> ()); + + // Similar to match we cannot attribute post hoc failures to specific + // targets so it seems the best we can do is just fail them all. + // + if (!posthoc_fail) + { + // Note that here we call executed_state() directly instead of + // execute_complete() since we know there is no need to wait. + // + at.state = t.executed_state (a, false /* fail */); + } + else + at.state = /*t.state[a].state =*/ target_state::failed; + + switch (at.state) + { + case target_state::unknown: + case target_state::unchanged: + case target_state::changed: + break; + case target_state::failed: + { + fail = true; + break; + } + default: + assert (false); + } + } + + // Call the post operation callbacks. + // + perform_post_operation_callbacks (ctx, a, ts, fail); // Clear the progress if present. // @@ -709,7 +824,11 @@ namespace build2 diag_progress.clear (); } - // Restore original scheduler settings. + // Clear the dry-run flag. + // + ctx.dry_run = false; + + // Restore original scheduler and keep_going settings. } // Print skip count if not zero. Note that we print it regardless of the @@ -733,19 +852,6 @@ namespace build2 { const target& t (at.as<target> ()); - // Similar to match we cannot attribute post hoc failures to specific - // targets so it seems the best we can do is just fail them all. - // - if (!posthoc_fail) - { - // Note that here we call executed_state() directly instead of - // execute_complete() since we know there is no need to wait. - // - at.state = t.executed_state (a, false /* fail */); - } - else - at.state = /*t.state[a].state =*/ target_state::failed; - switch (at.state) { case target_state::unknown: @@ -780,7 +886,6 @@ namespace build2 if (verb != 0 && diag >= 1) info << "failed to " << diag_do (a, t); - fail = true; break; } default: @@ -978,6 +1083,19 @@ namespace build2 #endif } + void + perform_post_operation_callbacks (context& ctx, + action a, + const action_targets& ts, + bool failed) + { + auto cs (ctx.operation_callbacks.equal_range (a)); + + for (auto i (cs.first); i != cs.second; ++i) + if (const auto& f = i->second.post) + f (ctx, a, ts, failed); + } + const meta_operation_info mo_perform { perform_id, "perform", @@ -1391,7 +1509,8 @@ namespace build2 "", "", execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, @@ -1419,7 +1538,8 @@ namespace build2 "updated", "is up to date", execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, @@ -1437,7 +1557,8 @@ namespace build2 "cleaned", "is clean", execution_mode::last, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, diff --git a/libbuild2/operation.hxx b/libbuild2/operation.hxx index e8ff38a..4ca1305 100644 --- a/libbuild2/operation.hxx +++ b/libbuild2/operation.hxx @@ -177,9 +177,19 @@ namespace build2 // diagnostics (unless quiet). // LIBBUILD2_SYMEXPORT void - perform_execute (const values&, action, const action_targets&, + perform_execute (const values&, action, action_targets&, uint16_t diag, bool prog); + // Call the context-wide post operation callbacks. Should be called after + // perfrom_match() if perform_execute() will not be called. Note that + // perform_match() handles its own failures but not the match_only case. + // + LIBBUILD2_SYMEXPORT void + perform_post_operation_callbacks (context&, + action, + const action_targets&, + bool failed); + LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_noop; LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_perform; LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_info; @@ -225,6 +235,12 @@ namespace build2 // const size_t concurrency; + // Whether to keep going in case of errors for this operation. If the + // value is false, then the context's keep_going flag is overridden for + // the duration of the operation. + // + const bool keep_going; + // The values argument in the callbacks is the operation parameters. If // the operation expects parameters, then it should have a non-NULL // operation_pre() callback. Failed that, any parameters will be diagnosed diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index 8af5804..53f808c 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -1757,11 +1757,18 @@ namespace build2 } else { - // Note that p cannot point to the last character since then it - // would have been a directory, not a simple name. - // - d = dir_path (ns[0].value, 0, p + 1); - ns[0].value.erase (0, p + 1); + try + { + // Note that p cannot point to the last character since then it + // would have been a directory, not a simple name. + // + d = dir_path (ns[0].value, 0, p + 1); + ns[0].value.erase (0, p + 1); + } + catch (const invalid_path& e) + { + fail (nloc) << "invalid scope path '" << e.path << "'"; + } } } @@ -5600,7 +5607,7 @@ namespace build2 } d {var, val_attrs, line, block, lhs, is}; - function<void (value&&, bool first)> iteration = + function<bool (value&&, bool first)> iteration = [this, &d] (value&& v, bool first) { // Rewind the stream. @@ -5638,12 +5645,15 @@ namespace build2 << "instead of " << t; lexer_ = ol; + return true; }; if (!iterate) { for (auto b (ns->begin ()), i (b), e (ns->end ()); i != e; ++i) { + bool first (i == b); + // Set the variable value. // bool pair (i->pair); @@ -5655,7 +5665,7 @@ namespace build2 if (etype != nullptr) typify (v, *etype, &var); - iteration (move (v), i == b); + iteration (move (v), first); } } else @@ -8802,15 +8812,22 @@ namespace build2 (p = path_traits::rfind_separator (ns[0].value)) != string::npos) { - // Note that p cannot point to the last character since - // then it would have been a directory, not a simple name. - // - string& s (ns[0].value); - - name = string (s, p + 1); - s.resize (p + 1); - qual.push_back (name_type (dir_path (move (s)))); - qual.back ().pair = '/'; + try + { + // Note that p cannot point to the last character since + // then it would have been a directory, not a simple name. + // + string& s (ns[0].value); + + name = string (s, p + 1); + s.resize (p + 1); + qual.push_back (name_type (dir_path (move (s)))); + qual.back ().pair = '/'; + } + catch (const invalid_path& e) + { + fail (loc) << "invalid scope path '" << e.path << "'"; + } } else name = move (ns[n - 1].value); diff --git a/libbuild2/rule.cxx b/libbuild2/rule.cxx index dc1c96c..b504dc7 100644 --- a/libbuild2/rule.cxx +++ b/libbuild2/rule.cxx @@ -430,9 +430,9 @@ namespace build2 // noop_rule // bool noop_rule:: - match (action, target&) const + match (action, target& t) const { - return true; + return !exclude_group_ || !t.is_a<group> (); } recipe noop_rule:: diff --git a/libbuild2/rule.hxx b/libbuild2/rule.hxx index eceb6ad..71782c0 100644 --- a/libbuild2/rule.hxx +++ b/libbuild2/rule.hxx @@ -209,8 +209,17 @@ namespace build2 virtual recipe apply (action, target&) const override; - noop_rule () {} - static const noop_rule instance; + // If exclude_group is true then exclude the group-based targets (since + // their membership can only be accurately determined by the ad hoc + // recipe). + // + explicit + noop_rule (bool exclude_group = false): exclude_group_ (exclude_group) {} + + static const noop_rule instance; // Note: does not exclude group. + + private: + bool exclude_group_; }; // Ad hoc rule. diff --git a/libbuild2/scheduler.cxx b/libbuild2/scheduler.cxx index 69673e6..ebb38e4 100644 --- a/libbuild2/scheduler.cxx +++ b/libbuild2/scheduler.cxx @@ -701,7 +701,7 @@ namespace build2 } scheduler::monitor_guard scheduler:: - monitor (atomic_count& c, size_t t, function<size_t (size_t)> f) + monitor (atomic_count& c, size_t t, function<size_t (size_t, size_t)> f) { assert (monitor_count_ == nullptr && t != 0); @@ -713,7 +713,7 @@ namespace build2 monitor_count_ = &c; monitor_tshold_.store (t, memory_order_relaxed); - monitor_init_ = c.load (memory_order_relaxed); + monitor_prev_ = c.load (memory_order_relaxed); monitor_func_ = move (f); return monitor_guard (this); diff --git a/libbuild2/scheduler.hxx b/libbuild2/scheduler.hxx index 3cc206e..2d4d189 100644 --- a/libbuild2/scheduler.hxx +++ b/libbuild2/scheduler.hxx @@ -452,9 +452,11 @@ namespace build2 // should be set before any tasks are queued and cleared after all of // them have completed. // - // The counter must go in one direction, either increasing or decreasing, - // and should contain the initial value during the call. Zero threshold - // value is reserved. + // The counter can go in either direction and should contain the initial + // value during the call. The callback function is called when the current + // value differs form the previous/initial by at least the specified + // threshold. The callback returns the new threshold. Zero threshold value + // is reserved. The callback calls are serialized and synchronized. // struct monitor_guard { @@ -488,7 +490,9 @@ namespace build2 }; monitor_guard - monitor (atomic_count&, size_t threshold, function<size_t (size_t)>); + monitor (atomic_count&, + size_t threshold, + function<size_t (size_t previous, size_t current)>); // If initially active thread(s) (besides the one that calls startup()) // exist before the call to startup(), then they must call join() before @@ -598,10 +602,10 @@ namespace build2 // Monitor. // - atomic_count* monitor_count_ = nullptr; // NULL if not used. - atomic_count monitor_tshold_; // 0 means locked. - size_t monitor_init_; // Initial count. - function<size_t (size_t)> monitor_func_; + atomic_count* monitor_count_ = nullptr; // NULL if not used. + atomic_count monitor_tshold_; // 0 means locked. + size_t monitor_prev_; // Previous values. + function<size_t (size_t, size_t)> monitor_func_; build2::mutex mutex_; bool shutdown_ = true; // Shutdown flag. @@ -914,19 +918,19 @@ namespace build2 if (monitor_tshold_.compare_exchange_strong ( t, 0, - memory_order_release, + memory_order_acq_rel, // Synchronize on success. memory_order_relaxed)) { - // Now we are the only ones messing with this. + // Now we are the only ones messing with this and everything + // is synchronized. // + size_t p (monitor_prev_); size_t v (monitor_count_->load (memory_order_relaxed)); - if (v != monitor_init_) + if ((p > v ? p - v : p < v ? v - p : 0) >= t) { - // See which direction we are going. - // - if (v > monitor_init_ ? (v >= t) : (v <= t)) - t = monitor_func_ (v); + t = monitor_func_ (p, v); + monitor_prev_= v; } monitor_tshold_.store (t, memory_order_release); diff --git a/libbuild2/scheduler.txx b/libbuild2/scheduler.txx index 87c9384..15fefcc 100644 --- a/libbuild2/scheduler.txx +++ b/libbuild2/scheduler.txx @@ -29,12 +29,13 @@ namespace build2 // if (monitor_count_ != nullptr) { + size_t t (monitor_tshold_.load (memory_order_relaxed)); + size_t p (monitor_prev_); size_t v (monitor_count_->load (memory_order_relaxed)); - if (v != monitor_init_) + if ((p > v ? p - v : p < v ? v - p : 0) >= t) { - size_t t (monitor_tshold_.load (memory_order_relaxed)); - if (v > monitor_init_ ? (v >= t) : (v <= t)) - monitor_tshold_.store (monitor_func_ (v), memory_order_relaxed); + monitor_tshold_.store (monitor_func_ (p, v), memory_order_relaxed); + monitor_prev_ = v; } } diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx index ece78b7..d792ed2 100644 --- a/libbuild2/scope.hxx +++ b/libbuild2/scope.hxx @@ -459,14 +459,20 @@ namespace build2 // when an action is executed on the dir{} target that corresponds to this // scope. The pre callback is called just before the recipe and the post // -- immediately after. The callbacks are only called if the recipe - // (including noop recipe) is executed for the corresponding target. The - // callbacks should only be registered during the load phase. + // (including noop recipe) is executed for the corresponding target. + // + // The callback should only be registered during the load phase. Note that + // it's registered for the inner action, meaning that it will be called + // for any outer action (which is discernible from the first argument of + // the callback). // // It only makes sense for callbacks to return target_state changed or // unchanged and to throw failed in case of an error. These pre/post // states will be merged with the recipe state and become the target // state. See execute_recipe() for details. // + // See also context::operation_callback. + // public: struct operation_callback { diff --git a/libbuild2/script/parser.cxx b/libbuild2/script/parser.cxx index 84d2afc..a82ccb8 100644 --- a/libbuild2/script/parser.cxx +++ b/libbuild2/script/parser.cxx @@ -2460,19 +2460,19 @@ namespace build2 // struct loop_data { - lines::const_iterator i; - lines::const_iterator e; const function<exec_set_function>& exec_set; const function<exec_cmd_function>& exec_cmd; const function<exec_cond_function>& exec_cond; const function<exec_for_function>& exec_for; + lines::const_iterator i; + lines::const_iterator e; const iteration_index* ii; size_t& li; variable_pool* var_pool; decltype (fcend)& fce; lines::const_iterator& fe; - } ld {i, e, - exec_set, exec_cmd, exec_cond, exec_for, + } ld {exec_set, exec_cmd, exec_cond, exec_for, + i, e, ii, li, var_pool, fcend, @@ -2558,7 +2558,6 @@ namespace build2 const location& ll; size_t fli; iteration_index& fi; - } d {ld, env, vname, attrs, ll, fli, fi}; function<void (string&&)> f ( @@ -2676,12 +2675,19 @@ namespace build2 if (val) { - // If this value is a vector, then save its element type so + // If the value type provides custom iterate function, then + // use that (see value_type::iterate for details). + // + auto iterate (val.type != nullptr + ? val.type->iterate + : nullptr); + + // If this value is a container, then save its element type so // that we can typify each element below. // const value_type* etype (nullptr); - if (val.type != nullptr) + if (!iterate && val.type != nullptr) { etype = val.type->element_type; @@ -2693,37 +2699,84 @@ namespace build2 size_t fli (li); iteration_index fi {1, ii}; - names& ns (val.as<names> ()); - for (auto ni (ns.begin ()), ne (ns.end ()); ni != ne; ++ni) + names* ns (!iterate ? &val.as<names> () : nullptr); + + // Similar to above. + // + struct loop_data + { + const function<exec_set_function>& exec_set; + const function<exec_cmd_function>& exec_cmd; + const function<exec_cond_function>& exec_cond; + const function<exec_for_function>& exec_for; + lines::const_iterator i; + lines::const_iterator e; + const location& ll; + size_t& li; + variable_pool* var_pool; + const variable& var; + const attributes& val_attrs; + decltype (fcend)& fce; + lines::const_iterator& fe; + iteration_index& fi; + + } ld {exec_set, exec_cmd, exec_cond, exec_for, + i, e, + ll, li, + var_pool, *var, val_attrs, + fcend, fe, fi}; + + function<bool (value&&, bool first)> iteration = + [this, &ld] (value&& v, bool) { - li = fli; + ld.exec_for (ld.var, move (v), ld.val_attrs, ld.ll); - // Set the variable value. + // Find the construct end, if it is not found yet. // - bool pair (ni->pair); - names n; - n.push_back (move (*ni)); - if (pair) n.push_back (move (*++ni)); - value v (move (n)); // Untyped. + if (ld.fe == ld.e) + ld.fe = ld.fce (ld.i, true, false); + + if (!exec_lines ( + ld.i + 1, ld.fe, + ld.exec_set, ld.exec_cmd, ld.exec_cond, ld.exec_for, + &ld.fi, ld.li, + ld.var_pool)) + return false; + + ld.fi.index++; + return true; + }; - if (etype != nullptr) - typify (v, *etype, var); + if (!iterate) + { + for (auto nb (ns->begin ()), ni (nb), ne (ns->end ()); + ni != ne; + ++ni) + { + bool first (ni == nb); - exec_for (*var, move (v), val_attrs, ll); + li = fli; - // Find the construct end, if it is not found yet. - // - if (fe == e) - fe = fcend (i, true, false); + // Set the variable value. + // + bool pair (ni->pair); + names n; + n.push_back (move (*ni)); + if (pair) n.push_back (move (*++ni)); + value v (move (n)); // Untyped. - if (!exec_lines (i + 1, fe, - exec_set, exec_cmd, exec_cond, exec_for, - &fi, li, - var_pool)) - return false; + if (etype != nullptr) + typify (v, *etype, var); - fi.index++; + if (!iteration (move (v), first)) + return false; + } + } + else + { + if (!iterate (val, iteration)) + return false; } } diff --git a/libbuild2/test/operation.cxx b/libbuild2/test/operation.cxx index 2535adb..640d2fe 100644 --- a/libbuild2/test/operation.cxx +++ b/libbuild2/test/operation.cxx @@ -63,7 +63,8 @@ namespace build2 "tested", "has nothing to test", // We cannot "be tested". execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, &pre_test, nullptr, nullptr, @@ -84,6 +85,7 @@ namespace build2 op_update.name_done, op_update.mode, op_update.concurrency, + op_update.keep_going, op_update.pre_operation, op_update.post_operation, op_update.operation_pre, diff --git a/libbuild2/variable.cxx b/libbuild2/variable.cxx index fb9e840..0abc360 100644 --- a/libbuild2/variable.cxx +++ b/libbuild2/variable.cxx @@ -2122,9 +2122,9 @@ namespace build2 return r; } - static void + static bool json_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { // Implement in terms of subscript for consistency (in particular, // iterating over simple values like number, string). @@ -2136,8 +2136,11 @@ namespace build2 if (!e.second) break; - f (move (e.first), i == 0); + if (!f (move (e.first), i == 0)) + return false; } + + return true; } const json_value value_traits<json_value>::empty_instance; @@ -3317,10 +3320,13 @@ namespace build2 value_traits<vector<pair<string, optional<string>>>>; template struct LIBBUILD2_DEFEXPORT + value_traits<vector<pair<string, optional<bool>>>>; + + template struct LIBBUILD2_DEFEXPORT value_traits<vector<pair<optional<string>, string>>>; template struct LIBBUILD2_DEFEXPORT - value_traits<vector<pair<string, optional<bool>>>>; + value_traits<vector<pair<optional<string>, bool>>>; template struct LIBBUILD2_DEFEXPORT value_traits<set<string>>; template struct LIBBUILD2_DEFEXPORT value_traits<set<json_value>>; diff --git a/libbuild2/variable.hxx b/libbuild2/variable.hxx index eebb767..e55a121 100644 --- a/libbuild2/variable.hxx +++ b/libbuild2/variable.hxx @@ -124,12 +124,13 @@ namespace build2 const location& sloc, const location& bloc); - // Custom iteration function. It should invoked the specified function for + // Custom iteration function. It should invoke the specified function for // each element in order. If NULL, then the generic implementation is - // used. The passed value is never NULL. + // used. The passed value is never NULL. If the specified function returns + // false, then stop the iteration and return false. Otherwise return true. // - void (*const iterate) (const value&, - const function<void (value&&, bool first)>&); + bool (*const iterate) (const value&, + const function<bool (value&&, bool first)>&); }; // The order of the enumerators is arranged so that their integral values @@ -1379,6 +1380,9 @@ namespace build2 extern template struct LIBBUILD2_DECEXPORT value_traits<vector<pair<string, optional<bool>>>>; + extern template struct LIBBUILD2_DECEXPORT + value_traits<vector<pair<optional<string>, bool>>>; + extern template struct LIBBUILD2_DECEXPORT value_traits<set<string>>; extern template struct LIBBUILD2_DECEXPORT value_traits<set<json_value>>; diff --git a/libbuild2/variable.txx b/libbuild2/variable.txx index a1ee340..6e00f89 100644 --- a/libbuild2/variable.txx +++ b/libbuild2/variable.txx @@ -686,16 +686,19 @@ namespace build2 // Provide iterate for vector<T> for efficiency. // template <typename T> - void + bool vector_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { const auto& v (val.as<vector<T>> ()); // Never NULL. for (auto b (v.begin ()), i (b), e (v.end ()); i != e; ++i) { - f (value (*i), i == b); + if (!f (value (*i), i == b)) + return false; } + + return true; } // Make sure these are static-initialized together. Failed that VC will make @@ -1071,16 +1074,19 @@ namespace build2 // Provide iterate for set<T> for efficiency. // template <typename T> - void + bool set_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { const auto& v (val.as<set<T>> ()); // Never NULL. for (auto b (v.begin ()), i (b), e (v.end ()); i != e; ++i) { - f (value (*i), i == b); + if (!f (value (*i), i == b)) + return false; } + + return true; } // Make sure these are static-initialized together. Failed that VC will make diff --git a/tests/test/script/runner/for.testscript b/tests/test/script/runner/for.testscript index f43fcc2..0ca67c3 100644 --- a/tests/test/script/runner/for.testscript +++ b/tests/test/script/runner/for.testscript @@ -30,6 +30,19 @@ %.+ -b% EOO + : custom-iteration + : + $c <<EOI && $b >>EOO + j = $json.parse('[1, 2, 3]') + for e: $j + echo $e >| + end + EOI + 1 + 2 + 3 + EOO + : special-var : $c <<EOI && $b 2>>EOE != 0 |