From 83bb02cada0b894d9134cc5489999e0f0fe8bd7c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 8 Jul 2019 14:55:35 +0200 Subject: Move in build system module to separate library --- libbuild2/in/buildfile | 62 +++++++ libbuild2/in/export.hxx | 34 ++++ libbuild2/in/init.cxx | 126 +++++++++++++ libbuild2/in/init.hxx | 30 +++ libbuild2/in/rule.cxx | 485 ++++++++++++++++++++++++++++++++++++++++++++++++ libbuild2/in/rule.hxx | 90 +++++++++ libbuild2/in/target.cxx | 62 +++++++ libbuild2/in/target.hxx | 48 +++++ 8 files changed, 937 insertions(+) create mode 100644 libbuild2/in/buildfile create mode 100644 libbuild2/in/export.hxx create mode 100644 libbuild2/in/init.cxx create mode 100644 libbuild2/in/init.hxx create mode 100644 libbuild2/in/rule.cxx create mode 100644 libbuild2/in/rule.hxx create mode 100644 libbuild2/in/target.cxx create mode 100644 libbuild2/in/target.hxx (limited to 'libbuild2/in') diff --git a/libbuild2/in/buildfile b/libbuild2/in/buildfile new file mode 100644 index 0000000..ae07e27 --- /dev/null +++ b/libbuild2/in/buildfile @@ -0,0 +1,62 @@ +# file : libbuild2/in/buildfile +# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import int_libs = libbutl%lib{butl} + +include ../ +int_libs += ../lib{build2} + +./: lib{build2-in}: libul{build2-in}: {hxx ixx txx cxx}{** -**.test...} \ + $int_libs + +# Unit tests. +# +exe{*.test}: +{ + test = true + install = false +} + +for t: cxx{**.test...} +{ + d = $directory($t) + n = $name($t)... + + ./: $d/exe{$n}: $t $d/{hxx ixx txx}{+$n} $d/testscript{+$n} + $d/exe{$n}: libul{build2-in}: bin.whole = false +} + +# Build options. +# +obja{*}: cxx.poptions += -DLIBBUILD2_IN_STATIC_BUILD +objs{*}: cxx.poptions += -DLIBBUILD2_IN_SHARED_BUILD + +# Export options. +# +lib{build2-in}: +{ + cxx.export.poptions = "-I$out_root" "-I$src_root" + cxx.export.libs = $int_libs +} + +liba{build2-in}: cxx.export.poptions += -DLIBBUILD2_IN_STATIC +libs{build2-in}: cxx.export.poptions += -DLIBBUILD2_IN_SHARED + +# For pre-releases use the complete version to make sure they cannot be used +# in place of another pre-release or the final version. See the version module +# for details on the version.* variable values. +# +if $version.pre_release + lib{build2-in}: bin.lib.version = @"-$version.project_id" +else + lib{build2-in}: bin.lib.version = @"-$version.major.$version.minor" + +# Install into the libbuild2/in/ subdirectory of, say, /usr/include/ +# recreating subdirectories. +# +{hxx ixx txx}{*}: +{ + install = include/libbuild2/in/ + install.subdirs = true +} diff --git a/libbuild2/in/export.hxx b/libbuild2/in/export.hxx new file mode 100644 index 0000000..47909e7 --- /dev/null +++ b/libbuild2/in/export.hxx @@ -0,0 +1,34 @@ +#pragma once + +// Normally we don't export class templates (but do complete specializations), +// inline functions, and classes with only inline member functions. Exporting +// classes that inherit from non-exported/imported bases (e.g., std::string) +// will end up badly. The only known workarounds are to not inherit or to not +// export. Also, MinGW GCC doesn't like seeing non-exported functions being +// used before their inline definition. The workaround is to reorder code. In +// the end it's all trial and error. + +#if defined(LIBBUILD2_IN_STATIC) // Using static. +# define LIBBUILD2_IN_SYMEXPORT +#elif defined(LIBBUILD2_IN_STATIC_BUILD) // Building static. +# define LIBBUILD2_IN_SYMEXPORT +#elif defined(LIBBUILD2_IN_SHARED) // Using shared. +# ifdef _WIN32 +# define LIBBUILD2_IN_SYMEXPORT __declspec(dllimport) +# else +# define LIBBUILD2_IN_SYMEXPORT +# endif +#elif defined(LIBBUILD2_IN_SHARED_BUILD) // Building shared. +# ifdef _WIN32 +# define LIBBUILD2_IN_SYMEXPORT __declspec(dllexport) +# else +# define LIBBUILD2_IN_SYMEXPORT +# endif +#else +// If none of the above macros are defined, then we assume we are being used +// by some third-party build system that cannot/doesn't signal the library +// type. Note that this fallback works for both static and shared but in case +// of shared will be sub-optimal compared to having dllimport. +// +# define LIBBUILD2_IN_SYMEXPORT // Using static or shared. +#endif diff --git a/libbuild2/in/init.cxx b/libbuild2/in/init.cxx new file mode 100644 index 0000000..6ef996b --- /dev/null +++ b/libbuild2/in/init.cxx @@ -0,0 +1,126 @@ +// file : libbuild2/in/init.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + namespace in + { + static const rule rule_ ("in", "in"); + + bool + base_init (scope& rs, + scope&, + const location&, + unique_ptr&, + bool first, + bool, + const variable_map&) + { + tracer trace ("in::base_init"); + l5 ([&]{trace << "for " << rs;}); + + assert (first); + + // Enter variables. + // + { + auto& vp (var_pool.rw (rs)); + + // Alternative variable substitution symbol with '$' being the + // default. + // + vp.insert ("in.symbol"); + + // Substitution mode. Valid values are 'strict' (default) and 'lax'. + // In the strict mode every substitution symbol is expected to start a + // substitution with the double symbol (e.g., $$) serving as an escape + // sequence. + // + // In the lax mode a pair of substitution symbols is only treated as a + // substitution if what's between them looks like a build2 variable + // name (i.e., doesn't contain spaces, etc). Everything else, + // including unterminated substitution symbols, is copied as is. Note + // also that in this mode the double symbol is not treated as an + // escape sequence. + // + // The lax mode is mostly useful when trying to reuse existing .in + // files, for example, from autoconf. Note, however, that the lax mode + // is still stricter than the autoconf's semantics which also leaves + // unknown substitutions as is. + // + vp.insert ("in.substitution"); + } + + // Register target types. + // + rs.target_types.insert (); + + return true; + } + + bool + init (scope& rs, + scope& bs, + const location& loc, + unique_ptr&, + bool, + bool, + const variable_map&) + { + tracer trace ("in::init"); + l5 ([&]{trace << "for " << bs;}); + + // Load in.base. + // + if (!cast_false (rs["in.base.loaded"])) + load_module (rs, rs, "in.base", loc); + + // Register rules. + // + { + auto& r (bs.rules); + + // There are rules that are "derived" from this generic in rule in + // order to provide extended preprocessing functionality (see the + // version module for an example). To make sure they are tried first + // we register for path_target, not file, but in rule::match() we only + // match if the target is a file. A bit of a hack. + // + r.insert (perform_update_id, "in", rule_); + r.insert (perform_clean_id, "in", rule_); + r.insert (configure_update_id, "in", rule_); + } + + return true; + } + + static const module_functions mod_functions[] = + { + // NOTE: don't forget to also update the documentation in init.hxx if + // changing anything here. + + {"in.base", nullptr, base_init}, + {"in", nullptr, init}, + {nullptr, nullptr, nullptr} + }; + + const module_functions* + build2_in_load () + { + return mod_functions; + } + } +} diff --git a/libbuild2/in/init.hxx b/libbuild2/in/init.hxx new file mode 100644 index 0000000..b82c77e --- /dev/null +++ b/libbuild2/in/init.hxx @@ -0,0 +1,30 @@ +// file : libbuild2/in/init.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_IN_INIT_HXX +#define LIBBUILD2_IN_INIT_HXX + +#include +#include + +#include + +#include + +namespace build2 +{ + namespace in + { + // Module `in` does not require bootstrapping. + // + // Submodules: + // + // `in.base` -- only variables and target types. + // + extern "C" LIBBUILD2_IN_SYMEXPORT const module_functions* + build2_in_load (); + } +} + +#endif // LIBBUILD2_IN_INIT_HXX diff --git a/libbuild2/in/rule.cxx b/libbuild2/in/rule.cxx new file mode 100644 index 0000000..bdc9b24 --- /dev/null +++ b/libbuild2/in/rule.cxx @@ -0,0 +1,485 @@ +// file : libbuild2/in/rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include // strtoull() + +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace in + { + bool rule:: + match (action a, target& xt, const string&) const + { + tracer trace ("in::rule::match"); + + if (!xt.is_a ()) // See module init() for details. + return false; + + file& t (static_cast (xt)); + + bool fi (false); // Found in. + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (include (a, t, p) != include_type::normal) // Excluded/ad hoc. + continue; + + fi = fi || p.is_a (); + } + + // Note that while normally we print these at verbosity level 4, this + // one gets quite noisy since we try this rule for any file target. + // + if (!fi) + l5 ([&]{trace << "no in file prerequisite for target " << t;}); + + return fi; + } + + recipe rule:: + apply (action a, target& xt) const + { + file& t (static_cast (xt)); + + // Derive the file name. + // + t.derive_path (); + + // Inject dependency on the output directory. + // + inject_fsdir (a, t); + + // Match prerequisite members. + // + match_prerequisite_members (a, + t, + [this] (action a, + const target& t, + const prerequisite_member& p, + include_type i) + { + return search (a, t, p, i); + }); + + switch (a) + { + case perform_update_id: return [this] (action a, const target& t) + { + return perform_update (a, t); + }; + case perform_clean_id: return &perform_clean_depdb; // Standard clean. + default: return noop_recipe; // Configure update. + } + } + + target_state rule:: + perform_update (action a, const target& xt) const + { + tracer trace ("in::rule::perform_update"); + + const file& t (xt.as ()); + const path& tp (t.path ()); + + // Substitution symbol. + // + char sym (symbol_); + if (const string* s = cast_null (t["in.symbol"])) + { + if (s->size () == 1) + sym = s->front (); + else + fail << "invalid substitution symbol '" << *s << "'"; + } + + // Substitution mode. + // + bool strict (strict_); + if (const string* s = cast_null (t["in.substitution"])) + { + if (*s == "lax") + strict = false; + else if (*s != "strict") + fail << "invalid substitution mode '" << *s << "'"; + } + + // Determine if anything needs to be updated. + // + timestamp mt (t.load_mtime ()); + auto pr (execute_prerequisites (a, t, mt)); + + bool update (!pr.first); + target_state ts (update ? target_state::changed : *pr.first); + + const in& i (pr.second); + const path& ip (i.path ()); + + // We use depdb to track changes to the .in file name, symbol/mode, and + // variable values that have been substituted. + // + depdb dd (tp + ".d"); + + // First should come the rule name/version. + // + if (dd.expect (rule_id_ + " 1") != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + // Then the substitution symbol. + // + if (dd.expect (string (1, sym)) != nullptr) + l4 ([&]{trace << "substitution symbol mismatch forcing update of" + << t;}); + + // Then the substitution mode. + // + if (dd.expect (strict ? "strict" : "lax") != nullptr) + l4 ([&]{trace << "substitution mode mismatch forcing update of" + << t;}); + + // Then the .in file. + // + if (dd.expect (i.path ()) != nullptr) + l4 ([&]{trace << "in file mismatch forcing update of " << t;}); + + // Update if any mismatch or depdb is newer that the output. + // + if (dd.writing () || dd.mtime > mt) + update = true; + + // Substituted variable values. + // + // The plan is to save each substituted variable name and the hash of + // its value one entry per line. Plus the line location of its expansion + // for diagnostics. + // + // If update is true (i.e., the .in file has changes), then we simply + // overwrite the whole list. + // + // If update is false, then we need to read each name/hash, query and + // hash its current value, and compare. If hashes differ, then we need + // to start overwriting from this variable (the prefix of variables + // couldn't have changed since the .in file hasn't changed). + // + // Note that if the .in file substitutes the same variable multiple + // times, then we will end up with multiple entries for such a variable. + // For now we assume this is ok since this is probably not very common + // and it makes the overall logic simpler. + // + // Note also that because updating the depdb essentially requires + // performing the substitutions, this rule ignored the dry-run mode. + // + size_t dd_skip (0); // Number of "good" variable lines. + + if (update) + { + // If we are still reading, mark the next line for overwriting. + // + if (dd.reading ()) + { + dd.read (); // Read the first variable line, if any. + dd.write (); // Mark it for overwriting. + } + } + else + { + while (dd.more ()) + { + if (string* s = dd.read ()) + { + // The line format is: + // + // + // + // Note that can contain spaces (see the constraint check + // expressions in the version module). + // + char* e (nullptr); + uint64_t ln (strtoull (s->c_str (), &e, 10)); + + size_t p1 (*e == ' ' ? e - s->c_str () : string::npos); + size_t p2 (s->rfind (' ')); + + if (p1 != string::npos && p2 != string::npos && p2 - p1 > 1) + { + string n (*s, p1 + 1, p2 - p1 - 1); + + // Note that we have to call substitute(), not lookup() since it + // can be overriden with custom substitution semantics. + // + optional v ( + substitute (location (&ip, ln), a, t, n, strict)); + + assert (v); // Rule semantics change without version increment? + + if (s->compare (p2 + 1, + string::npos, + sha256 (*v).string ()) == 0) + { + dd_skip++; + continue; + } + else + l4 ([&]{trace << n << " variable value mismatch forcing " + << "update of " << t;}); + // Fall through. + } + + dd.write (); // Mark this line for overwriting. + + // Fall through. + } + + break; + } + } + + if (dd.writing ()) // Recheck. + update = true; + + // If nothing changed, then we are done. + // + if (!update) + { + dd.close (); + return ts; + } + + if (verb >= 2) + text << program_ << ' ' << ip << " >" << tp; + else if (verb) + text << program_ << ' ' << ip; + + // Read and process the file, one line at a time, while updating depdb. + // + const char* what; + const path* whom; + try + { + what = "open"; whom = &ip; + ifdstream ifs (ip, fdopen_mode::in, ifdstream::badbit); + + // See fdopen() for details (umask, etc). + // + permissions prm (permissions::ru | permissions::wu | + permissions::rg | permissions::wg | + permissions::ro | permissions::wo); + + if (t.is_a ()) + prm |= permissions::xu | permissions::xg | permissions::xo; + + // Remove the existing file to make sure permissions take effect. If + // this fails then presumable writing to it will fail as well and we + // will complain there. + // + try_rmfile (tp, true /* ignore_error */); + + what = "open"; whom = &tp; + ofdstream ofs (fdopen (tp, + fdopen_mode::out | fdopen_mode::create, + prm)); + auto_rmfile arm (tp); + + string s; // Reuse the buffer. + for (size_t ln (1);; ++ln) + { + what = "read"; whom = &ip; + if (!getline (ifs, s)) + break; // Could not read anything, not even newline. + + // Not tracking column for now (see also depdb above). + // + const location l (&ip, ln); + + // Scan the line looking for substiutions in the $$ form. In + // the strict mode treat $$ as an escape sequence. + // + for (size_t b (0), n, d; b != (n = s.size ()); b += d) + { + d = 1; + + if (s[b] != sym) + continue; + + // Note that in the lax mode these should still be substitutions: + // + // @project@@ + // @@project@ + + // Find the other end. + // + size_t e (b + 1); + for (; e != (n = s.size ()); ++e) + { + if (s[e] == sym) + { + if (strict && e + 1 != n && s[e + 1] == sym) // Escape. + s.erase (e, 1); // Keep one, erase the other. + else + break; + } + } + + if (e == n) + { + if (strict) + fail (l) << "unterminated '" << sym << "'" << endf; + + break; + } + + if (e - b == 1) // Escape (or just double symbol in the lax mode). + { + if (strict) + s.erase (b, 1); // Keep one, erase the other. + + continue; + } + + // We have a (potential, in the lax mode) substition with b + // pointing to the opening symbol and e -- to the closing. + // + string name (s, b + 1, e - b -1); + if (optional val = substitute (l, a, t, name, strict)) + { + // Save in depdb. + // + if (dd_skip == 0) + { + // The line format is: + // + // + // + string s (to_string (ln)); + s += ' '; + s += name; + s += ' '; + s += sha256 (*val).string (); + dd.write (s); + } + else + --dd_skip; + + // Patch the result in and adjust the delta. + // + s.replace (b, e - b + 1, *val); + d = val->size (); + } + else + d = e - b + 1; // Ignore this substitution. + } + + what = "write"; whom = &tp; + if (ln != 1) + ofs << '\n'; // See below. + ofs << s; + } + + // Close depdb before closing the output file so its mtime is not + // newer than of the output. + // + dd.close (); + + what = "close"; whom = &tp; + ofs << '\n'; // Last write to make sure our mtime is older than dd. + ofs.close (); + arm.cancel (); + + what = "close"; whom = &ip; + ifs.close (); + } + catch (const io_error& e) + { + fail << "unable to " << what << ' ' << *whom << ": " << e; + } + + dd.check_mtime (tp); + + t.mtime (system_clock::now ()); + return target_state::changed; + } + + prerequisite_target rule:: + search (action, + const target& t, + const prerequisite_member& p, + include_type i) const + { + return prerequisite_target (&build2::search (t, p), i); + } + + string rule:: + lookup (const location& l, action, const target& t, const string& n) const + { + if (auto x = t[n]) + { + value v (*x); + + // For typed values call string() for conversion. + // + try + { + return convert ( + v.type == nullptr + ? move (v) + : functions.call (&t.base_scope (), + "string", + vector_view (&v, 1), + l)); + } + catch (const invalid_argument& e) + { + fail (l) << e << + info << "while substituting '" << n << "'" << endf; + } + } + else + fail (l) << "undefined variable '" << n << "'" << endf; + } + + optional rule:: + substitute (const location& l, + action a, + const target& t, + const string& n, + bool strict) const + { + // In the lax mode scan the fragment to make sure it is a variable name + // (that is, it can be expanded in a buildfile as just $; see + // lexer's variable mode for details). + // + if (!strict) + { + for (size_t i (0), e (n.size ()); i != e; ) + { + bool f (i == 0); // First. + char c (n[i++]); + bool l (i == e); // Last. + + if (c == '_' || (f ? alpha (c) : alnum (c))) + continue; + + if (c == '.' && !l) + continue; + + return nullopt; // Ignore this substitution. + } + } + + return lookup (l, a, t, n); + } + } +} diff --git a/libbuild2/in/rule.hxx b/libbuild2/in/rule.hxx new file mode 100644 index 0000000..0daf6c0 --- /dev/null +++ b/libbuild2/in/rule.hxx @@ -0,0 +1,90 @@ +// file : libbuild2/in/rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_IN_RULE_HXX +#define LIBBUILD2_IN_RULE_HXX + +#include +#include + +#include + +#include + +namespace build2 +{ + namespace in + { + // Preprocess an .in file. + // + // Note that a derived rule can use the target data pad to cache data + // (e.g., in match()) to be used in substitute/lookup() calls. + // + // Note also that currently this rule ignores the dry-run mode (see + // perform_update() for the rationale). + // + class LIBBUILD2_IN_SYMEXPORT rule: public build2::rule + { + public: + // The rule id is used to form the rule name/version entry in depdb. The + // program argument is the pseudo-program name to use in the command + // line diagnostics. + // + rule (string rule_id, + string program, + char symbol = '$', + bool strict = true) + : rule_id_ (move (rule_id)), + program_ (move (program)), + symbol_ (symbol), + strict_ (strict) {} + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + virtual target_state + perform_update (action, const target&) const; + + // Customization hooks. + // + + // Perform prerequisite search. + // + virtual prerequisite_target + search (action, + const target&, + const prerequisite_member&, + include_type) const; + + // Perform variable lookup. + // + virtual string + lookup (const location&, + action, + const target&, + const string& name) const; + + // Perform variable substitution. Return nullopt if it should be + // ignored. + // + virtual optional + substitute (const location&, + action, + const target&, + const string& name, + bool strict) const; + + protected: + const string rule_id_; + const string program_; + char symbol_; + bool strict_; + }; + } +} + +#endif // LIBBUILD2_IN_RULE_HXX diff --git a/libbuild2/in/target.cxx b/libbuild2/in/target.cxx new file mode 100644 index 0000000..490ab10 --- /dev/null +++ b/libbuild2/in/target.cxx @@ -0,0 +1,62 @@ +// file : libbuild2/in/target.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +using namespace std; + +namespace build2 +{ + namespace in + { + static const target* + in_search (const target& xt, const prerequisite_key& cpk) + { + // If we have no extension then derive it from our target. Then delegate + // to file_search(). + // + prerequisite_key pk (cpk); + optional& e (pk.tk.ext); + + if (!e) + { + if (const file* t = xt.is_a ()) + { + const string& te (t->derive_extension ()); + e = te + (te.empty () ? "" : ".") + "in"; + } + else + fail << "prerequisite " << pk << " for a non-file target " << xt; + } + + return file_search (xt, pk); + } + + static bool + in_pattern (const target_type&, + const scope&, + string&, + optional&, + const location& l, + bool) + { + fail (l) << "pattern in in{} prerequisite" << endf; + } + + extern const char in_ext_def[] = ""; // No extension by default. + + const target_type in::static_type + { + "in", + &file::static_type, + &target_factory, + &target_extension_fix, + nullptr, /* default_extension */ // Taken care of by search. + &in_pattern, + &target_print_1_ext_verb, // Same as file. + &in_search, + false + }; + } +} diff --git a/libbuild2/in/target.hxx b/libbuild2/in/target.hxx new file mode 100644 index 0000000..5e3438d --- /dev/null +++ b/libbuild2/in/target.hxx @@ -0,0 +1,48 @@ +// file : libbuild2/in/target.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_IN_TARGET_HXX +#define LIBBUILD2_IN_TARGET_HXX + +#include +#include + +#include + +#include + +namespace build2 +{ + namespace in + { + // This is the venerable .in ("input") file that needs some kind of + // preprocessing. + // + // One interesting aspect of this target type is that the prerequisite + // search is target-dependent. Consider: + // + // hxx{version}: in{version.hxx} // version.hxx.in -> version.hxx + // + // Having to specify the header extension explicitly is inelegant. Instead + // what we really want to write is this: + // + // hxx{version}: in{version} + // + // But how do we know that in{version} means version.hxx.in? That's where + // the target-dependent search comes in: we take into account the target + // we are a prerequisite of. + // + class LIBBUILD2_IN_SYMEXPORT in: public file + { + public: + using file::file; + + public: + static const target_type static_type; + virtual const target_type& dynamic_type () const {return static_type;} + }; + } +} + +#endif // LIBBUILD2_IN_TARGET_HXX -- cgit v1.1