Split build system into library and driver

1 files changed, 673 insertions, 0 deletions

diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx
new file mode 100644
index 0000000..658f266
--- /dev/null
+++ b/libbuild2/parser.hxx
@@ -0,0 +1,673 @@
+// file      : libbuild2/parser.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_PARSER_HXX
+#define LIBBUILD2_PARSER_HXX
+
+#include <stack>
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/spec.hxx>
+#include <libbuild2/lexer.hxx>
+#include <libbuild2/token.hxx>
+#include <libbuild2/variable.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#include <libbuild2/export.hxx>
+
+namespace build2
+{
+  class scope;
+  class target;
+  class prerequisite;
+
+  class LIBBUILD2_SYMEXPORT parser
+  {
+  public:
+    // If boot is true, then we are parsing bootstrap.build and modules
+    // should only be bootstrapped.
+    //
+    explicit
+    parser (bool boot = false): fail ("error", &path_), boot_ (boot) {}
+
+    // Issue diagnostics and throw failed in case of an error.
+    //
+    void
+    parse_buildfile (istream&, const path& name, scope& root, scope& base);
+
+    buildspec
+    parse_buildspec (istream&, const path& name);
+
+    token
+    parse_variable (lexer&, scope&, const variable&, token_type kind);
+
+    pair<value, token>
+    parse_variable_value (lexer&, scope&, const dir_path*, const variable&);
+
+    names
+    parse_export_stub (istream& is, const path& p, scope& r, scope& b)
+    {
+      parse_buildfile (is, p, r, b);
+      return move (export_value_);
+    }
+
+    // Recursive descent parser.
+    //
+  protected:
+
+    // Pattern expansion mode.
+    //
+    enum class pattern_mode
+    {
+      ignore, // Treat as ordinary names.
+      detect, // Ignore pair/dir/type if the first name is a pattern.
+      expand  // Expand to ordinary names.
+    };
+
+    // If one is true then parse a single (logical) line (logical means it
+    // can actually be several lines, e.g., an if-block). Return false if
+    // nothing has been parsed (i.e., we are still on the same token).
+    //
+    // Note that after this function returns, the token is the first token of
+    // the next line (or eos).
+    //
+    bool
+    parse_clause (token&, token_type&, bool one = false);
+
+    void
+    parse_variable_block (token&, token_type&, const target_type*, string);
+
+    // Ad hoc target names inside < ... >.
+    //
+    struct adhoc_names_loc
+    {
+      names ns;
+      location loc;
+    };
+
+    using adhoc_names = small_vector<adhoc_names_loc, 1>;
+
+    void
+    enter_adhoc_members (adhoc_names_loc&&, bool);
+
+    small_vector<reference_wrapper<target>, 1>
+    enter_targets (names&&, const location&, adhoc_names&&, size_t);
+
+    bool
+    parse_dependency (token&, token_type&,
+                      names&&, const location&,
+                      adhoc_names&&,
+                      names&&, const location&,
+                      bool = false);
+
+    void
+    parse_assert (token&, token_type&);
+
+    void
+    parse_print (token&, token_type&);
+
+    void
+    parse_diag (token&, token_type&);
+
+    void
+    parse_dump (token&, token_type&);
+
+    void
+    parse_source (token&, token_type&);
+
+    void
+    parse_include (token&, token_type&);
+
+    void
+    parse_run (token&, token_type&);
+
+    void
+    parse_import (token&, token_type&);
+
+    void
+    parse_export (token&, token_type&);
+
+    void
+    parse_using (token&, token_type&);
+
+    void
+    parse_define (token&, token_type&);
+
+    void
+    parse_if_else (token&, token_type&);
+
+    void
+    parse_for (token&, token_type&);
+
+    void
+    parse_variable (token&, token_type&, const variable&, token_type);
+
+    void
+    parse_type_pattern_variable (token&, token_type&,
+                                 const target_type&, string,
+                                 const variable&, token_type, const location&);
+
+    const variable&
+    parse_variable_name (names&&, const location&);
+
+    // Note: calls attributes_push() that the caller must pop.
+    //
+    value
+    parse_variable_value (token&, token_type&);
+
+    void
+    apply_variable_attributes (const variable&);
+
+    void
+    apply_value_attributes (const variable*, // Optional.
+                            value& lhs,
+                            value&& rhs,
+                            token_type assign_kind);
+
+    // Return the value pack (values can be NULL/typed). Note that for an
+    // empty eval context ('()' potentially with whitespaces in between) the
+    // result is an empty pack, not a pack of one empty.
+    //
+    values
+    parse_eval (token&, token_type&, pattern_mode);
+
+    values
+    parse_eval_comma (token&, token_type&, pattern_mode, bool = false);
+
+    value
+    parse_eval_ternary (token&, token_type&, pattern_mode, bool = false);
+
+    value
+    parse_eval_or (token&, token_type&, pattern_mode, bool = false);
+
+    value
+    parse_eval_and (token&, token_type&, pattern_mode, bool = false);
+
+    value
+    parse_eval_comp (token&, token_type&, pattern_mode, bool = false);
+
+    value
+    parse_eval_value (token&, token_type&, pattern_mode, bool = false);
+
+    // Attributes stack. We can have nested attributes, for example:
+    //
+    // x = [bool] ([uint64] $x == [uint64] $y)
+    //
+    // In this example we only apply the value attributes after evaluating
+    // the context, which has its own attributes.
+    //
+    struct attributes
+    {
+      bool has;                         // Has attributes flag.
+      location loc;                     // Start of attributes location.
+      vector<pair<string, string>> ats; // Attributes.
+
+      explicit operator bool () const {return has;}
+    };
+
+    // Push a new entry into the attributes_ stack. If the next token is '['
+    // parse the attribute sequence until ']' storing the result in the new
+    // stack entry and setting the 'has' flag (unless the attribute list is
+    // empty). Then get the next token and, if standalone is false, verify
+    // it is not newline/eos (i.e., there is something after it). Return the
+    // indication of whether there are any attributes and their location.
+    //
+    // Note that during pre-parsing nothing is pushed into the stack and
+    // the returned attributes object indicates there are no attributes.
+    //
+    pair<bool, location>
+    attributes_push (token&, token_type&, bool standalone = false);
+
+    attributes
+    attributes_pop ()
+    {
+      assert (!pre_parse_);
+      attributes r (move (attributes_.top ()));
+      attributes_.pop ();
+      return r;
+    }
+
+    attributes&
+    attributes_top () {return attributes_.top ();}
+
+    // Source a stream optionnaly entering it as a buildfile and performing
+    // the default target processing.
+    //
+    void
+    source (istream&,
+            const path&,
+            const location&,
+            bool enter,
+            bool default_target);
+
+    // If chunk is true, then parse the smallest but complete, name-wise,
+    // chunk of input. Note that in this case you may still end up with
+    // multiple names, for example, {foo bar} or $foo. In the pre-parse mode
+    // always return empty list of names.
+    //
+    // The what argument is used in diagnostics (e.g., "expected <what>
+    // instead of ...".
+    //
+    // The separators argument specifies the special characters to recognize
+    // inside the name. These can be the directory separators and the '%'
+    // project separator. Note that even if it is NULL, the result may still
+    // contain non-simple names due to variable expansions.
+    //
+
+    static const string name_separators;
+
+    names
+    parse_names (token& t, token_type& tt,
+                 pattern_mode pmode,
+                 bool chunk = false,
+                 const char* what = "name",
+                 const string* separators = &name_separators)
+    {
+      names ns;
+      parse_names (t, tt,
+                   ns,
+                   pmode,
+                   chunk,
+                   what,
+                   separators,
+                   0,
+                   nullopt, nullptr, nullptr);
+      return ns;
+    }
+
+    // Return true if this token starts a name. Or, to put it another way,
+    // calling parse_names() on this token won't fail with the "expected name
+    // instead of <this-token>" error. Only consider '(' if the second
+    // argument is true.
+    //
+    bool
+    start_names (token_type&, bool lparen = true);
+
+    // As above but return the result as a value, which can be typed and NULL.
+    //
+    value
+    parse_value (token& t, token_type& tt,
+                 pattern_mode pmode,
+                 const char* what = "name",
+                 const string* separators = &name_separators,
+                 bool chunk = false)
+    {
+      names ns;
+      auto r (parse_names (t, tt,
+                           ns,
+                           pmode,
+                           chunk,
+                           what,
+                           separators,
+                           0,
+                           nullopt, nullptr, nullptr));
+
+      value v (r.type); // Potentially typed NULL value.
+
+      // This should not fail since we are typing the result of reversal from
+      // the typed value.
+      //
+      if (r.not_null)
+        v.assign (move (ns), nullptr);
+
+      return v;
+    }
+
+    // Append names and return the indication if the parsed value is not NULL
+    // and whether it is typed (and whether it is a pattern if pattern_mode is
+    // detect).
+    //
+    // You may have noticed that what we return here is essentially a value
+    // and doing it this way (i.e., reversing it to untyped names and
+    // returning its type so that it can potentially be "typed back") is kind
+    // of backwards. The reason we are doing it this way is because in many
+    // places we expect things untyped and if we were to always return a
+    // (potentially typed) value, then we would have to reverse it in all
+    // those places. Still it may make sense to look into redesigning the
+    // whole thing one day.
+    //
+    // Currently the only way for the result to be NULL or have a type is if
+    // it is the result of a sole, unquoted variable expansion, function call,
+    // or context evaluation.
+    //
+    struct parse_names_result
+    {
+      bool not_null;
+      const value_type* type;
+      optional<const target_type*> pattern;
+    };
+
+    parse_names_result
+    parse_names (token&, token_type&,
+                 names&,
+                 pattern_mode,
+                 bool chunk = false,
+                 const char* what = "name",
+                 const string* separators = &name_separators,
+                 size_t pairn = 0,
+                 const optional<project_name>& prj = nullopt,
+                 const dir_path* dir = nullptr,
+                 const string* type = nullptr,
+                 bool cross = true,
+                 bool curly = false);
+
+    size_t
+    parse_names_trailer (token&, token_type&,
+                         names&,
+                         pattern_mode,
+                         const char* what,
+                         const string* separators,
+                         size_t pairn,
+                         const optional<project_name>& prj,
+                         const dir_path* dir,
+                         const string* type,
+                         bool cross);
+
+    size_t
+    expand_name_pattern (const location&,
+                         names&&,
+                         names&,
+                         const char* what,
+                         size_t pairn,
+                         const dir_path* dir,
+                         const string* type,
+                         const target_type*);
+
+    size_t
+    splice_names (const location&,
+                  const names_view&,
+                  names&&,
+                  names&,
+                  const char* what,
+                  size_t pairn,
+                  const optional<project_name>& prj,
+                  const dir_path* dir,
+                  const string* type);
+
+    // Skip until newline or eos.
+    //
+    void
+    skip_line (token&, token_type&);
+
+    // Skip until block-closing } or eos, taking into account nested blocks.
+    //
+    void
+    skip_block (token&, token_type&);
+
+    // Return true if the name token can be considered a directive keyword.
+    //
+    bool
+    keyword (token&);
+
+    // Buildspec.
+    //
+    buildspec
+    parse_buildspec_clause (token&, token_type&, size_t);
+
+    // Customization hooks.
+    //
+  protected:
+    // If qual is not empty, then its pair member should indicate the kind
+    // of qualification: ':' -- target, '/' -- scope.
+    //
+    virtual lookup
+    lookup_variable (name&& qual, string&& name, const location&);
+
+    // Utilities.
+    //
+  protected:
+    class enter_scope;
+    class enter_target;
+    class enter_prerequisite;
+
+    // Switch to a new current scope. Note that this function might also have
+    // to switch to a new root scope if the new current scope is in another
+    // project. So both must be saved and restored.
+    //
+    void
+    switch_scope (const dir_path&);
+
+    void
+    process_default_target (token&);
+
+    // Enter buildfile as a target.
+    //
+    void
+    enter_buildfile (const path&);
+
+    // Lexer.
+    //
+  protected:
+    location
+    get_location (const token& t) const
+    {
+      return build2::get_location (t, *path_);
+    }
+
+    token_type
+    next (token&, token_type&);
+
+    // If the current token is newline, then get the next token. Otherwise,
+    // fail unless the current token is eos (i.e., optional newline at the end
+    // of stream). If the after argument is not \0, use it in diagnostics as
+    // the token after which the newline was expectd.
+    //
+    token_type
+    next_after_newline (token&, token_type&, char after = '\0');
+
+    // Be careful with peeking and switching the lexer mode. See keyword()
+    // for more information.
+    //
+    token_type
+    peek ();
+
+    token_type
+    peek (lexer_mode m, char ps = '\0')
+    {
+      // The idea is that if we already have something peeked, then it should
+      // be in the same mode. We also don't re-set the mode since it may have
+      // expired after the first token.
+      //
+      if (peeked_)
+      {
+        assert (peek_.mode == m);
+        return peek_.token.type;
+      }
+
+      mode (m, ps);
+      return peek ();
+    }
+
+    const token&
+    peeked () const
+    {
+      assert (peeked_);
+      return peek_.token;
+    }
+
+    void
+    mode (lexer_mode m, char ps = '\0')
+    {
+      if (replay_ != replay::play)
+        lexer_->mode (m, ps);
+      else
+        // As a sanity check, make sure the mode matches the next token. Note
+        // that we don't check the pair separator since it can be overriden by
+        // the lexer's mode() implementation.
+        //
+        assert (replay_i_ != replay_data_.size () &&
+                replay_data_[replay_i_].mode == m);
+    }
+
+    lexer_mode
+    mode () const
+    {
+      if (replay_ != replay::play)
+        return lexer_->mode ();
+      else
+      {
+        assert (replay_i_ != replay_data_.size ());
+        return replay_data_[replay_i_].mode;
+      }
+    }
+
+    void
+    expire_mode ()
+    {
+      if (replay_ != replay::play)
+        lexer_->expire_mode ();
+    }
+
+    // Token saving and replaying. Note that it can only be used in certain
+    // contexts. Specifically, the code that parses a replay must not interact
+    // with the lexer directly (e.g., the keyword() test). Replays also cannot
+    // nest. For now we don't enforce any of this.
+    //
+    // Note also that the peeked token is not part of the replay, until it
+    // is "got".
+    //
+    void
+    replay_save ()
+    {
+      assert (replay_ == replay::stop);
+      replay_ = replay::save;
+    }
+
+    void
+    replay_play ()
+    {
+      assert ((replay_ == replay::save && !replay_data_.empty ()) ||
+              (replay_ == replay::play && replay_i_ == replay_data_.size ()));
+
+      if (replay_ == replay::save)
+        replay_path_ = path_; // Save old path.
+
+      replay_i_ = 0;
+      replay_ = replay::play;
+    }
+
+    void
+    replay_stop ()
+    {
+      if (replay_ == replay::play)
+        path_ = replay_path_; // Restore old path.
+
+      replay_data_.clear ();
+      replay_ = replay::stop;
+    }
+
+    struct replay_guard
+    {
+      replay_guard (parser& p, bool start = true)
+          : p_ (start ? &p : nullptr)
+      {
+        if (p_ != nullptr)
+          p_->replay_save ();
+      }
+
+      void
+      play ()
+      {
+        if (p_ != nullptr)
+          p_->replay_play ();
+      }
+
+      ~replay_guard ()
+      {
+        if (p_ != nullptr)
+          p_->replay_stop ();
+      }
+
+    private:
+      parser* p_;
+    };
+
+    // Stop saving and get the data.
+    //
+    replay_tokens
+    replay_data ()
+    {
+      assert (replay_ == replay::save);
+
+      replay_tokens r (move (replay_data_));
+      replay_data_.clear ();
+      replay_ = replay::stop;
+      return r;
+    }
+
+    // Set the data and start playing.
+    //
+    void
+    replay_data (replay_tokens&& d)
+    {
+      assert (replay_ == replay::stop);
+
+      replay_path_ = path_; // Save old path.
+
+      replay_data_ = move (d);
+      replay_i_ = 0;
+      replay_ = replay::play;
+    }
+
+    // Implementation details, don't call directly.
+    //
+    replay_token
+    lexer_next ()
+    {
+      lexer_mode m (lexer_->mode ()); // Get it first since it may expire.
+      return replay_token {lexer_->next (), path_, m};
+    }
+
+    const replay_token&
+    replay_next ()
+    {
+      assert (replay_i_ != replay_data_.size ());
+      const replay_token& rt (replay_data_[replay_i_++]);
+
+      // Update the path. Note that theoretically it is possible that peeking
+      // at the next token will "change" the path of the current token. The
+      // workaround would be to call get_location() before peeking.
+      //
+      path_ = rt.file;
+
+      return rt;
+    }
+
+    // Diagnostics.
+    //
+  protected:
+    const fail_mark fail;
+
+  protected:
+    bool pre_parse_ = false;
+    bool boot_;
+
+    const path* path_; // Current path.
+    lexer*      lexer_;
+
+    prerequisite* prerequisite_ = nullptr; // Current prerequisite, if any.
+    target*       target_       = nullptr; // Current target, if any.
+    scope*        scope_        = nullptr; // Current base scope (out_base).
+    scope*        root_         = nullptr; // Current root scope (out_root).
+
+    const dir_path* pbase_ = nullptr; // Current pattern base directory.
+
+    std::stack<attributes> attributes_;
+
+    target* default_target_;
+    names export_value_;
+
+    replay_token peek_;
+    bool peeked_ = false;
+
+    enum class replay {stop, save, play} replay_ = replay::stop;
+    replay_tokens replay_data_;
+    size_t replay_i_;         // Position of the next token during replay.
+    const path* replay_path_; // Path before replay began (to be restored).
+  };
+}
+
+#endif // LIBBUILD2_PARSER_HXX