Implement complete root/base detection, basic module support

This is the initial groundwork for the configuration support.

15 files changed, 482 insertions, 53 deletions

diff --git a/build/b.cxx b/build/b.cxx
index 6bd1d86..a97235c 100644
--- a/build/b.cxx
+++ b/build/b.cxx
@@ -27,6 +27,7 @@
 #include <build/target>
 #include <build/prerequisite>
 #include <build/rule>
+#include <build/module>
 #include <build/algorithm>
 #include <build/process>
 #include <build/diagnostics>
@@ -62,6 +63,115 @@ namespace build
 
     cout << endl;
   }
+
+  inline bool
+  is_src_root (const path& d)
+  {
+    return path_mtime (d / path ("build/root.build")) !=
+      timestamp_nonexistent;
+  }
+
+  inline bool
+  is_out_root (const path& d)
+  {
+    return path_mtime (d / path ("build/bootstrap/src-root.build")) !=
+      timestamp_nonexistent;
+  }
+
+  // Given an src_base directory, look for the project's src_root
+  // based on the presence of known special files. Return empty
+  // path if not found.
+  //
+  path
+  find_src_root (const path& b)
+  {
+    for (path d (b); !d.root () && d != home; d = d.directory ())
+    {
+      if (is_src_root (d))
+        return d;
+    }
+
+    return path ();
+  }
+
+  // The same but for out. Note that we also check whether a
+  // directory happens to be src_root, in case this is an in-
+  // tree build.
+  //
+  path
+  find_out_root (const path& b)
+  {
+    for (path d (b); !d.root () && d != home; d = d.directory ())
+    {
+      if (is_out_root (d) || is_src_root (d))
+        return d;
+    }
+
+    return path ();
+  }
+
+  void
+  bootstrap (scope& rs)
+  {
+    tracer trace ("bootstrap");
+
+    path bf (rs.path () / path ("build/bootstrap/src-root.build"));
+
+    if (path_mtime (bf) == timestamp_nonexistent)
+      return;
+
+    //@@ TODO: if bootstrap files can source other bootstrap files
+    //   (the way to express dependecies), then we need a way to
+    //   prevent multiple sourcing.
+    //
+
+    level4 ([&]{trace << "loading " << bf;});
+
+    ifstream ifs (bf.string ());
+    if (!ifs.is_open ())
+      fail << "unable to open " << bf;
+
+    ifs.exceptions (ifstream::failbit | ifstream::badbit);
+    parser p;
+
+    try
+    {
+      p.parse_buildfile (ifs, bf, rs, rs);
+    }
+    catch (const std::ios_base::failure&)
+    {
+      fail << "failed to read from " << bf;
+    }
+  }
+
+  void
+  root_pre (scope& rs, const path& src_root)
+  {
+    tracer trace ("root_pre");
+
+    path bf (src_root / path ("build/root.build"));
+
+    if (path_mtime (bf) == timestamp_nonexistent)
+      return;
+
+    level4 ([&]{trace << "loading " << bf;});
+
+    ifstream ifs (bf.string ());
+    if (!ifs.is_open ())
+      fail << "unable to open " << bf;
+
+    ifs.exceptions (ifstream::failbit | ifstream::badbit);
+    parser p;
+
+    try
+    {
+      p.parse_buildfile (ifs, bf, rs, rs);
+    }
+    catch (const std::ios_base::failure&)
+    {
+      fail << "failed to read from " << bf;
+    }
+  }
 }
 
 #include <build/native>
@@ -69,6 +179,7 @@ namespace build
 #include <build/cxx/target>
 #include <build/cxx/rule>
 
+#include <build/config/module>
 
 using namespace build;
 
@@ -87,6 +198,10 @@ main (int argc, char* argv[])
     //
     verb = 5;
 
+    // Register modules.
+    //
+    modules["config"] = &config::load;
+
     // Register target types.
     //
     target_types.insert (file::static_type);
@@ -234,53 +349,128 @@ main (int argc, char* argv[])
 
           out_base.normalize ();
 
-          path& src_base (ts.src_base);
-          if (src_base.empty ())
+          // The order in which we determine the roots depends on whether
+          // src_base was specified explicitly. There will also be a few
+          // cases where we are guessing things that can turn out wrong.
+          // Keep track of that so that we can issue more extensive
+          // diagnostics for such cases.
+          //
+          bool guessing (false);
+          path src_root;
+          path out_root;
+
+          path& src_base (ts.src_base); // Update it in buildspec.
+
+          if (!src_base.empty ())
           {
-            //@@ TODO: Configured case: find out_root (looking for
-            //   "build/bootstrap.build" or some such), then src_root
-            //   (stored in this file). Need to also detect the in-tree
-            //   build.
+            if (src_base.relative ())
+              src_base = work / src_base;
+
+            src_base.normalize ();
+
+            // If the src_base was explicitly specified, search for src_root.
             //
+            src_root = find_src_root (src_base);
 
-            // If that doesn't work out (e.g., the first build), then
-            // default to the working directory as src_base.
+            // If not found, assume this is a simple project with src_root
+            // being the same as src_base.
             //
-            src_base = work;
+            if (src_root.empty ())
+            {
+              src_root = src_base;
+              out_root = out_base;
+            }
+            else
+              // Calculate out_root based on src_root/src_base.
+              //
+              out_root = out_base.directory (src_base.leaf (src_root));
+          }
+          else
+          {
+            // If no src_base was explicitly specified, search for out_root.
+            //
+            out_root = find_out_root (out_base);
+
+            // If not found (i.e., we have no idea where the roots are),
+            // then this can mean two things: an in-tree build of a
+            // simple project or a fresh out-of-tree build. Assume this
+            // is the former and set out_root to out_base. If we are
+            // wrong (most likely) and this is the latter, then things
+            // will go badly when we try to load the buildfile.
+            //
+            if (out_root.empty ())
+            {
+              src_root = src_base = out_root = out_base;
+              guessing = true;
+            }
           }
 
-          if (src_base.relative ())
-            src_base = work / src_base;
-
-          src_base.normalize ();
+          // Now we know out_root and, if it was explicitly specified,
+          // src_root. The next step is to create the root scope and
+          // load the bootstrap files, if any. Note that we might already
+          // have done this as a result of one of the preceding target
+          // processing.
+          //
+          auto rsp (scopes.insert (out_root));
+          scope& rs (rsp.first);
 
-          path src_root;
-          path out_root;
+          if (rsp.second)
+          {
+            rs.variables["out_root"] = out_root;
+            bootstrap (rs);
+          }
 
-          // The project's root directory is the one that contains the build/
-          // sub-directory which contains the pre.build file.
+          // See if the bootstrap process set src_root.
           //
-          for (path d (src_base), f ("build/pre.build");
-               !d.root () && d != home;
-               d = d.directory ())
           {
-            if (path_mtime (d / f) != timestamp_nonexistent)
+            auto v (rs.variables["src_root"]);
+
+            if (v)
             {
-              src_root = d;
-              break;
+              // If we also have src_root specified by the user, make
+              // sure they match.
+              //
+              const path& p (v.as<const path&> ());
+
+              if (src_root.empty ())
+                src_root = p;
+              else if (src_root != p)
+                fail << "bootstrapped src_root " << p << " does not match "
+                     << "specified " << src_root;
+            }
+            else
+            {
+              // Bootstrap didn't produce src_root.
+              //
+              if (src_root.empty ())
+              {
+                // If it also wasn't explicitly specified, see if it is
+                // the same as out_root.
+                //
+                if (is_src_root (out_root))
+                  src_root = out_root;
+                else
+                {
+                  // If not, then assume we are running from src_base
+                  // and calculate src_root based on out_root/out_base.
+                  // Note that this is different from the above case
+                  // were we couldn't determine either root.
+                  //
+                  src_base = work;
+                  src_root = src_base.directory (out_base.leaf (out_root));
+                  guessing = true;
+                }
+              }
+
+              v = src_root;
             }
           }
 
-          // If there is no such sub-directory, assume this is a simple
-          // project with src_root being the same as src_base.
+          // At this stage we should have both roots and out_base figured
+          // out. If src_base is still undetermined, calculate it.
           //
-          if (src_root.empty ())
-          {
-            src_root = src_base;
-            out_root = out_base;
-          }
-          else
-            out_root = out_base.directory (src_base.leaf (src_root));
+          if (src_base.empty ())
+            src_base = src_root / out_base.leaf (out_root);
 
           if (verb >= 4)
           {
@@ -291,18 +481,18 @@ main (int argc, char* argv[])
             trace << "  src_root: " << src_root.string ();
           }
 
-          // Create project root and base scopes, set the corresponding
-          // variables. Note that we might already have all of this set
-          // up as a result of one of the preceding target processing.
+          // Load project's root[-pre].build.
           //
-          scope& proot_scope (scopes[out_root]);
-          scope& pbase_scope (scopes[out_base]);
+          root_pre (rs, src_root);
 
-          proot_scope.variables["out_root"] = move (out_root);
-          proot_scope.variables["src_root"] = move (src_root);
+          // Create the base scope. Note that its existence doesn't
+          // mean it was already processed as a base scope; it can
+          // be the same as root.
+          //
+          scope& bs (scopes[out_base]);
 
-          pbase_scope.variables["out_base"] = out_base;
-          pbase_scope.variables["src_base"] = src_base;
+          bs.variables["out_base"] = out_base;
+          bs.variables["src_base"] = src_base;
 
           // Parse the buildfile.
           //
@@ -310,7 +500,7 @@ main (int argc, char* argv[])
 
           // Check if this buildfile has already been loaded.
           //
-          if (!proot_scope.buildfiles.insert (bf).second)
+          if (!rs.buildfiles.insert (bf).second)
           {
             level4 ([&]{trace << "skipping already loaded " << bf;});
             continue;
@@ -320,14 +510,20 @@ main (int argc, char* argv[])
 
           ifstream ifs (bf.string ());
           if (!ifs.is_open ())
-            fail << "unable to open " << bf;
+          {
+            diag_record dr;
+            dr << fail << "unable to open " << bf;
+            if (guessing)
+              dr << info << "consider explicitly specifying src_base "
+                 << "for " << tn;
+          }
 
           ifs.exceptions (ifstream::failbit | ifstream::badbit);
           parser p;
 
           try
           {
-            p.parse_buildfile (ifs, bf, pbase_scope, proot_scope);
+            p.parse_buildfile (ifs, bf, bs, rs);
           }
           catch (const std::ios_base::failure&)
           {
diff --git a/build/buildfile b/build/buildfile
index 4a3b6b3..246e2f6 100644
--- a/build/buildfile
+++ b/build/buildfile
@@ -1,3 +1,4 @@
 exe{b1}: cxx{b algorithm parser lexer name operation spec scope variable \
-  target prerequisite rule native context search diagnostics cxx/target \
-  cxx/rule process timestamp path utility filesystem dump}
+  target prerequisite rule module native context search diagnostics \
+  config/module cxx/target cxx/rule process timestamp path utility \
+  filesystem dump}
diff --git a/build/config/module b/build/config/module
new file mode 100644
index 0000000..5a9d362
--- /dev/null
+++ b/build/config/module
@@ -0,0 +1,19 @@
+// file      : build/config/module -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_CONFIG_MODULE
+#define BUILD_CONFIG_MODULE
+
+#include <build/module>
+
+namespace build
+{
+  namespace config
+  {
+    void
+    load (scope&, scope&, const location&);
+  }
+}
+
+#endif // BUILD_CONFIG_MODULE
diff --git a/build/config/module.cxx b/build/config/module.cxx
new file mode 100644
index 0000000..bbaccdc
--- /dev/null
+++ b/build/config/module.cxx
@@ -0,0 +1,43 @@
+// file      : build/config/module.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/config/module>
+
+#include <build/path>
+#include <build/scope>
+#include <build/diagnostics>
+
+using namespace std;
+
+namespace build
+{
+  namespace config
+  {
+    static bool
+    trigger (scope&, const path& p)
+    {
+      tracer trace ("config::trigger");
+
+      level4 ([&]{trace << "intercepted sourcing of " << p;});
+      return false;
+    }
+
+    void
+    load (scope& root, scope& base, const location& l)
+    {
+      tracer trace ("config::load");
+
+      if (&root != &base)
+        fail (l) << "config module must be loaded in project root scope";
+
+      //@@ TODO: avoid multiple loads (generally, for modules).
+      //
+      level4 ([&]{trace << "for " << root.path () << '/';});
+
+      // Register the build/config.build loading trigger.
+      //
+      root.triggers[path ("build/config.build")] = &trigger;
+    }
+  }
+}
diff --git a/build/config/operation b/build/config/operation
new file mode 100644
index 0000000..a233e63
--- /dev/null
+++ b/build/config/operation
@@ -0,0 +1,17 @@
+// file      : build/config/operation -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_CONFIG_OPERATION
+#define BUILD_CONFIG_OPERATION
+
+#include <build/operation>
+
+namespace build
+{
+  namespace config
+  {
+  }
+}
+
+#endif // BUILD_CONFIG_OPERATION
diff --git a/build/config/operation.cxx b/build/config/operation.cxx
new file mode 100644
index 0000000..cc04929
--- /dev/null
+++ b/build/config/operation.cxx
@@ -0,0 +1,14 @@
+// file      : build/config/operation.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/config/operation>
+
+using namespace std;
+
+namespace build
+{
+  namespace config
+  {
+  }
+}
diff --git a/build/module b/build/module
new file mode 100644
index 0000000..81595d3
--- /dev/null
+++ b/build/module
@@ -0,0 +1,22 @@
+// file      : build/module -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_MODULE
+#define BUILD_MODULE
+
+#include <string>
+#include <unordered_map>
+
+namespace build
+{
+  class scope;
+  class location;
+
+  using module_load_function = void (scope& root, scope& base, const location&);
+  using module_map = std::unordered_map<std::string, module_load_function*>;
+
+  extern module_map modules;
+}
+
+#endif // BUILD_MODULE
diff --git a/build/module.cxx b/build/module.cxx
new file mode 100644
index 0000000..495d849
--- /dev/null
+++ b/build/module.cxx
@@ -0,0 +1,12 @@
+// file      : build/module.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/module>
+
+using namespace std;
+
+namespace build
+{
+  module_map modules;
+}
diff --git a/build/parser b/build/parser
index 50e306d..0d42d73 100644
--- a/build/parser
+++ b/build/parser
@@ -50,6 +50,9 @@ namespace build
     void
     include (token&, token_type&);
 
+    void
+    load (token&, token_type&);
+
     names_type
     names (token& t, token_type& tt)
     {
diff --git a/build/parser.cxx b/build/parser.cxx
index 74b41cc..c7687d3 100644
--- a/build/parser.cxx
+++ b/build/parser.cxx
@@ -19,6 +19,7 @@
 #include <build/target>
 #include <build/prerequisite>
 #include <build/variable>
+#include <build/module>
 #include <build/diagnostics>
 #include <build/context>
 
@@ -49,7 +50,13 @@ namespace build
     default_target_ = nullptr;
 
     out_root_ = &root["out_root"].as<const path&> ();
-    src_root_ = &root["src_root"].as<const path&> ();
+
+    // During bootstrap we may not know src_root yet.
+    //
+    {
+      auto v (root["src_root"]);
+      src_root_ = v ? &v.as<const path&> () : nullptr;
+    }
 
     token t (type::eos, false, 0, 0);
     type tt;
@@ -105,6 +112,12 @@ namespace build
           include (t, tt);
           continue;
         }
+        else if (n == "load")
+        {
+          next (t, tt);
+          load (t, tt);
+          continue;
+        }
       }
 
       // ': foo' is equvalent to '{}: foo' and to 'dir{}: foo'.
@@ -388,9 +401,24 @@ namespace build
       // If the path is relative then use the src directory corresponding
       // to the current directory scope.
       //
-      if (p.relative ())
+      if (src_root_ != nullptr && p.relative ())
         p = src_out (scope_->path (), *out_root_, *src_root_) / p;
 
+      p.normalize ();
+
+      // See if there is a trigger for this path.
+      //
+      if (src_root_ != nullptr && p.sub (*src_root_))
+      {
+        auto i (root_->triggers.find (p.leaf (*src_root_)));
+
+        if (i != root_->triggers.end () && !i->second (*root_, p))
+        {
+          level4 ([&]{trace (l) << "trigger instructed to skip " << p;});
+          continue;
+        }
+      }
+
       ifstream ifs (p.string ());
 
       if (!ifs.is_open ())
@@ -420,6 +448,21 @@ namespace build
 
       lexer_ = ol;
       path_ = op;
+
+      // If src_root is unknown (happens during bootstrap), reload it
+      // in case the just sourced buildfile set it. This way, once it
+      // is set, all the parser mechanism that were disabled (like
+      // relative file source'ing) will start working. Note that they
+      // will still be disabled inside the file that set src_root. For
+      // this to work we would need to keep a reference to the value
+      // stored in the variable plus the update would need to update
+      // the value in place (see value_proxy).
+      //
+      if (src_root_ == nullptr)
+      {
+        auto v ((*root_)["src_root"]);
+        src_root_ = v ? &v.as<const path&> () : nullptr;
+      }
     }
 
     if (tt == type::newline)
@@ -433,6 +476,9 @@ namespace build
   {
     tracer trace ("parser::include", &path_);
 
+    if (src_root_ == nullptr)
+      fail (t) << "inclusion during bootstrap";
+
     // The rest should be a list of buildfiles. Parse them as names
     // to get variable expansion and directory prefixes.
     //
@@ -551,6 +597,41 @@ namespace build
   }
 
   void parser::
+  load (token& t, token_type& tt)
+  {
+    tracer trace ("parser::load", &path_);
+
+    // The rest should be a list of module names. Parse them as names
+    // to get variable expansion, etc.
+    //
+    location l (get_location (t, &path_));
+    names_type ns (tt != type::newline && tt != type::eos
+                   ? names (t, tt)
+                   : names_type ());
+
+    for (name& n: ns)
+    {
+      // For now it should be a simple name.
+      //
+      if (!n.type.empty () || !n.dir.empty ())
+        fail (l) << "module name expected instead of " << n;
+
+      const string& name (n.value);
+      auto i (modules.find (name));
+
+      if (i == modules.end ())
+        fail (l) << "unknown module " << name;
+
+      i->second (*root_, *scope_, l);
+    }
+
+    if (tt == type::newline)
+      next (t, tt);
+    else if (tt != type::eos)
+      fail (t) << "expected newline instead of " << t;
+  }
+
+  void parser::
   print (token& t, token_type& tt)
   {
     for (; tt != type::newline && tt != type::eos; next (t, tt))
diff --git a/build/path b/build/path
index 5c492f7..699882e 100644
--- a/build/path
+++ b/build/path
@@ -1,4 +1,4 @@
-// file      : cutl/path -*- C++ -*-
+// file      : build/path -*- C++ -*-
 // copyright : Copyright (c) 2014-2015 Code Synthesis Tools CC
 // license   : MIT; see accompanying LICENSE file
 
diff --git a/build/pre.build b/build/pre.build
deleted file mode 100644
index e69de29..0000000
--- a/build/pre.build
+++ /dev/null
diff --git a/build/root.build b/build/root.build
new file mode 100644
index 0000000..d10a80c
--- /dev/null
+++ b/build/root.build
@@ -0,0 +1,4 @@
+print root.build
+load config
+source build/config.build
+
diff --git a/build/scope b/build/scope
index e5c5fec..f86d0ed 100644
--- a/build/scope
+++ b/build/scope
@@ -5,7 +5,9 @@
 #ifndef BUILD_SCOPE
 #define BUILD_SCOPE
 
+#include <functional>    // function
 #include <unordered_set>
+#include <unordered_map>
 
 #include <build/path>
 #include <build/path-map>
@@ -57,6 +59,19 @@ namespace build
     //
     std::unordered_set<path_type> buildfiles;
 
+
+    // A map of buildfiles to trigger functions that are executed when
+    // such files are sourced. The path is is assumed to be relative to
+    // the src directory corresponding to this scope.
+    //
+    // The passed path is the actual, absolute buildfile path. If the
+    // returned value is true, then the file is sourced. If false --
+    // the file is ignored. Note that currently triggers can only be
+    // registered on the project root scope.
+    //
+    using trigger_type = std::function<bool (scope&, const path_type&)>;
+    std::unordered_map<path_type, trigger_type> triggers;
+
   private:
     iterator i_;
     scope* parent_;
@@ -68,9 +83,11 @@ namespace build
     // Note that we assume the first insertion into the map is that
     // of the root scope.
     //
+    std::pair<scope&, bool>
+    insert (const path&);
 
     scope&
-    operator[] (const path&);
+    operator[] (const path& p) {return insert (p).first;}
 
     // Find the most qualified scope that encompasses this path.
     //
diff --git a/build/scope.cxx b/build/scope.cxx
index b9b576e..60c83b4 100644
--- a/build/scope.cxx
+++ b/build/scope.cxx
@@ -35,8 +35,8 @@ namespace build
   scope_map scopes;
   scope* root_scope;
 
-  scope& scope_map::
-  operator[] (const path& k)
+  pair<scope&, bool> scope_map::
+  insert (const path& k)
   {
     auto er (emplace (k, scope ()));
     scope& s (er.first->second);
@@ -79,7 +79,7 @@ namespace build
       s.init (er.first, p);
     }
 
-    return s;
+    return pair<scope&, bool> (s, er.second);
   }
 
   // Find the most qualified scope that encompasses this path.