Implement automatic loading of directory buildfiles

Now instead of explicitly writing:

d = foo/ bar/
./: $d
include $d

We can (usually) just write:

./: foo/ bar/

20 files changed, 290 insertions, 105 deletions

diff --git a/build2/config/operation.cxx b/build2/config/operation.cxx
index 68b61c3..e8ba1c2 100644
--- a/build2/config/operation.cxx
+++ b/build2/config/operation.cxx
@@ -571,6 +571,7 @@ namespace build2
                             dir_path (), // Out tree.
                             "",
                             nullopt,
+                            false,       // Real (not implied).
                             trace).first);
 
           if (!quiet)
diff --git a/build2/file b/build2/file
index 191e3dc..c8d8f6f 100644
--- a/build2/file
+++ b/build2/file
@@ -59,14 +59,14 @@ namespace build2
   source (const path& buildfile, scope& root, scope& base);
 
   // As above but first check if this buildfile has already been sourced for
-  // the base scope.
+  // the base scope. Return false if the file has already been sourced.
   //
-  void
+  bool
   source_once (const path& buildfile, scope& root, scope& base);
 
   // As above but checks against the specified scope rather than base.
   //
-  void
+  bool
   source_once (const path& buildfile, scope& root, scope& base, scope& once);
 
   // Create project's root scope. Only set the src_root variable if the
@@ -88,6 +88,14 @@ namespace build2
               const dir_path& out_base,
               const dir_path& src_base);
 
+  // Return a scope for the specified directory (first). Note that switching
+  // to this scope might also involve switch to a new root scope (second) if
+  // the new scope is in another project. In the new scope is not in any
+  // project, then NULL is returned in second.
+  //
+  pair<scope&, scope*>
+  switch_scope (scope& root, const dir_path&);
+
   // Bootstrap the project's root scope, the out part.
   //
   void
diff --git a/build2/file.cxx b/build2/file.cxx
index 2889cf7..527f752 100644
--- a/build2/file.cxx
+++ b/build2/file.cxx
@@ -114,10 +114,10 @@ namespace build2
   void
   source (const path& bf, scope& root, scope& base)
   {
-    return source (bf, root, base, false);
+    source (bf, root, base, false);
   }
 
-  void
+  bool
   source_once (const path& bf, scope& root, scope& base, scope& once)
   {
     tracer trace ("source_once");
@@ -125,10 +125,11 @@ namespace build2
     if (!once.buildfiles.insert (bf).second)
     {
       l5 ([&]{trace << "skipping already sourced " << bf;});
-      return;
+      return false;
     }
 
     source (bf, root, base);
+    return true;
   }
 
   scope&
@@ -253,6 +254,48 @@ namespace build2
     return s;
   }
 
+  pair<scope&, scope*>
+  switch_scope (scope& root, const dir_path& p)
+  {
+    // First, enter the scope into the map and see if it is in any project. If
+    // it is not, then there is nothing else to do.
+    //
+    auto i (scopes.insert (p, false));
+    scope& base (i->second);
+    scope* rs (base.root_scope ());
+
+    if (rs != nullptr)
+    {
+      // Path p can be src_base or out_base. Figure out which one it is.
+      //
+      dir_path out_base (p.sub (rs->out_path ()) ? p : src_out (p, *rs));
+
+      // Create and bootstrap root scope(s) of subproject(s) that this scope
+      // may belong to. If any were created, load them. Note that we need to
+      // do this before figuring out src_base since we may switch the root
+      // project (and src_root with it).
+      //
+      {
+        scope* nrs (&create_bootstrap_inner (*rs, out_base));
+
+        if (rs != nrs)
+          rs = nrs;
+      }
+
+      // Switch to the new root scope.
+      //
+      if (rs != &root)
+        load_root_pre (*rs); // Load new root(s) recursively.
+
+      // Now we can figure out src_base and finish setting the scope.
+      //
+      dir_path src_base (src_out (out_base, *rs));
+      setup_base (i, move (out_base), move (src_base));
+    }
+
+    return pair<scope&, scope*> (base, rs);
+  }
+
   void
   bootstrap_out (scope& root)
   {
diff --git a/build2/file.ixx b/build2/file.ixx
index 44eab91..050a3e9 100644
--- a/build2/file.ixx
+++ b/build2/file.ixx
@@ -4,7 +4,7 @@
 
 namespace build2
 {
-  inline void
+  inline bool
   source_once (const path& bf, scope& root, scope& base)
   {
     return source_once (bf, root, base, base);
diff --git a/build2/parser.cxx b/build2/parser.cxx
index 7997d10..9d118e8 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -87,6 +87,7 @@ namespace build2
     enter_target (parser& p,
                   name&& n,  // If n.pair, then o is out dir.
                   name&& o,
+                  bool implied,
                   const location& loc,
                   tracer& tr)
         : p_ (&p), t_ (p.target_)
@@ -130,8 +131,13 @@ namespace build2
 
       // Find or insert.
       //
-      p.target_ = &targets.insert (
-        *ti, move (d), move (out), move (n.value), move (e), tr).first;
+      p.target_ = &targets.insert (*ti,
+                                   move (d),
+                                   move (out),
+                                   move (n.value),
+                                   move (e),
+                                   implied,
+                                   tr).first;
     }
 
     ~enter_target ()
@@ -497,7 +503,8 @@ namespace build2
                 if (p == string::npos)
                 {
                   name o (n.pair ? move (*++i) : name ());
-                  enter_target tg (*this, move (n), move (o), nloc, trace);
+                  enter_target tg (
+                    *this, move (n), move (o), true, nloc, trace);
                   parse_variable (t, tt, var, att);
                 }
                 else
@@ -646,7 +653,7 @@ namespace build2
 
               // @@ OUT TODO
               //
-              enter_target tg (*this, move (tn), name (), nloc, trace);
+              enter_target tg (*this, move (tn), name (), false, nloc, trace);
 
               if (default_target_ == nullptr)
                 default_target_ = target_;
@@ -3477,7 +3484,8 @@ namespace build2
     else if (!qual.empty ())
       // @@ OUT TODO
       //
-      tg = enter_target (*this, move (qual), build2::name (), loc, trace);
+      tg = enter_target (
+        *this, move (qual), build2::name (), true, loc, trace);
 
     // Lookup.
     //
@@ -3496,50 +3504,18 @@ namespace build2
   }
 
   void parser::
-  switch_scope (const dir_path& p)
+  switch_scope (const dir_path& d)
   {
     tracer trace ("parser::switch_scope", &path_);
 
-    // First, enter the scope into the map and see if it is in any project. If
-    // it is not, then there is nothing else to do.
-    //
-    auto i (scopes.insert (p, false));
-    scope_ = &i->second;
-    scope* rs (scope_->root_scope ());
-
-    if (rs == nullptr)
-      return;
-
-    // Path p can be src_base or out_base. Figure out which one it is.
-    //
-    dir_path out_base (p.sub (rs->out_path ()) ? p : src_out (p, *rs));
-
-    // Create and bootstrap root scope(s) of subproject(s) that this
-    // scope may belong to. If any were created, load them. Note that
-    // we need to do this before figuring out src_base since we may
-    // switch the root project (and src_root with it).
-    //
-    {
-      scope* nrs (&create_bootstrap_inner (*rs, out_base));
-
-      if (rs != nrs)
-        rs = nrs;
-    }
+    auto p (build2::switch_scope (*root_, d));
+    scope_ = &p.first;
 
-    // Switch to the new root scope.
-    //
-    if (rs != root_)
+    if (p.second != nullptr && p.second != root_)
     {
-      load_root_pre (*rs); // Load new root(s) recursively.
-
-      l5 ([&]{trace << "switching to root scope " << rs->out_path ();});
-      root_ = rs;
+      root_ = p.second;
+      l5 ([&]{trace << "switching to root scope " << root_->out_path ();});
     }
-
-    // Now we can figure out src_base and finish setting the scope.
-    //
-    dir_path src_base (src_out (out_base, *rs));
-    setup_base (i, move (out_base), move (src_base));
   }
 
   void parser::
@@ -3573,6 +3549,7 @@ namespace build2
                       dir_path (),
                       string (),
                       nullopt,
+                      false,         // Enter as real (not implied).
                       trace).first);
 
     ct.prerequisites.emplace_back (dt, ct);
diff --git a/build2/search.cxx b/build2/search.cxx
index 50318a7..ef4814d 100644
--- a/build2/search.cxx
+++ b/build2/search.cxx
@@ -159,7 +159,7 @@ namespace build2
     // Find or insert. Note that we are using our updated extension.
     //
     auto r (targets.insert (
-              *tk.type, move (d), move (out), *tk.name, ext, trace));
+              *tk.type, move (d), move (out), *tk.name, ext, false, trace));
 
     // Has to be a file_target.
     //
@@ -203,7 +203,7 @@ namespace build2
     // @@ OUT: same story as in search_existing_target() re out.
     //
     auto r (targets.insert (
-              *tk.type, move (d), *tk.out, *tk.name, tk.ext, trace));
+              *tk.type, move (d), *tk.out, *tk.name, tk.ext, false, trace));
     assert (r.second);
 
     target& t (r.first);
diff --git a/build2/target b/build2/target
index a41c5b9..3bc9d5c 100644
--- a/build2/target
+++ b/build2/target
@@ -378,8 +378,7 @@ namespace build2
     value&
     assign (string name) {return vars.assign<T> (move (name)).first.get ();}
 
-    // Return a value suitable for appending. See class scope for
-    // details.
+    // Return a value suitable for appending. See class scope for details.
     //
     value&
     append (const variable&);
@@ -390,6 +389,15 @@ namespace build2
       return append (var_pool[name]);
     }
 
+    // A target that is not (yet) entered as part of a real dependency
+    // declaration (for example, that is entered as part of a target-specific
+    // variable assignment) is called implied.
+    //
+  public:
+    bool implied;
+
+    // Target state.
+    //
   public:
     target_state raw_state = target_state::unknown;
 
@@ -498,11 +506,12 @@ namespace build2
       }
     }
 
+    // Recipe.
+    //
   public:
-    action_type action; // Action this recipe is for.
+    using recipe_type = build2::recipe;
 
-  public:
-    typedef build2::recipe recipe_type;
+    action_type action; // Action this recipe is for.
 
     const recipe_type&
     recipe (action_type a) const {return a > action ? empty_recipe : recipe_;}
@@ -510,6 +519,9 @@ namespace build2
     void
     recipe (action_type, recipe_type);
 
+  private:
+    recipe_type recipe_;
+
     // Target type info.
     //
   public:
@@ -547,13 +559,8 @@ namespace build2
     // The only way to create a target should be via the targets set below.
     //
   public:
-    friend class target_set;
-
     target (dir_path d, dir_path o, string n, optional<string> e)
         : dir (move (d)), out (move (o)), name (move (n)), ext (move (e)) {}
-
-  private:
-    recipe_type recipe_;
   };
 
   // All targets are from the targets set below.
@@ -1010,6 +1017,7 @@ namespace build2
             dir_path out,
             string name,
             optional<string> ext,
+            bool implied,
             tracer&);
 
     template <typename T>
@@ -1022,7 +1030,13 @@ namespace build2
             tracer& t)
     {
       return static_cast<T&> (
-        insert (tt, move (dir), move (out), move (name), move (ext), t).first);
+        insert (tt,
+                move (dir),
+                move (out),
+                move (name),
+                move (ext),
+                false,       // Always real (not implied).
+                t).first);
     }
 
     template <typename T>
@@ -1034,7 +1048,7 @@ namespace build2
             tracer& t)
     {
       return static_cast<T&> (
-        insert (T::static_type, dir, out, name, ext, t).first);
+        insert (T::static_type, dir, out, name, ext, false, t).first);
     }
 
     template <typename T>
@@ -1045,7 +1059,7 @@ namespace build2
             tracer& t)
     {
       return static_cast<T&> (
-        insert (T::static_type, dir, out, name, nullopt, t).first);
+        insert (T::static_type, dir, out, name, nullopt, false, t).first);
     }
 
     void
diff --git a/build2/target.cxx b/build2/target.cxx
index 40967c3..0a97566 100644
--- a/build2/target.cxx
+++ b/build2/target.cxx
@@ -6,9 +6,11 @@
 
 #include <butl/filesystem> // file_mtime()
 
+#include <build2/file>
 #include <build2/scope>
 #include <build2/search>
 #include <build2/algorithm>
+#include <build2/filesystem>
 #include <build2/diagnostics>
 
 using namespace std;
@@ -239,6 +241,7 @@ namespace build2
           dir_path out,
           string name,
           optional<string> ext,
+          bool implied,
           tracer& trace)
   {
     iterator i (find (target_key {&tt, &dir, &out, &name, ext}, trace));
@@ -248,10 +251,22 @@ namespace build2
     {
       unique_ptr<target> pt (
         tt.factory (tt, move (dir), move (out), move (name), move (ext)));
+
+      pt->implied = implied;
+
       i = map_.emplace (
         make_pair (target_key {&tt, &pt->dir, &pt->out, &pt->name, pt->ext},
                    move (pt))).first;
     }
+    else if (!implied)
+    {
+      // Clear the implied flag.
+      //
+      target& t (**i);
+
+      if (t.implied)
+        t.implied = false;
+    }
 
     return pair<target&, bool> (**i, r);
   }
@@ -442,20 +457,6 @@ namespace build2
     return pk.tk.dir->relative () ? search_existing_file (pk) : nullptr;
   }
 
-  static target*
-  search_alias (const prerequisite_key& pk)
-  {
-    // For an alias we don't want to silently create a target since it
-    // will do nothing and it most likely not what the user intended.
-    //
-    target* t (search_existing_target (pk));
-
-    if (t == nullptr)
-      fail << "no explicit target for prerequisite " << pk;
-
-    return t;
-  }
-
   optional<string>
   target_extension_null (const target_key&, scope&, bool)
   {
@@ -556,6 +557,20 @@ namespace build2
     false
   };
 
+  static target*
+  search_alias (const prerequisite_key& pk)
+  {
+    // For an alias we don't want to silently create a target since it will do
+    // nothing and it most likely not what the user intended.
+    //
+    target* t (search_existing_target (pk));
+
+    if (t == nullptr || t->implied)
+      fail << "no explicit target for prerequisite " << pk;
+
+    return t;
+  }
+
   const target_type alias::static_type
   {
     "alias",
@@ -567,6 +582,80 @@ namespace build2
     false
   };
 
+  static target*
+  search_dir (const prerequisite_key& pk)
+  {
+    tracer trace ("search_dir");
+
+    // The first step is like in search_alias(): looks for an existing target.
+    //
+    target* t (search_existing_target (pk));
+
+    if (t != nullptr && !t->implied)
+      return t;
+
+    // If not found (or is implied), then try to load the corresponding
+    // buildfile which would normally define this target.
+    //
+    const dir_path& d (*pk.tk.dir);
+
+    // We only do this for relative paths.
+    //
+    if (d.relative ())
+    {
+      // Note: this code is a custom version of parser::parse_include().
+
+      scope& s (*pk.scope);
+
+      // Calculate the new out_base.
+      //
+      dir_path out_base (s.out_path () / d);
+      out_base.normalize ();
+
+      // In our world modifications to the scope structure during search &
+      // match should be "pure" in the sense that they should not affect any
+      // existing targets that have already been searched & matched.
+      //
+      // A straightforward way to enforce this is to not allow any existing
+      // targets to be inside any newly created scopes (except, perhaps for
+      // the directory target itself which we know hasn't been searched yet).
+      // This, however, is not that straightforward to implement: we would
+      // need to keep a directory prefix map for all the targets (e.g., in
+      // target_set). Also, a buildfile could load from a directory that is
+      // not a subdirectory of out_base. So for now we just assume that this
+      // is so. And so it is.
+
+      pair<scope&, scope*> sp (switch_scope (*s.root_scope (), out_base));
+
+      if (sp.second != nullptr) // Ignore scopes out of any project.
+      {
+        scope& base (sp.first);
+        scope& root (*sp.second);
+
+        path bf (base.src_path () / "buildfile");
+
+        if (exists (bf))
+        {
+          l5 ([&]{trace << "loading buildfile " << bf << " for " << pk;});
+
+          if (source_once (bf, root, base, root))
+          {
+            // If we loaded the buildfile, examine the target again.
+            //
+            if (t == nullptr)
+              t = search_existing_target (pk);
+
+            if (t != nullptr && !t->implied)
+              return t;
+          }
+        }
+      }
+    }
+
+    fail << "no explicit target for prerequisite " << pk <<
+      info << "did you forget to include the corresponding buildfile?" << endf;
+  }
+
   const target_type dir::static_type
   {
     "dir",
@@ -574,7 +663,7 @@ namespace build2
     &target_factory<dir>,
     nullptr, // Extension not used.
     nullptr,
-    &search_alias,
+    &search_dir,
     false
   };
 
diff --git a/buildfile b/buildfile
index 54cd0ff..8f52e29 100644
--- a/buildfile
+++ b/buildfile
@@ -2,15 +2,11 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = build2/ tests/ unit-tests/ doc/
-
-./: $d \
+./: build2/ tests/ unit-tests/ doc/ \
 doc{INSTALL LICENSE NEWS README version} \
 file{bootstrap.sh bootstrap-msvc.bat bootstrap-mingw.bat} \
 file{INSTALL.cli config.guess config.sub manifest}
 
-include $d
-
 # Don't install tests or the INSTALL file.
 #
 dir{tests/}:      install = false
diff --git a/tests/buildfile b/tests/buildfile
index c0a50e3..0384c45 100644
--- a/tests/buildfile
+++ b/tests/buildfile
@@ -2,6 +2,6 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = directive/ eval/ expansion/ function/ test/ value/
-./: $d file{common.test}
-include $d
+./: directive/ eval/ expansion/ function/ search/ test/ value/ \
+file{common.test}
+
diff --git a/tests/common.test b/tests/common.test
index 2311ae1..d495da0 100644
--- a/tests/common.test
+++ b/tests/common.test
@@ -15,6 +15,6 @@ test.options += --jobs 1 --quiet --buildfile -
 
 # By default just load the buildfile.
 #
-if ($empty($test.arguments))
+if ($null($test.arguments))
   test.arguments = noop
 end
diff --git a/tests/function/buildfile b/tests/function/buildfile
index 4e043cf..8034f2b 100644
--- a/tests/function/buildfile
+++ b/tests/function/buildfile
@@ -2,6 +2,4 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = builtin/ path/
-./: $d
-include $d
+./: builtin/ path/
diff --git a/tests/function/builtin/testscript b/tests/function/builtin/testscript
index 82df683..0b005f3 100644
--- a/tests/function/builtin/testscript
+++ b/tests/function/builtin/testscript
@@ -1,4 +1,4 @@
-# file      : tests/function/path/testscript
+# file      : tests/function/builtin/testscript
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
diff --git a/tests/search/buildfile b/tests/search/buildfile
new file mode 100644
index 0000000..a72ca71
--- /dev/null
+++ b/tests/search/buildfile
@@ -0,0 +1,5 @@
+# file      : tests/search/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: dir/
diff --git a/tests/search/dir/buildfile b/tests/search/dir/buildfile
new file mode 100644
index 0000000..03afc85
--- /dev/null
+++ b/tests/search/dir/buildfile
@@ -0,0 +1,8 @@
+# file      : tests/search/dir/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Test loading of dir{} buildfiles during target search.
+#
+
+./: test{testscript} $b
diff --git a/tests/search/dir/testscript b/tests/search/dir/testscript
new file mode 100644
index 0000000..4c427b2
--- /dev/null
+++ b/tests/search/dir/testscript
@@ -0,0 +1,51 @@
+# file      : tests/search/dir/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+test.arguments = 'update(../)'
+
+.include ../../common.test
+
+# foo/ has no buildfile
+# bar/ has valid buildfile
+# baz/ has invalid buildfile
+#
++mkdir foo bar baz
++cat <<EOI >>>bar/buildfile
+print bar
+./:
+EOI
++cat <'assert false' >>>baz/buildfile
+
+: no-buildfile
+:
+$* <'./: foo/' 2>>/EOE != 0
+error: no explicit target for prerequisite ../:dir{foo/}
+  info: did you forget to include the corresponding buildfile?
+info: while applying rule alias to update dir{../}
+EOE
+
+: basic
+:
+$* <'./: bar/' >'bar'
+
+: existing-scope
+:
+$* <<EOI >'bar'
+bar/: x = y
+./: bar/
+EOI
+
+: existing-target-implied
+:
+$* <<EOI >'bar'
+dir{bar/}: x = y
+./: bar/
+EOI
+
+: existing-target-real
+:
+$* <<EOI
+dir{baz/}:
+./: baz/
+EOI
diff --git a/tests/test/buildfile b/tests/test/buildfile
index 85ae8be..e01048f 100644
--- a/tests/test/buildfile
+++ b/tests/test/buildfile
@@ -2,6 +2,6 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = config-test/ script/ script-integration/
-./: $d file{common.test}
-include $d
+./: config-test/ script/ script-integration/ \
+file{common.test}
+
diff --git a/tests/test/script/buildfile b/tests/test/script/buildfile
index b57c983..8ce3cc7 100644
--- a/tests/test/script/buildfile
+++ b/tests/test/script/buildfile
@@ -2,6 +2,5 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = builtin/ runner/
-./: $d file{common.test}
-include $d
+./: builtin/ runner/ file{common.test}
+
diff --git a/unit-tests/buildfile b/unit-tests/buildfile
index 7f1f83d..0e39bfe 100644
--- a/unit-tests/buildfile
+++ b/unit-tests/buildfile
@@ -2,6 +2,4 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = function/ lexer/ scheduler/ test/script/
-./: $d
-include $d
+./: function/ lexer/ scheduler/ test/script/
diff --git a/unit-tests/test/script/buildfile b/unit-tests/test/script/buildfile
index cc5d36b..7e2aa9b 100644
--- a/unit-tests/test/script/buildfile
+++ b/unit-tests/test/script/buildfile
@@ -2,6 +2,4 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-d = lexer/ parser/ regex/
-./: $d
-include $d
+./: lexer/ parser/ regex/